From ec7ee6c2a6618e049c70d5a5578c563f230339b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sat, 18 Jul 2026 20:07:25 -0400 Subject: [PATCH 01/12] chore(labrinth): bump to redis 1.4.1 --- Cargo.lock | 83 ++++++++++++------------ Cargo.toml | 4 +- apps/labrinth/src/database/redis/mod.rs | 16 +++-- apps/labrinth/src/queue/analytics/mod.rs | 24 ++++--- apps/labrinth/src/sync/friends.rs | 4 +- apps/labrinth/src/sync/status.rs | 2 +- 6 files changed, 75 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c3493e8e0..f8651bf97d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -244,7 +244,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.6.1", + "socket2 0.6.5", "time", "tracing", "url", @@ -458,6 +458,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + [[package]] name = "arg_enum_proc_macro" version = "0.3.4" @@ -2801,7 +2807,7 @@ checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" [[package]] name = "deadpool" version = "0.12.3" -source = "git+https://github.com/modrinth/deadpool?rev=db5fb00b036ecc8fe5f18853c559b745ffe47bde#db5fb00b036ecc8fe5f18853c559b745ffe47bde" +source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b" dependencies = [ "deadpool-runtime", "num_cpus", @@ -2811,8 +2817,8 @@ dependencies = [ [[package]] name = "deadpool-redis" -version = "0.22.1" -source = "git+https://github.com/modrinth/deadpool?rev=db5fb00b036ecc8fe5f18853c559b745ffe47bde#db5fb00b036ecc8fe5f18853c559b745ffe47bde" +version = "0.23.0" +source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b" dependencies = [ "deadpool", "redis", @@ -2822,7 +2828,7 @@ dependencies = [ [[package]] name = "deadpool-runtime" version = "0.1.5" -source = "git+https://github.com/modrinth/deadpool?rev=db5fb00b036ecc8fe5f18853c559b745ffe47bde#db5fb00b036ecc8fe5f18853c559b745ffe47bde" +source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b" dependencies = [ "tokio", ] @@ -4765,7 +4771,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -5106,17 +5112,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "io-uring" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "libc", -] - [[package]] name = "ipconfig" version = "0.3.2" @@ -5575,7 +5570,7 @@ dependencies = [ "quoted_printable", "rustls 0.23.32", "rustls-native-certs 0.8.1", - "socket2 0.6.1", + "socket2 0.6.5", "tokio", "tokio-rustls 0.26.4", "url", @@ -5613,9 +5608,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libfuzzer-sys" @@ -6029,14 +6024,14 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.4" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "log", "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7955,7 +7950,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.32", - "socket2 0.6.1", + "socket2 0.5.10", "thiserror 2.0.17", "tokio", "tracing", @@ -7992,7 +7987,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -8277,11 +8272,13 @@ dependencies = [ [[package]] name = "redis" -version = "0.32.7" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44" +checksum = "b0b9503711b03773e43b31668c7b5bd279ee7cd9b7d18cff7c23a42cc1d08e5a" dependencies = [ "ahash 0.8.12", + "arcstr", + "async-lock", "bytes", "cfg-if", "combine", @@ -8293,10 +8290,11 @@ dependencies = [ "r2d2", "ryu", "sha1_smol", - "socket2 0.6.1", + "socket2 0.6.5", "tokio", "tokio-util", "url", + "xxhash-rust", ] [[package]] @@ -9749,12 +9747,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -11193,30 +11191,27 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.1" +version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.6.1", + "socket2 0.6.5", "tokio-macros", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", @@ -11420,7 +11415,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2 0.6.1", + "socket2 0.6.5", "sync_wrapper", "tokio", "tokio-stream", @@ -13350,6 +13345,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xxhash-rust" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" + [[package]] name = "yaserde" version = "0.12.0" diff --git a/Cargo.toml b/Cargo.toml index a5aed6cf76..5bca9c5ee7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,7 @@ daedalus = { path = "packages/daedalus" } darling = { version = "0.23" } dashmap = "6.1.0" data-url = "0.3.2" -deadpool-redis = { git = "https://github.com/modrinth/deadpool", rev = "db5fb00b036ecc8fe5f18853c559b745ffe47bde", version = "0.22.1" } +deadpool-redis = { git = "https://github.com/modrinth/deadpool", rev = "c7209b086829572f368f1194aba54bdea8dc540b", version = "0.23.0" } derive_more = "2.1.1" directories = "6.0.0" dirs = "6.0.0" @@ -150,7 +150,7 @@ quote = { version = "1.0" } rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 rdkafka = { version = "0.36.2", features = ["cmake-build"] } -redis = "0.32.7" +redis = "1.4.1" regex = "1.12.2" reqwest = { version = "0.12.24", default-features = false } rgb = "0.8.52" diff --git a/apps/labrinth/src/database/redis/mod.rs b/apps/labrinth/src/database/redis/mod.rs index 833ed1ac40..acab21af1e 100644 --- a/apps/labrinth/src/database/redis/mod.rs +++ b/apps/labrinth/src/database/redis/mod.rs @@ -4,7 +4,7 @@ use super::models::DatabaseError; use ariadne::ids::base62_impl::{parse_base62, to_base62}; use chrono::{TimeZone, Utc}; use dashmap::DashMap; -use deadpool_redis::{Config, Runtime}; +use deadpool_redis::Runtime; use futures::TryStreamExt; use futures::future::Either; use futures::stream::{FuturesUnordered, StreamExt}; @@ -178,9 +178,17 @@ impl RedisPool { let wait_timeout = Duration::from_millis(ENV.REDIS_WAIT_TIMEOUT_MS); let url = &ENV.REDIS_URL; - let pool = Config::from_url(url.clone()) - .builder() - .expect("Error building Redis pool") + // Redis 1.x enables async connection and response timeouts by default. + // Preserve the previous behavior because this pool still serves BRPOP + let connection_config = redis::AsyncConnectionConfig::new() + .set_connection_timeout(None) + .set_response_timeout(None); + let manager = deadpool_redis::Manager::new_with_config( + url.clone(), + connection_config, + ) + .expect("Error building Redis pool"); + let pool = deadpool_redis::Pool::builder(manager) .max_size(ENV.REDIS_MAX_CONNECTIONS as usize) .wait_timeout(Some(wait_timeout)) .runtime(Runtime::Tokio1) diff --git a/apps/labrinth/src/queue/analytics/mod.rs b/apps/labrinth/src/queue/analytics/mod.rs index 79b02b6cd5..090145765f 100644 --- a/apps/labrinth/src/queue/analytics/mod.rs +++ b/apps/labrinth/src/queue/analytics/mod.rs @@ -175,9 +175,11 @@ impl AnalyticsQueue { MINECRAFT_SERVER_PLAYS_EXPIRY, ); } - pipe.query_async::<()>(&mut *redis) - .await - .map_err(DatabaseError::CacheError)?; + if !pipe.is_empty() { + pipe.query_async::<()>(&mut *redis) + .await + .map_err(DatabaseError::CacheError)?; + } let mut plays = client .insert::(MINECRAFT_SERVER_PLAYS) @@ -243,9 +245,11 @@ impl AnalyticsQueue { 6 * 60 * 60, ); } - pipe.query_async::<()>(&mut *redis) - .await - .map_err(DatabaseError::CacheError)?; + if !pipe.is_empty() { + pipe.query_async::<()>(&mut *redis) + .await + .map_err(DatabaseError::CacheError)?; + } let mut views = client.insert::("views").await?; @@ -311,9 +315,11 @@ impl AnalyticsQueue { 6 * 60 * 60, ); } - pipe.query_async::<()>(&mut *redis) - .await - .map_err(DatabaseError::CacheError)?; + if !pipe.is_empty() { + pipe.query_async::<()>(&mut *redis) + .await + .map_err(DatabaseError::CacheError)?; + } let mut transaction = pool.begin().await?; let mut downloads = client.insert::("downloads").await?; diff --git a/apps/labrinth/src/sync/friends.rs b/apps/labrinth/src/sync/friends.rs index 770cb1d6c9..35f7141e66 100644 --- a/apps/labrinth/src/sync/friends.rs +++ b/apps/labrinth/src/sync/friends.rs @@ -11,7 +11,7 @@ use ariadne::ids::UserId; use ariadne::networking::message::ServerToClientMessage; use ariadne::users::UserStatus; use redis::aio::PubSub; -use redis::{RedisWrite, ToRedisArgs}; +use redis::{RedisWrite, ToRedisArgs, ToSingleRedisArg}; use serde::{Deserialize, Serialize}; use tokio_stream::StreamExt; @@ -44,6 +44,8 @@ impl ToRedisArgs for RedisFriendsMessage { } } +impl ToSingleRedisArg for RedisFriendsMessage {} + pub async fn handle_pubsub( mut pubsub: PubSub, pool: PgPool, diff --git a/apps/labrinth/src/sync/status.rs b/apps/labrinth/src/sync/status.rs index f16ed3702b..0d7892a57a 100644 --- a/apps/labrinth/src/sync/status.rs +++ b/apps/labrinth/src/sync/status.rs @@ -18,7 +18,7 @@ pub async fn get_user_status( if let Ok(mut conn) = redis.pool.get().await && let Ok(mut statuses) = conn.sscan::<_, Vec>(get_field_name(user)).await - && let Some(status) = statuses.next_item().await + && let Some(Ok(status)) = statuses.next_item().await { return postcard::from_bytes::(&status).ok(); } From 032b1e705a332ab2a76220a6cb2bbc0eb2ad66c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sat, 18 Jul 2026 23:16:40 -0400 Subject: [PATCH 02/12] feat(labrinth): redis cluster --- .github/workflows/turbo-ci.yml | 5 +- Cargo.lock | 9 + apps/labrinth/.env.docker-compose | 3 + apps/labrinth/.env.local | 8 + apps/labrinth/Cargo.toml | 10 +- .../database/models/analytics_event_item.rs | 27 +- .../src/database/models/categories.rs | 40 +- .../src/database/models/collection_item.rs | 3 +- .../labrinth/src/database/models/flow_item.rs | 14 +- .../src/database/models/image_item.rs | 3 +- .../database/models/legacy_loader_fields.rs | 6 +- .../src/database/models/loader_fields.rs | 62 +- .../database/models/moderation_note_item.rs | 80 +- .../src/database/models/notification_item.rs | 35 +- .../models/notifications_template_item.rs | 55 +- .../models/notifications_type_item.rs | 14 +- .../src/database/models/organization_item.rs | 18 +- apps/labrinth/src/database/models/pat_item.rs | 44 +- .../src/database/models/product_item.rs | 8 +- .../src/database/models/project_item.rs | 53 +- .../src/database/models/session_item.rs | 54 +- .../labrinth/src/database/models/team_item.rs | 3 +- .../labrinth/src/database/models/user_item.rs | 58 +- .../src/database/models/version_item.rs | 45 +- apps/labrinth/src/database/redis/blocking.rs | 153 +++ apps/labrinth/src/database/redis/cache.rs | 614 ++++++++++++ apps/labrinth/src/database/redis/commands.rs | 190 ++++ apps/labrinth/src/database/redis/config.rs | 217 ++++ .../labrinth/src/database/redis/connection.rs | 266 +++++ apps/labrinth/src/database/redis/key.rs | 72 ++ apps/labrinth/src/database/redis/metrics.rs | 161 +++ apps/labrinth/src/database/redis/mod.rs | 935 +++--------------- apps/labrinth/src/database/redis/pubsub.rs | 129 +++ apps/labrinth/src/database/redis/util.rs | 32 - apps/labrinth/src/env.rs | 5 + apps/labrinth/src/lib.rs | 7 +- apps/labrinth/src/main.rs | 2 +- apps/labrinth/src/models/exp/project.rs | 32 +- apps/labrinth/src/queue/analytics/cache.rs | 10 +- apps/labrinth/src/queue/analytics/mod.rs | 137 ++- apps/labrinth/src/queue/server_ping.rs | 32 +- apps/labrinth/src/routes/internal/campaign.rs | 15 +- apps/labrinth/src/routes/internal/flows.rs | 66 +- .../labrinth/src/routes/internal/gotenberg.rs | 16 +- apps/labrinth/src/routes/internal/statuses.rs | 9 +- apps/labrinth/src/routes/v3/content/mod.rs | 35 +- apps/labrinth/src/search/mod.rs | 16 +- apps/labrinth/src/sync/friends.rs | 14 +- apps/labrinth/src/sync/status.rs | 26 +- apps/labrinth/src/test/database.rs | 4 +- apps/labrinth/src/util/gotenberg.rs | 30 +- apps/labrinth/src/util/mod.rs | 1 - apps/labrinth/src/util/ratelimit.rs | 10 +- apps/labrinth/src/util/redis.rs | 18 - apps/labrinth/tests/project.rs | 23 +- apps/labrinth/tests/v2/project.rs | 9 +- apps/labrinth/tests/version.rs | 9 +- docker-compose.yml | 252 +++++ turbo.jsonc | 1 + 59 files changed, 2779 insertions(+), 1426 deletions(-) create mode 100644 apps/labrinth/src/database/redis/blocking.rs create mode 100644 apps/labrinth/src/database/redis/cache.rs create mode 100644 apps/labrinth/src/database/redis/commands.rs create mode 100644 apps/labrinth/src/database/redis/config.rs create mode 100644 apps/labrinth/src/database/redis/connection.rs create mode 100644 apps/labrinth/src/database/redis/key.rs create mode 100644 apps/labrinth/src/database/redis/metrics.rs create mode 100644 apps/labrinth/src/database/redis/pubsub.rs delete mode 100644 apps/labrinth/src/util/redis.rs diff --git a/.github/workflows/turbo-ci.yml b/.github/workflows/turbo-ci.yml index ad4147a313..a3533e0358 100644 --- a/.github/workflows/turbo-ci.yml +++ b/.github/workflows/turbo-ci.yml @@ -82,6 +82,9 @@ jobs: SCCACHE_DIR: ${{ needs.skip-if-clean.outputs.internal == 'true' && '/mnt/sccache' || '' }} SCCACHE_CACHE_SIZE: ${{ needs.skip-if-clean.outputs.internal == 'true' && '10G' || '' }} RUSTC_WRAPPER: ${{ needs.skip-if-clean.outputs.internal == 'true' && 'sccache' || '' }} + REDIS_MODE: cluster + REDIS_CONNECTION_TYPE: multiplexed + REDIS_URL: redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005 steps: - name: Check out code @@ -185,7 +188,7 @@ jobs: - name: Start services if: steps.check-labrinth.outputs.needs_services == 'true' - run: docker compose up --wait + run: docker compose --profile clustered-redis up --wait - name: Setup labrinth environment and database if: steps.check-labrinth.outputs.needs_services == 'true' diff --git a/Cargo.lock b/Cargo.lock index f8651bf97d..82040aa753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2484,6 +2484,12 @@ dependencies = [ "spin 0.10.0", ] +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + [[package]] name = "crc32fast" version = "1.5.0" @@ -8282,12 +8288,15 @@ dependencies = [ "bytes", "cfg-if", "combine", + "crc16", "futures-util", "itoa", + "log", "num-bigint", "percent-encoding", "pin-project-lite", "r2d2", + "rand 0.10.1", "ryu", "sha1_smol", "socket2 0.6.5", diff --git a/apps/labrinth/.env.docker-compose b/apps/labrinth/.env.docker-compose index 55bc2f232b..f806260962 100644 --- a/apps/labrinth/.env.docker-compose +++ b/apps/labrinth/.env.docker-compose @@ -29,9 +29,12 @@ TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth +REDIS_MODE=standalone +REDIS_CONNECTION_TYPE=pooled REDIS_URL=redis://labrinth-redis REDIS_MIN_CONNECTIONS=0 REDIS_MAX_CONNECTIONS=10000 +REDIS_BLOCKING_MAX_CONNECTIONS=8 KAFKA_BOOTSTRAP_SERVERS=redpanda:9092 KAFKA_CLIENT_ID=labrinth diff --git a/apps/labrinth/.env.local b/apps/labrinth/.env.local index 364f90f1e5..2ebc8fad8b 100644 --- a/apps/labrinth/.env.local +++ b/apps/labrinth/.env.local @@ -47,9 +47,17 @@ TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth +REDIS_MODE=standalone +REDIS_CONNECTION_TYPE=pooled REDIS_URL=redis://localhost REDIS_MIN_CONNECTIONS=0 REDIS_MAX_CONNECTIONS=10000 +REDIS_BLOCKING_MAX_CONNECTIONS=8 + +# For a clustered Redis setup (`clustered-redis` Docker Compose profile) +# REDIS_MODE=cluster +# REDIS_CONNECTION_TYPE=multiplexed +# REDIS_URL=redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005 KAFKA_BOOTSTRAP_SERVERS=localhost:19092 KAFKA_CLIENT_ID=labrinth diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 3da97e6fa8..c7f6e9f4e8 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -41,7 +41,7 @@ color-eyre = { workspace = true } color-thief = { workspace = true } const_format = { workspace = true } dashmap = { workspace = true } -deadpool-redis.workspace = true +deadpool-redis = { workspace = true, features = ["cluster-async"] } derive_more = { workspace = true, features = ["deref", "deref_mut"] } dotenvy = { workspace = true } either = { workspace = true } @@ -87,7 +87,13 @@ quick-xml = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } rdkafka = { workspace = true } -redis = { workspace = true, features = ["ahash", "r2d2", "tokio-comp"] } +redis = { workspace = true, features = [ + "ahash", + "cluster", + "cluster-async", + "r2d2", + "tokio-comp" +] } regex = { workspace = true } reqwest = { workspace = true, features = [ "http2", diff --git a/apps/labrinth/src/database/models/analytics_event_item.rs b/apps/labrinth/src/database/models/analytics_event_item.rs index df9555efd5..913872ff1f 100644 --- a/apps/labrinth/src/database/models/analytics_event_item.rs +++ b/apps/labrinth/src/database/models/analytics_event_item.rs @@ -86,14 +86,11 @@ impl DBAnalyticsEvent { redis: &RedisPool, ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY); - if let Some(events) = redis - .get_deserialized( - ANALYTICS_EVENTS_NAMESPACE, - ANALYTICS_EVENTS_ALL_KEY, - ) - .await? - { + if let Some(events) = redis.get_deserialized(&key).await? { return Ok(events); } @@ -118,23 +115,17 @@ impl DBAnalyticsEvent { .try_collect::>() .await?; - redis - .set_serialized( - ANALYTICS_EVENTS_NAMESPACE, - ANALYTICS_EVENTS_ALL_KEY, - &events, - None, - ) - .await?; + redis.set_serialized(&key, &events, None).await?; Ok(events) } pub async fn clear_cache(redis: &RedisPool) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - redis - .delete(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY) - .await?; + let key = redis + .keyspace() + .metadata(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY); + redis.delete(&key).await?; Ok(()) } } diff --git a/apps/labrinth/src/database/models/categories.rs b/apps/labrinth/src/database/models/categories.rs index 3b4a98dca4..60e6e5f988 100644 --- a/apps/labrinth/src/database/models/categories.rs +++ b/apps/labrinth/src/database/models/categories.rs @@ -95,9 +95,10 @@ impl Category { { { let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(TAGS_NAMESPACE, "category"); let res: Option> = - redis.get_deserialized(TAGS_NAMESPACE, "category").await?; + redis.get_deserialized(&key).await?; if let Some(res) = res { return Ok(res); @@ -124,10 +125,9 @@ impl Category { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(TAGS_NAMESPACE, "category"); - redis - .set_serialized(TAGS_NAMESPACE, "category", &result, None) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } @@ -163,10 +163,11 @@ impl LinkPlatform { { { let mut redis = redis.connect().await?; + let key = + redis.keyspace().metadata(TAGS_NAMESPACE, "link_platform"); - let res: Option> = redis - .get_deserialized(TAGS_NAMESPACE, "link_platform") - .await?; + let res: Option> = + redis.get_deserialized(&key).await?; if let Some(res) = res { return Ok(res); @@ -188,10 +189,9 @@ impl LinkPlatform { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(TAGS_NAMESPACE, "link_platform"); - redis - .set_serialized(TAGS_NAMESPACE, "link_platform", &result, None) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } @@ -227,10 +227,9 @@ impl ReportType { { { let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(TAGS_NAMESPACE, "report_type"); - let res: Option> = redis - .get_deserialized(TAGS_NAMESPACE, "report_type") - .await?; + let res: Option> = redis.get_deserialized(&key).await?; if let Some(res) = res { return Ok(res); @@ -248,10 +247,9 @@ impl ReportType { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(TAGS_NAMESPACE, "report_type"); - redis - .set_serialized(TAGS_NAMESPACE, "report_type", &result, None) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } @@ -287,10 +285,9 @@ impl ProjectType { { { let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(TAGS_NAMESPACE, "project_type"); - let res: Option> = redis - .get_deserialized(TAGS_NAMESPACE, "project_type") - .await?; + let res: Option> = redis.get_deserialized(&key).await?; if let Some(res) = res { return Ok(res); @@ -308,10 +305,9 @@ impl ProjectType { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(TAGS_NAMESPACE, "project_type"); - redis - .set_serialized(TAGS_NAMESPACE, "project_type", &result, None) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } diff --git a/apps/labrinth/src/database/models/collection_item.rs b/apps/labrinth/src/database/models/collection_item.rs index 43070397d1..0fad263ca9 100644 --- a/apps/labrinth/src/database/models/collection_item.rs +++ b/apps/labrinth/src/database/models/collection_item.rs @@ -217,8 +217,9 @@ impl DBCollection { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(COLLECTIONS_NAMESPACE, id.0); - redis.delete(COLLECTIONS_NAMESPACE, id.0).await?; + redis.delete(&key).await?; Ok(()) } } diff --git a/apps/labrinth/src/database/models/flow_item.rs b/apps/labrinth/src/database/models/flow_item.rs index b85f64bdd7..c44855d486 100644 --- a/apps/labrinth/src/database/models/flow_item.rs +++ b/apps/labrinth/src/database/models/flow_item.rs @@ -75,14 +75,10 @@ impl DBFlow { state: &str, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(FLOWS_NAMESPACE, state); redis - .set_serialized( - FLOWS_NAMESPACE, - &state, - &self, - Some(expires.num_seconds()), - ) + .set_serialized(&key, &self, Some(expires.num_seconds())) .await?; Ok(()) } @@ -107,8 +103,9 @@ impl DBFlow { redis: &RedisPool, ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(FLOWS_NAMESPACE, id); - redis.get_deserialized(FLOWS_NAMESPACE, id).await + redis.get_deserialized(&key).await } /// Gets the flow and removes it from the cache, but only removes if the flow was present and the predicate returned true @@ -132,8 +129,9 @@ impl DBFlow { redis: &RedisPool, ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(FLOWS_NAMESPACE, id); - redis.delete(FLOWS_NAMESPACE, id).await?; + redis.delete(&key).await?; Ok(Some(())) } } diff --git a/apps/labrinth/src/database/models/image_item.rs b/apps/labrinth/src/database/models/image_item.rs index 7f72f82bb1..a44f275912 100644 --- a/apps/labrinth/src/database/models/image_item.rs +++ b/apps/labrinth/src/database/models/image_item.rs @@ -229,8 +229,9 @@ impl DBImage { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(IMAGES_NAMESPACE, id.0); - redis.delete(IMAGES_NAMESPACE, id.0).await?; + redis.delete(&key).await?; Ok(()) } } diff --git a/apps/labrinth/src/database/models/legacy_loader_fields.rs b/apps/labrinth/src/database/models/legacy_loader_fields.rs index 84e012f0af..a0ec65a20c 100644 --- a/apps/labrinth/src/database/models/legacy_loader_fields.rs +++ b/apps/labrinth/src/database/models/legacy_loader_fields.rs @@ -220,11 +220,11 @@ impl<'a> MinecraftGameVersionBuilder<'a> { .await?; let mut conn = redis.connect().await?; - conn.delete( + let key = conn.keyspace().entity( crate::database::models::loader_fields::LOADER_FIELD_ENUM_VALUES_NAMESPACE, game_versions_enum.id.0, - ) - .await?; + ); + conn.delete(&key).await?; Ok(LoaderFieldEnumValueId(result.id)) } diff --git a/apps/labrinth/src/database/models/loader_fields.rs b/apps/labrinth/src/database/models/loader_fields.rs index 38f3befd55..09b82eaa9b 100644 --- a/apps/labrinth/src/database/models/loader_fields.rs +++ b/apps/labrinth/src/database/models/loader_fields.rs @@ -54,9 +54,9 @@ impl Game { { { let mut redis = redis.connect().await?; - let cached_games: Option> = redis - .get_deserialized(GAMES_LIST_NAMESPACE, "games") - .await?; + let key = redis.keyspace().metadata(GAMES_LIST_NAMESPACE, "games"); + let cached_games: Option> = + redis.get_deserialized(&key).await?; if let Some(cached_games) = cached_games { return Ok(cached_games); } @@ -79,10 +79,9 @@ impl Game { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(GAMES_LIST_NAMESPACE, "games"); - redis - .set_serialized(GAMES_LIST_NAMESPACE, "games", &result, None) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } @@ -109,8 +108,8 @@ impl Loader { { { let mut redis = redis.connect().await?; - let cached_id: Option = - redis.get_deserialized(LOADER_ID, name).await?; + let key = redis.keyspace().metadata(LOADER_ID, name); + let cached_id: Option = redis.get_deserialized(&key).await?; if let Some(cached_id) = cached_id { return Ok(Some(LoaderId(cached_id))); } @@ -129,9 +128,8 @@ impl Loader { if let Some(result) = result { let mut redis = redis.connect().await?; - redis - .set_serialized(LOADER_ID, name, &result.0, None) - .await?; + let key = redis.keyspace().metadata(LOADER_ID, name); + redis.set_serialized(&key, &result.0, None).await?; } Ok(result) @@ -146,9 +144,9 @@ impl Loader { { { let mut redis = redis.connect().await?; - let cached_loaders: Option> = redis - .get_deserialized(LOADERS_LIST_NAMESPACE, "all") - .await?; + let key = redis.keyspace().metadata(LOADERS_LIST_NAMESPACE, "all"); + let cached_loaders: Option> = + redis.get_deserialized(&key).await?; if let Some(cached_loaders) = cached_loaders { return Ok(cached_loaders); } @@ -187,10 +185,9 @@ impl Loader { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(LOADERS_LIST_NAMESPACE, "all"); - redis - .set_serialized(LOADERS_LIST_NAMESPACE, "all", &result, None) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } @@ -460,10 +457,11 @@ impl LoaderField { { { let mut redis = redis.connect().await?; + let key = + redis.keyspace().metadata(LOADER_FIELDS_NAMESPACE_ALL, ""); - let cached_fields: Option> = redis - .get_deserialized(LOADER_FIELDS_NAMESPACE_ALL, "") - .await?; + let cached_fields: Option> = + redis.get_deserialized(&key).await?; if let Some(cached_fields) = cached_fields { return Ok(cached_fields); @@ -494,10 +492,9 @@ impl LoaderField { .collect(); let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(LOADER_FIELDS_NAMESPACE_ALL, ""); - redis - .set_serialized(LOADER_FIELDS_NAMESPACE_ALL, "", &result, None) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } @@ -513,10 +510,11 @@ impl LoaderFieldEnum { { { let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name); - let cached_enum = redis - .get_deserialized(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name) - .await?; + let cached_enum = redis.get_deserialized(&key).await?; if let Some(cached_enum) = cached_enum { return Ok(cached_enum); } @@ -541,15 +539,11 @@ impl LoaderFieldEnum { }); let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name); - redis - .set_serialized( - LOADER_FIELD_ENUMS_ID_NAMESPACE, - enum_name, - &result, - None, - ) - .await?; + redis.set_serialized(&key, &result, None).await?; Ok(result) } diff --git a/apps/labrinth/src/database/models/moderation_note_item.rs b/apps/labrinth/src/database/models/moderation_note_item.rs index d20d7327e7..678a5de616 100644 --- a/apps/labrinth/src/database/models/moderation_note_item.rs +++ b/apps/labrinth/src/database/models/moderation_note_item.rs @@ -32,19 +32,17 @@ impl DBModerationNote { where E: crate::database::Executor<'a, Database = sqlx::Postgres>, { - let ids = user_ids - .iter() - .map(|id| id.0.to_string()) - .collect::>(); - let cached = { let mut redis = redis.connect().await?; - redis - .get_many_deserialized::( - MODERATION_NOTES_USERS_NAMESPACE, - &ids, - ) - .await? + let keys = user_ids + .iter() + .map(|id| { + redis + .keyspace() + .entity(MODERATION_NOTES_USERS_NAMESPACE, id.0) + }) + .collect::>(); + redis.get_many_deserialized::(&keys).await? }; let mut notes = HashMap::new(); @@ -86,14 +84,10 @@ impl DBModerationNote { }; if let Some(user_id) = note.user_id { - redis - .set_serialized( - MODERATION_NOTES_USERS_NAMESPACE, - user_id.0, - ¬e, - None, - ) - .await?; + let key = redis + .keyspace() + .entity(MODERATION_NOTES_USERS_NAMESPACE, user_id.0); + redis.set_serialized(&key, ¬e, None).await?; notes.insert(user_id, note); } } @@ -122,19 +116,17 @@ impl DBModerationNote { where E: crate::database::Executor<'a, Database = sqlx::Postgres>, { - let ids = organization_ids - .iter() - .map(|id| id.0.to_string()) - .collect::>(); - let cached = { let mut redis = redis.connect().await?; - redis - .get_many_deserialized::( - MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, - &ids, - ) - .await? + let keys = organization_ids + .iter() + .map(|id| { + redis + .keyspace() + .entity(MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, id.0) + }) + .collect::>(); + redis.get_many_deserialized::(&keys).await? }; let mut notes = HashMap::new(); @@ -176,14 +168,11 @@ impl DBModerationNote { }; if let Some(organization_id) = note.organization_id { - redis - .set_serialized( - MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, - organization_id.0, - ¬e, - None, - ) - .await?; + let key = redis.keyspace().entity( + MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, + organization_id.0, + ); + redis.set_serialized(&key, ¬e, None).await?; notes.insert(organization_id, note); } } @@ -289,9 +278,10 @@ impl DBModerationNote { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - redis - .delete(MODERATION_NOTES_USERS_NAMESPACE, user_id.0) - .await + let key = redis + .keyspace() + .entity(MODERATION_NOTES_USERS_NAMESPACE, user_id.0); + redis.delete(&key).await } pub async fn clear_organization_cache( @@ -299,8 +289,10 @@ impl DBModerationNote { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - redis - .delete(MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, organization_id.0) - .await + let key = redis.keyspace().entity( + MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, + organization_id.0, + ); + redis.delete(&key).await } } diff --git a/apps/labrinth/src/database/models/notification_item.rs b/apps/labrinth/src/database/models/notification_item.rs index 90891de843..b567c4cad7 100644 --- a/apps/labrinth/src/database/models/notification_item.rs +++ b/apps/labrinth/src/database/models/notification_item.rs @@ -433,13 +433,12 @@ impl DBNotification { { { let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0); - let cached_notifications: Option> = redis - .get_deserialized( - USER_NOTIFICATIONS_NAMESPACE, - &user_id.0.to_string(), - ) - .await?; + let cached_notifications: Option> = + redis.get_deserialized(&key).await?; if let Some(notifications) = cached_notifications { return Ok(notifications); @@ -491,15 +490,11 @@ impl DBNotification { .await?; let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0); - redis - .set_serialized( - USER_NOTIFICATIONS_NAMESPACE, - user_id.0, - &db_notifications, - None, - ) - .await?; + redis.set_serialized(&key, &db_notifications, None).await?; Ok(db_notifications) } @@ -638,12 +633,14 @@ impl DBNotification { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let keys = user_ids + .into_iter() + .map(|id| { + redis.keyspace().entity(USER_NOTIFICATIONS_NAMESPACE, id.0) + }) + .collect::>(); - redis - .delete_many(user_ids.into_iter().map(|id| { - (USER_NOTIFICATIONS_NAMESPACE, Some(id.0.to_string())) - })) - .await?; + redis.delete_many(&keys).await?; Ok(()) } diff --git a/apps/labrinth/src/database/models/notifications_template_item.rs b/apps/labrinth/src/database/models/notifications_template_item.rs index ee53bb93c5..2fef6ec4f8 100644 --- a/apps/labrinth/src/database/models/notifications_template_item.rs +++ b/apps/labrinth/src/database/models/notifications_template_item.rs @@ -55,10 +55,11 @@ impl NotificationTemplate { ) -> Result, DatabaseError> { { let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(TEMPLATES_NAMESPACE, channel.as_str()); - let maybe_cached_templates = redis - .get_deserialized(TEMPLATES_NAMESPACE, channel.as_str()) - .await?; + let maybe_cached_templates = redis.get_deserialized(&key).await?; if let Some(cached) = maybe_cached_templates { return Ok(cached); @@ -78,14 +79,12 @@ impl NotificationTemplate { let templates = results.into_iter().map(Into::into).collect(); let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(TEMPLATES_NAMESPACE, channel.as_str()); redis - .set_serialized( - TEMPLATES_NAMESPACE, - channel.as_str(), - &templates, - Some(TEMPLATES_CACHE_EXPIRY), - ) + .set_serialized(&key, &templates, Some(TEMPLATES_CACHE_EXPIRY)) .await?; Ok(templates) @@ -96,12 +95,10 @@ impl NotificationTemplate { redis: &RedisPool, ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; - redis - .get_deserialized( - TEMPLATES_HTML_DATA_NAMESPACE, - &self.id.to_string(), - ) - .await + let key = redis + .keyspace() + .metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id); + redis.get_deserialized(&key).await } pub async fn set_cached_html_data( @@ -110,13 +107,11 @@ impl NotificationTemplate { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id); redis - .set_serialized( - TEMPLATES_HTML_DATA_NAMESPACE, - &self.id.to_string(), - &data, - Some(HTML_DATA_CACHE_EXPIRY), - ) + .set_serialized(&key, &data, Some(HTML_DATA_CACHE_EXPIRY)) .await } } @@ -135,9 +130,11 @@ where } let mut redis_conn = redis.connect().await?; - if let Some(body) = redis_conn - .get_deserialized::(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key) - .await? + let redis_key = redis_conn + .keyspace() + .metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key); + if let Some(body) = + redis_conn.get_deserialized::(&redis_key).await? { return Ok(body.html); } @@ -146,14 +143,12 @@ where let cached = HtmlBody { html: get().await? }; let mut redis_conn = redis.connect().await?; + let redis_key = redis_conn + .keyspace() + .metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key); redis_conn - .set_serialized( - TEMPLATES_DYNAMIC_HTML_NAMESPACE, - key, - &cached, - Some(HTML_DATA_CACHE_EXPIRY), - ) + .set_serialized(&redis_key, &cached, Some(HTML_DATA_CACHE_EXPIRY)) .await?; Ok(cached.html) diff --git a/apps/labrinth/src/database/models/notifications_type_item.rs b/apps/labrinth/src/database/models/notifications_type_item.rs index 482c78564b..b4ba2cd2cc 100644 --- a/apps/labrinth/src/database/models/notifications_type_item.rs +++ b/apps/labrinth/src/database/models/notifications_type_item.rs @@ -41,10 +41,11 @@ impl NotificationTypeItem { { { let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(NOTIFICATION_TYPES_NAMESPACE, "all"); - let cached_types = redis - .get_deserialized(NOTIFICATION_TYPES_NAMESPACE, "all") - .await?; + let cached_types = redis.get_deserialized(&key).await?; if let Some(types) = cached_types { return Ok(types); @@ -61,10 +62,11 @@ impl NotificationTypeItem { let types = results.into_iter().map(Into::into).collect(); let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .metadata(NOTIFICATION_TYPES_NAMESPACE, "all"); - redis - .set_serialized(NOTIFICATION_TYPES_NAMESPACE, "all", &types, None) - .await?; + redis.set_serialized(&key, &types, None).await?; Ok(types) } diff --git a/apps/labrinth/src/database/models/organization_item.rs b/apps/labrinth/src/database/models/organization_item.rs index 9e87af6e6f..0538b2bc13 100644 --- a/apps/labrinth/src/database/models/organization_item.rs +++ b/apps/labrinth/src/database/models/organization_item.rs @@ -256,16 +256,18 @@ impl DBOrganization { redis: &RedisPool, ) -> Result<(), super::DatabaseError> { let mut redis = redis.connect().await?; - - redis - .delete_many([ - (ORGANIZATIONS_NAMESPACE, Some(id.0.to_string())), - ( + let mut keys = + vec![redis.keyspace().entity(ORGANIZATIONS_NAMESPACE, id.0)]; + if let Some(slug) = slug { + keys.push( + redis.keyspace().entity( ORGANIZATIONS_TITLES_NAMESPACE, - slug.map(|x| x.to_lowercase()), + slug.to_lowercase(), ), - ]) - .await?; + ); + } + + redis.delete_many(&keys).await?; Ok(()) } } diff --git a/apps/labrinth/src/database/models/pat_item.rs b/apps/labrinth/src/database/models/pat_item.rs index 4ba9bbbd87..7373bd67fa 100644 --- a/apps/labrinth/src/database/models/pat_item.rs +++ b/apps/labrinth/src/database/models/pat_item.rs @@ -159,13 +159,9 @@ impl DBPersonalAccessToken { { { let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(PATS_USERS_NAMESPACE, user_id.0); - let res = redis - .get_deserialized::>( - PATS_USERS_NAMESPACE, - &user_id.0.to_string(), - ) - .await?; + let res = redis.get_deserialized::>(&key).await?; if let Some(res) = res { return Ok(res.into_iter().map(DBPatId).collect()); @@ -187,10 +183,9 @@ impl DBPersonalAccessToken { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(PATS_USERS_NAMESPACE, user_id.0); - redis - .set_serialized(PATS_USERS_NAMESPACE, user_id.0, &db_pats, None) - .await?; + redis.set_serialized(&key, &db_pats, None).await?; Ok(db_pats) } @@ -204,20 +199,23 @@ impl DBPersonalAccessToken { return Ok(()); } - redis - .delete_many(clear_pats.into_iter().flat_map( - |(id, token, user_id)| { - [ - (PATS_NAMESPACE, id.map(|i| i.0.to_string())), - (PATS_TOKENS_NAMESPACE, token), - ( - PATS_USERS_NAMESPACE, - user_id.map(|i| i.0.to_string()), - ), - ] - }, - )) - .await?; + let keys = clear_pats + .into_iter() + .flat_map(|(id, token, user_id)| { + [ + id.map(|id| redis.keyspace().entity(PATS_NAMESPACE, id.0)), + token.map(|token| { + redis.keyspace().entity(PATS_TOKENS_NAMESPACE, token) + }), + user_id.map(|user_id| { + redis.keyspace().entity(PATS_USERS_NAMESPACE, user_id.0) + }), + ] + .into_iter() + .flatten() + }) + .collect::>(); + redis.delete_many(&keys).await?; Ok(()) } diff --git a/apps/labrinth/src/database/models/product_item.rs b/apps/labrinth/src/database/models/product_item.rs index bbfe4c9fd5..5f7b220e1f 100644 --- a/apps/labrinth/src/database/models/product_item.rs +++ b/apps/labrinth/src/database/models/product_item.rs @@ -152,9 +152,10 @@ impl QueryProductWithPrices { { { let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(PRODUCTS_NAMESPACE, "all"); let res: Option> = - redis.get_deserialized(PRODUCTS_NAMESPACE, "all").await?; + redis.get_deserialized(&key).await?; if let Some(res) = res { return Ok(res); @@ -193,10 +194,9 @@ impl QueryProductWithPrices { .collect::>(); let mut redis = redis.connect().await?; + let key = redis.keyspace().metadata(PRODUCTS_NAMESPACE, "all"); - redis - .set_serialized(PRODUCTS_NAMESPACE, "all", &products, None) - .await?; + redis.set_serialized(&key, &products, None).await?; Ok(products) } diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs index 22034123a0..2f8790fdf2 100644 --- a/apps/labrinth/src/database/models/project_item.rs +++ b/apps/labrinth/src/database/models/project_item.rs @@ -974,13 +974,12 @@ impl DBProject { { let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0); - let dependencies = redis - .get_deserialized::( - PROJECTS_DEPENDENCIES_NAMESPACE, - &id.0.to_string(), - ) - .await?; + let dependencies = + redis.get_deserialized::(&key).await?; if let Some(dependencies) = dependencies { return Ok(dependencies); } @@ -1012,15 +1011,11 @@ impl DBProject { .await?; let mut redis = redis.connect().await?; + let key = redis + .keyspace() + .entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0); - redis - .set_serialized( - PROJECTS_DEPENDENCIES_NAMESPACE, - id.0, - &dependencies, - None, - ) - .await?; + redis.set_serialized(&key, &dependencies, None).await?; Ok(dependencies) } @@ -1031,21 +1026,23 @@ impl DBProject { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let mut keys = vec![redis.keyspace().entity(PROJECTS_NAMESPACE, id.0)]; + if let Some(slug) = slug { + keys.push( + redis + .keyspace() + .entity(PROJECTS_SLUGS_NAMESPACE, slug.to_lowercase()), + ); + } + if clear_dependencies.unwrap_or(false) { + keys.push( + redis + .keyspace() + .entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0), + ); + } - redis - .delete_many([ - (PROJECTS_NAMESPACE, Some(id.0.to_string())), - (PROJECTS_SLUGS_NAMESPACE, slug.map(|x| x.to_lowercase())), - ( - PROJECTS_DEPENDENCIES_NAMESPACE, - if clear_dependencies.unwrap_or(false) { - Some(id.0.to_string()) - } else { - None - }, - ), - ]) - .await?; + redis.delete_many(&keys).await?; Ok(()) } } diff --git a/apps/labrinth/src/database/models/session_item.rs b/apps/labrinth/src/database/models/session_item.rs index f5198760bc..daf9493fe1 100644 --- a/apps/labrinth/src/database/models/session_item.rs +++ b/apps/labrinth/src/database/models/session_item.rs @@ -224,13 +224,10 @@ impl DBSession { { { let mut redis = redis.connect().await?; + let key = + redis.keyspace().entity(SESSIONS_USERS_NAMESPACE, user_id.0); - let res = redis - .get_deserialized::>( - SESSIONS_USERS_NAMESPACE, - &user_id.0.to_string(), - ) - .await?; + let res = redis.get_deserialized::>(&key).await?; if let Some(res) = res { return Ok(res.into_iter().map(DBSessionId).collect()); @@ -253,15 +250,9 @@ impl DBSession { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(SESSIONS_USERS_NAMESPACE, user_id.0); - redis - .set_serialized( - SESSIONS_USERS_NAMESPACE, - user_id.0, - &db_sessions, - None, - ) - .await?; + redis.set_serialized(&key, &db_sessions, None).await?; Ok(db_sessions) } @@ -280,20 +271,27 @@ impl DBSession { return Ok(()); } - redis - .delete_many(clear_sessions.into_iter().flat_map( - |(id, session, user_id)| { - [ - (SESSIONS_NAMESPACE, id.map(|i| i.0.to_string())), - (SESSIONS_IDS_NAMESPACE, session), - ( - SESSIONS_USERS_NAMESPACE, - user_id.map(|i| i.0.to_string()), - ), - ] - }, - )) - .await?; + let keys = clear_sessions + .into_iter() + .flat_map(|(id, session, user_id)| { + [ + id.map(|id| { + redis.keyspace().entity(SESSIONS_NAMESPACE, id.0) + }), + session.map(|session| { + redis.keyspace().entity(SESSIONS_IDS_NAMESPACE, session) + }), + user_id.map(|user_id| { + redis + .keyspace() + .entity(SESSIONS_USERS_NAMESPACE, user_id.0) + }), + ] + .into_iter() + .flatten() + }) + .collect::>(); + redis.delete_many(&keys).await?; Ok(()) } diff --git a/apps/labrinth/src/database/models/team_item.rs b/apps/labrinth/src/database/models/team_item.rs index da577866f3..9748f3db62 100644 --- a/apps/labrinth/src/database/models/team_item.rs +++ b/apps/labrinth/src/database/models/team_item.rs @@ -265,7 +265,8 @@ impl DBTeamMember { redis: &RedisPool, ) -> Result<(), super::DatabaseError> { let mut redis = redis.connect().await?; - redis.delete(TEAMS_NAMESPACE, id.0).await?; + let key = redis.keyspace().entity(TEAMS_NAMESPACE, id.0); + redis.delete(&key).await?; Ok(()) } diff --git a/apps/labrinth/src/database/models/user_item.rs b/apps/labrinth/src/database/models/user_item.rs index d8dada9188..b3e4a585f8 100644 --- a/apps/labrinth/src/database/models/user_item.rs +++ b/apps/labrinth/src/database/models/user_item.rs @@ -389,13 +389,11 @@ impl DBUser { { let mut redis = redis.connect().await?; + let key = + redis.keyspace().entity(USERS_PROJECTS_NAMESPACE, user_id.0); - let cached_projects = redis - .get_deserialized::>( - USERS_PROJECTS_NAMESPACE, - &user_id.0.to_string(), - ) - .await?; + let cached_projects = + redis.get_deserialized::>(&key).await?; if let Some(projects) = cached_projects { return Ok(projects); @@ -417,15 +415,9 @@ impl DBUser { .await?; let mut redis = redis.connect().await?; + let key = redis.keyspace().entity(USERS_PROJECTS_NAMESPACE, user_id.0); - redis - .set_serialized( - USERS_PROJECTS_NAMESPACE, - user_id.0, - &db_projects, - None, - ) - .await?; + redis.set_serialized(&key, &db_projects, None).await?; Ok(db_projects) } @@ -556,18 +548,24 @@ impl DBUser { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - - redis - .delete_many(user_ids.iter().flat_map(|(id, username)| { + let keys = user_ids + .iter() + .flat_map(|(id, username)| { [ - (USERS_NAMESPACE, Some(id.0.to_string())), - ( - USER_USERNAMES_NAMESPACE, - username.clone().map(|i| i.to_lowercase()), - ), + Some(redis.keyspace().entity(USERS_NAMESPACE, id.0)), + username.as_ref().map(|username| { + redis.keyspace().entity( + USER_USERNAMES_NAMESPACE, + username.to_lowercase(), + ) + }), ] - })) - .await?; + .into_iter() + .flatten() + }) + .collect::>(); + + redis.delete_many(&keys).await?; Ok(()) } @@ -576,14 +574,12 @@ impl DBUser { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let keys = user_ids + .iter() + .map(|id| redis.keyspace().entity(USERS_PROJECTS_NAMESPACE, id.0)) + .collect::>(); - redis - .delete_many( - user_ids.iter().map(|id| { - (USERS_PROJECTS_NAMESPACE, Some(id.0.to_string())) - }), - ) - .await?; + redis.delete_many(&keys).await?; Ok(()) } diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs index 1644526b4b..5317bf4b81 100644 --- a/apps/labrinth/src/database/models/version_item.rs +++ b/apps/labrinth/src/database/models/version_item.rs @@ -19,7 +19,6 @@ use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::collections::HashMap; -use std::iter; use tracing::error; pub const VERSIONS_NAMESPACE: &str = "versions:v1"; @@ -1051,25 +1050,21 @@ impl DBVersion { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let mut keys = vec![ + redis + .keyspace() + .entity(VERSIONS_NAMESPACE, version.inner.id.0), + ]; + keys.extend(version.files.iter().flat_map(|file| { + file.hashes.iter().map(|(algorithm, hash)| { + redis.keyspace().entity( + VERSION_FILES_NAMESPACE, + format!("{algorithm}_{hash}"), + ) + }) + })); - redis - .delete_many( - iter::once(( - VERSIONS_NAMESPACE, - Some(version.inner.id.0.to_string()), - )) - .chain(version.files.iter().flat_map( - |file| { - file.hashes.iter().map(|(algo, hash)| { - ( - VERSION_FILES_NAMESPACE, - Some(format!("{algo}_{hash}")), - ) - }) - }, - )), - ) - .await?; + redis.delete_many(&keys).await?; Ok(()) } @@ -1078,14 +1073,12 @@ impl DBVersion { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; + let keys = version_ids + .iter() + .map(|id| redis.keyspace().entity(VERSIONS_NAMESPACE, id.0)) + .collect::>(); - redis - .delete_many( - version_ids - .iter() - .map(|id| (VERSIONS_NAMESPACE, Some(id.0.to_string()))), - ) - .await?; + redis.delete_many(&keys).await?; Ok(()) } } diff --git a/apps/labrinth/src/database/redis/blocking.rs b/apps/labrinth/src/database/redis/blocking.rs new file mode 100644 index 0000000000..768c106fba --- /dev/null +++ b/apps/labrinth/src/database/redis/blocking.rs @@ -0,0 +1,153 @@ +use std::time::Duration; + +use prometheus::Registry; + +use crate::database::models::DatabaseError; + +use super::RedisPool; +use super::config::{RedisConfig, RedisMode}; +use super::connection::RedisBackendBuildError; +use super::metrics::{ + LogicalPoolStatus, LogicalPoolStatusProvider, + register_blocking_pool_metrics, +}; + +const POOL_RETAIN_INTERVAL: Duration = Duration::from_secs(30); +const MAX_IDLE_CONNECTION_AGE: Duration = Duration::from_secs(5 * 60); +const MAX_STANDALONE_CONNECTION_AGE: Duration = Duration::from_secs(120); + +/// A pool of Redis connections used for blocking operations. +#[derive(Clone)] +pub(super) struct RedisBlockingPool { + inner: RedisBlockingPoolInner, +} + +#[derive(Clone)] +enum RedisBlockingPoolInner { + Standalone(deadpool_redis::Pool), + Cluster(deadpool_redis::cluster::Pool), +} + +impl RedisBlockingPool { + pub(super) async fn new( + config: &RedisConfig, + ) -> Result { + let pool_size = config.blocking_pool_size(); + let inner = match config.mode() { + RedisMode::Standalone => { + let connection_config = redis::AsyncConnectionConfig::new() + .set_connection_timeout(None) + .set_response_timeout(None); + let manager = deadpool_redis::Manager::new_with_config( + config.seed_urls()[0].clone(), + connection_config, + )?; + let pool = deadpool_redis::Pool::builder(manager) + .max_size(pool_size.max()) + .wait_timeout(Some(Duration::from_millis( + config.wait_timeout_ms(), + ))) + .runtime(deadpool_redis::Runtime::Tokio1) + .build()?; + retain_standalone_pool(pool.clone()); + RedisBlockingPoolInner::Standalone(pool) + } + RedisMode::Cluster => { + let manager = deadpool_redis::cluster::Manager::new( + config.seed_urls().to_vec(), + false, + )?; + let pool = deadpool_redis::cluster::Pool::builder(manager) + .max_size(pool_size.max()) + .wait_timeout(Some(Duration::from_millis( + config.wait_timeout_ms(), + ))) + .runtime(deadpool_redis::Runtime::Tokio1) + .build()?; + retain_cluster_pool(pool.clone()); + RedisBlockingPoolInner::Cluster(pool) + } + }; + + Ok(Self { inner }) + } + + pub(super) fn register_metrics( + &self, + registry: &Registry, + ) -> Result<(), prometheus::Error> { + register_blocking_pool_metrics(registry, self.clone()) + } + + async fn brpop( + &self, + key: &str, + timeout: Duration, + ) -> Result; 2]>, DatabaseError> { + if timeout.is_zero() { + return Err(DatabaseError::Internal(eyre::eyre!( + "redis blocking timeout must be greater than zero" + ))); + } + + let mut command = redis::cmd("BRPOP"); + command.arg(key).arg(timeout.as_secs_f64()); + + let response: Option<(Vec, Vec)> = match &self.inner { + RedisBlockingPoolInner::Standalone(pool) => { + command.query_async(&mut pool.get().await?).await? + } + RedisBlockingPoolInner::Cluster(pool) => { + command.query_async(&mut pool.get().await?).await? + } + }; + + Ok(response.map(|(key, value)| [key, value])) + } +} + +impl LogicalPoolStatusProvider for RedisBlockingPool { + fn logical_pool_status(&self) -> LogicalPoolStatus { + match &self.inner { + RedisBlockingPoolInner::Standalone(pool) => { + LogicalPoolStatus::from_deadpool(pool.status()) + } + RedisBlockingPoolInner::Cluster(pool) => { + LogicalPoolStatus::from_deadpool(pool.status()) + } + } + } +} + +impl RedisPool { + pub async fn brpop( + &self, + key: &str, + timeout: Duration, + ) -> Result; 2]>, DatabaseError> { + self.blocking.brpop(key, timeout).await + } +} + +fn retain_standalone_pool(pool: deadpool_redis::Pool) { + tokio::spawn(async move { + loop { + tokio::time::sleep(POOL_RETAIN_INTERVAL).await; + pool.retain(|_, metrics| { + metrics.last_used() < MAX_IDLE_CONNECTION_AGE + && metrics.created.elapsed() < MAX_STANDALONE_CONNECTION_AGE + }); + } + }); +} + +fn retain_cluster_pool(pool: deadpool_redis::cluster::Pool) { + tokio::spawn(async move { + loop { + tokio::time::sleep(POOL_RETAIN_INTERVAL).await; + pool.retain(|_, metrics| { + metrics.last_used() < MAX_IDLE_CONNECTION_AGE + }); + } + }); +} diff --git a/apps/labrinth/src/database/redis/cache.rs b/apps/labrinth/src/database/redis/cache.rs new file mode 100644 index 0000000000..d97ca7f130 --- /dev/null +++ b/apps/labrinth/src/database/redis/cache.rs @@ -0,0 +1,614 @@ +use std::borrow::Cow; +use std::collections::HashMap; +use std::fmt::{Debug, Display}; +use std::future::Future; +use std::hash::Hash; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; + +use ariadne::ids::base62_impl::{parse_base62, to_base62}; +use chrono::{TimeZone, Utc}; +use dashmap::DashMap; +use futures::future::Either; +use futures::stream::{FuturesUnordered, StreamExt}; +use redis::aio::ConnectionLike; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tracing::{Instrument, info_span}; + +use crate::database::models::DatabaseError; + +use super::commands; +use super::key::KeyBuilder; +use super::util; + +const ACTUAL_EXPIRY: i64 = 60 * 30; +const VERSION_DEFAULT_EXPIRY: i64 = 60 * 60 * 48; +const VERSION_ACTUAL_EXPIRY: i64 = 60 * 60 * 24; + +pub(super) trait ConnectionProvider { + type Connection: ConnectionLike; + + fn connect( + &self, + ) -> impl Future> + Send; +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Codec { + Raw = 0, + Lz4 = 1, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncodingFormat { + Json, + Postcard, +} + +#[derive(Debug, Error)] +#[error("invalid redis codec")] +pub struct InvalidCodec; + +#[derive(Debug, Error)] +#[error("invalid redis encoding format")] +pub struct InvalidEncodingFormat; + +impl TryFrom for Codec { + type Error = InvalidCodec; + + fn try_from(value: u8) -> Result { + match value { + 0 => Ok(Self::Raw), + 1 => Ok(Self::Lz4), + _ => Err(InvalidCodec), + } + } +} + +impl FromStr for Codec { + type Err = InvalidCodec; + + fn from_str(value: &str) -> Result { + match value { + "lz4" => Ok(Self::Lz4), + _ => Err(InvalidCodec), + } + } +} + +impl FromStr for EncodingFormat { + type Err = InvalidEncodingFormat; + + fn from_str(value: &str) -> Result { + match value { + "json" => Ok(Self::Json), + "postcard" => Ok(Self::Postcard), + _ => Err(InvalidEncodingFormat), + } + } +} + +#[derive(Debug, Clone)] +pub struct CacheSettings { + pub encoding_format: EncodingFormat, + pub compression_algorithm: Codec, + pub compression_level: i32, + pub compression_threshold_bytes: usize, + pub compression_min_savings_ratio: f64, +} + +impl CacheSettings { + pub fn encode_value( + &self, + value: &T, + ) -> Result, DatabaseError> { + let mut value = match self.encoding_format { + EncodingFormat::Json => serde_json::to_vec(value)?, + EncodingFormat::Postcard => postcard::to_allocvec(value)?, + }; + + if self.compression_level > 0 + && self.compression_algorithm == Codec::Lz4 + && value.len() >= self.compression_threshold_bytes + { + let compressed = lz4_flex::block::compress_prepend_size(&value); + let savings_ratio = value.len().saturating_sub(compressed.len()) + as f64 + / value.len().max(1) as f64 + * 100.0; + + if savings_ratio >= self.compression_min_savings_ratio { + let mut encoded = Vec::with_capacity(compressed.len() + 1); + encoded.push(Codec::Lz4 as u8); + encoded.extend(compressed); + return Ok(encoded); + } + } + + let mut encoded = Vec::with_capacity(value.len() + 1); + encoded.push(Codec::Raw as u8); + encoded.append(&mut value); + Ok(encoded) + } + + pub fn decode_value(&self, value: &[u8]) -> Option + where + T: for<'a> Deserialize<'a>, + { + let (codec, value) = value.split_first()?; + let value = match Codec::try_from(*codec).ok()? { + Codec::Raw => Cow::Borrowed(value), + Codec::Lz4 => Cow::Owned( + lz4_flex::block::decompress_size_prepended(value).ok()?, + ), + }; + + match self.encoding_format { + EncodingFormat::Json => serde_json::from_slice(&value).ok(), + EncodingFormat::Postcard => postcard::from_bytes(&value).ok(), + } + } +} + +#[derive(Clone)] +pub struct CacheManager { + key_builder: KeyBuilder, + settings: CacheSettings, + cache_list: Arc>, +} + +impl CacheManager { + pub fn new(key_builder: KeyBuilder, settings: CacheSettings) -> Self { + Self { + key_builder, + settings, + cache_list: Arc::new(DashMap::with_capacity(2048)), + } + } + + pub fn settings(&self) -> &CacheSettings { + &self.settings + } + + #[tracing::instrument(skip(self, provider, closure))] + pub async fn get_cached_keys( + &self, + provider: &P, + namespace: &str, + keys: &[K], + closure: F, + ) -> Result, DatabaseError> + where + P: ConnectionProvider, + F: FnOnce(Vec) -> Fut, + Fut: Future, DatabaseError>>, + T: Serialize + DeserializeOwned, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize + + Debug, + { + Ok(self + .get_cached_keys_raw(provider, namespace, keys, closure) + .await? + .into_values() + .collect()) + } + + #[tracing::instrument(skip(self, provider, closure))] + pub async fn get_cached_keys_raw( + &self, + provider: &P, + namespace: &str, + keys: &[K], + closure: F, + ) -> Result, DatabaseError> + where + P: ConnectionProvider, + F: FnOnce(Vec) -> Fut, + Fut: Future, DatabaseError>>, + T: Serialize + DeserializeOwned, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize + + Debug, + { + self.get_cached_keys_raw_with_slug( + provider, + namespace, + None, + false, + keys, + |ids| async move { + Ok(closure(ids) + .await? + .into_iter() + .map(|(key, value)| (key, (None::, value))) + .collect()) + }, + ) + .await + } + + #[tracing::instrument(skip(self, provider, closure))] + pub async fn get_cached_keys_with_slug( + &self, + provider: &P, + namespace: &str, + slug_namespace: &str, + case_sensitive: bool, + keys: &[I], + closure: F, + ) -> Result, DatabaseError> + where + P: ConnectionProvider, + F: FnOnce(Vec) -> Fut, + Fut: Future, T)>, DatabaseError>>, + T: Serialize + DeserializeOwned, + I: Display + Hash + Eq + PartialEq + Clone + Debug, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize, + S: Display + Clone + DeserializeOwned + Serialize + Debug, + { + Ok(self + .get_cached_keys_raw_with_slug( + provider, + namespace, + Some(slug_namespace), + case_sensitive, + keys, + closure, + ) + .await? + .into_values() + .collect()) + } + + #[tracing::instrument(skip(self, provider, closure))] + pub async fn get_cached_keys_raw_with_slug( + &self, + provider: &P, + namespace: &str, + slug_namespace: Option<&str>, + case_sensitive: bool, + keys: &[I], + closure: F, + ) -> Result, DatabaseError> + where + P: ConnectionProvider, + F: FnOnce(Vec) -> Fut, + Fut: Future, T)>, DatabaseError>>, + T: Serialize + DeserializeOwned, + I: Display + Hash + Eq + PartialEq + Clone + Debug, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize, + S: Display + Clone + DeserializeOwned + Serialize + Debug, + { + let ids = keys + .iter() + .map(|key| (key.to_string(), key.clone())) + .collect::>(); + + if ids.is_empty() { + return Ok(HashMap::new()); + } + + let get_cached_values = |ids: DashMap| { + async move { + let slug_ids = if let Some(slug_namespace) = slug_namespace { + async { + let keys = ids + .iter() + .map(|entry| { + let logical_key = if case_sensitive { + entry.value().to_string() + } else { + entry.value().to_string().to_lowercase() + }; + self.key_builder + .entity(slug_namespace, logical_key) + }) + .collect::>(); + let mut connection = provider.connect().await?; + Ok::<_, DatabaseError>( + commands::get_many_strings(&mut connection, &keys) + .await? + .into_iter() + .flatten() + .collect::>(), + ) + } + .instrument(info_span!("get slug ids")) + .await? + } else { + Vec::new() + }; + + let keys = ids + .iter() + .map(|entry| entry.value().to_string()) + .chain(ids.iter().filter_map(|entry| { + parse_base62(&entry.value().to_string()) + .ok() + .map(|value| value.to_string()) + })) + .chain(slug_ids) + .map(|key| self.key_builder.entity(namespace, key)) + .collect::>(); + + let mut connection = provider.connect().await?; + let mut cached_values = HashMap::new(); + for value in commands::get_many(&mut connection, &keys).await? { + if let Some(value) = value.and_then(|value| { + self.settings + .decode_value::>(&value) + }) { + cached_values.insert(value.key.clone(), value); + } + } + + Ok::<_, DatabaseError>((cached_values, ids)) + } + .instrument(info_span!("get cached values")) + }; + + let (default_expiry, actual_expiry) = cache_expiries(namespace); + let current_time = Utc::now(); + let mut expired_values = HashMap::new(); + + let (cached_values_raw, ids) = get_cached_values(ids).await?; + let mut cached_values = cached_values_raw + .into_iter() + .filter_map(|(key, value)| { + if Utc.timestamp_opt(value.iat + actual_expiry, 0).unwrap() + < current_time + { + expired_values.insert(value.key.to_string(), value); + None + } else { + remove_resolved_ids(&ids, &value); + Some((key, value)) + } + }) + .collect::>(); + + let subscribe_ids = DashMap::new(); + let mut cache_writers = HashMap::new(); + + if !ids.is_empty() { + let fetch_ids = ids + .iter() + .map(|entry| entry.key().clone()) + .collect::>(); + + for key in fetch_ids { + let lock_key = self.key_builder.entity( + namespace, + if case_sensitive { + key.to_lowercase() + } else { + key.clone() + }, + ); + + match self.acquire_lock(lock_key) { + Either::Left(sentinel) => { + cache_writers.insert(key, sentinel); + } + Either::Right(subscriber) => { + if let Some((key, raw_key)) = ids.remove(&key) { + if let Some(value) = expired_values.remove(&key) { + remove_resolved_ids(&ids, &value); + cached_values.insert(value.key.clone(), value); + } else { + subscribe_ids.insert(raw_key, subscriber); + } + } + } + } + } + } + + let mut fetch_tasks = Vec::new(); + + if !ids.is_empty() { + fetch_tasks.push(Either::Left(async { + let fetch_ids = ids + .iter() + .map(|entry| entry.value().clone()) + .collect::>(); + let values = closure(fetch_ids).await?; + let mut return_values = HashMap::new(); + let mut connection = provider.connect().await?; + + for (key, (slug, value)) in values { + let value = RedisValue { + key: key.clone(), + iat: Utc::now().timestamp(), + val: value, + alias: slug.clone(), + }; + let redis_key = + self.key_builder.entity(namespace, key.to_string()); + commands::set( + &mut connection, + &redis_key, + self.settings.encode_value(&value)?, + Some(default_expiry), + ) + .await?; + + if let Some(slug) = slug { + ids.remove(&slug.to_string()); + if let Some(slug_namespace) = slug_namespace { + let actual_slug = if case_sensitive { + slug.to_string() + } else { + slug.to_string().to_lowercase() + }; + let slug_key = self + .key_builder + .entity(slug_namespace, actual_slug); + commands::set( + &mut connection, + &slug_key, + key.to_string(), + Some(default_expiry), + ) + .await?; + } + } + + let key_string = key.to_string(); + ids.remove(&key_string); + if let Ok(value) = key_string.parse::() { + ids.remove(&to_base62(value)); + } + return_values.insert(key, value); + } + + drop(cache_writers); + Result::<_, DatabaseError>::Ok(return_values) + })); + } + + if !subscribe_ids.is_empty() { + fetch_tasks.push(Either::Right(async move { + let mut futures = FuturesUnordered::new(); + let len = subscribe_ids.len(); + + for (key, subscriber) in subscribe_ids { + futures.push(async move { + ( + key, + subscriber + .wait_timeout(Duration::from_secs(5)) + .await, + ) + }); + } + + let fetch_ids = DashMap::with_capacity(len); + while let Some((key, result)) = futures.next().await { + result?; + fetch_ids.insert(key.to_string(), key); + } + + let (return_values, _) = get_cached_values(fetch_ids).await?; + Ok(return_values) + })); + } + + if !fetch_tasks.is_empty() { + for values in futures::future::try_join_all(fetch_tasks).await? { + cached_values.extend(values); + } + } + + Ok(cached_values + .into_iter() + .map(|(key, value)| (key, value.val)) + .collect()) + } + + fn acquire_lock( + &self, + key: String, + ) -> Either { + let mut writer = None; + let subscriber = + self.cache_list.entry(key.clone()).or_insert_with(|| { + let (new_writer, subscriber) = util::cache(); + writer = Some(new_writer); + subscriber + }); + + match writer { + Some(writer) => Either::Left(LockSentinel { + cache_list: self.cache_list.clone(), + key, + writer, + }), + None => Either::Right(subscriber.clone()), + } + } +} + +fn cache_expiries(namespace: &str) -> (i64, i64) { + match namespace + .split_once(':') + .map(|value| value.0) + .unwrap_or(namespace) + { + "versions" | "versions_files" => { + (VERSION_DEFAULT_EXPIRY, VERSION_ACTUAL_EXPIRY) + } + _ => (commands::DEFAULT_EXPIRY, ACTUAL_EXPIRY), + } +} + +fn remove_resolved_ids( + ids: &DashMap, + value: &RedisValue, +) where + K: Display, + S: Display, +{ + let key = value.key.to_string(); + ids.remove(&key); + if let Ok(value) = key.parse::() { + ids.remove(&to_base62(value)); + } + if let Some(alias) = &value.alias { + ids.remove(&alias.to_string()); + } +} + +struct LockSentinel { + cache_list: Arc>, + key: String, + writer: util::CacheWriter, +} + +impl Drop for LockSentinel { + fn drop(&mut self) { + self.writer.write(); + self.cache_list.remove(&self.key); + } +} + +#[derive(Serialize, Deserialize)] +pub struct RedisValue { + key: K, + alias: Option, + iat: i64, + val: T, +} + +impl RedisValue { + pub fn value(&self) -> &T { + &self.val + } +} diff --git a/apps/labrinth/src/database/redis/commands.rs b/apps/labrinth/src/database/redis/commands.rs new file mode 100644 index 0000000000..102475e170 --- /dev/null +++ b/apps/labrinth/src/database/redis/commands.rs @@ -0,0 +1,190 @@ +use std::fmt::Debug; + +use redis::aio::ConnectionLike; +use redis::{FromRedisValue, ToRedisArgs}; + +use crate::database::models::DatabaseError; + +use super::cache::CacheSettings; +use super::util::cmd; + +pub const DEFAULT_EXPIRY: i64 = 60 * 60 * 12; +pub const MGET_CHUNK_SIZE: usize = 32; + +#[tracing::instrument(skip(connection, data))] +pub async fn set( + connection: &mut C, + key: &str, + data: D, + expiry: Option, +) -> Result<(), DatabaseError> +where + C: ConnectionLike, + D: ToRedisArgs + Send + Sync + Debug, +{ + cmd("SET") + .arg(key) + .arg(data) + .arg("EX") + .arg(expiry.unwrap_or(DEFAULT_EXPIRY)) + .query_async::<()>(connection) + .await?; + Ok(()) +} + +#[tracing::instrument(skip(connection, data, settings))] +pub async fn set_serialized( + connection: &mut C, + key: &str, + data: D, + expiry: Option, + settings: &CacheSettings, +) -> Result<(), DatabaseError> +where + C: ConnectionLike, + D: serde::Serialize, +{ + set(connection, key, settings.encode_value(&data)?, expiry).await +} + +#[tracing::instrument(skip(connection))] +pub async fn get( + connection: &mut C, + key: &str, +) -> Result, DatabaseError> +where + C: ConnectionLike, +{ + Ok(cmd("GET").arg(key).query_async(connection).await?) +} + +/// Issues ordinary `MGET` commands in bounded chunks. Cluster routing and +/// result ordering remain redis-rs's responsibility; multiple chunks are not +/// an atomic snapshot. +#[tracing::instrument(skip(connection, keys))] +pub async fn get_many( + connection: &mut C, + keys: &[String], +) -> Result>>, DatabaseError> +where + C: ConnectionLike, +{ + get_many_as(connection, keys).await +} + +#[tracing::instrument(skip(connection, keys))] +pub async fn get_many_strings( + connection: &mut C, + keys: &[String], +) -> Result>, DatabaseError> +where + C: ConnectionLike, +{ + get_many_as(connection, keys).await +} + +async fn get_many_as( + connection: &mut C, + keys: &[String], +) -> Result>, DatabaseError> +where + C: ConnectionLike, + T: FromRedisValue, +{ + let mut values = Vec::with_capacity(keys.len()); + for chunk in keys.chunks(MGET_CHUNK_SIZE) { + let part = cmd("MGET") + .arg(chunk) + .query_async::>>(connection) + .await?; + values.extend(part); + } + Ok(values) +} + +#[tracing::instrument(skip(connection, settings))] +pub async fn get_deserialized( + connection: &mut C, + key: &str, + settings: &CacheSettings, +) -> Result, DatabaseError> +where + C: ConnectionLike, + R: for<'a> serde::Deserialize<'a>, +{ + let value: Option> = + cmd("GET").arg(key).query_async(connection).await?; + Ok(value.and_then(|value| settings.decode_value(&value))) +} + +#[tracing::instrument(skip(connection, keys, settings))] +pub async fn get_many_deserialized( + connection: &mut C, + keys: &[String], + settings: &CacheSettings, +) -> Result>, DatabaseError> +where + C: ConnectionLike, + R: for<'a> serde::Deserialize<'a>, +{ + Ok(get_many(connection, keys) + .await? + .into_iter() + .map(|value| value.and_then(|value| settings.decode_value(&value))) + .collect()) +} + +#[tracing::instrument(skip(connection))] +pub async fn delete( + connection: &mut C, + key: &str, +) -> Result<(), DatabaseError> +where + C: ConnectionLike, +{ + cmd("DEL").arg(key).query_async::<()>(connection).await?; + Ok(()) +} + +#[tracing::instrument(skip(connection, keys))] +pub async fn delete_many( + connection: &mut C, + keys: &[String], +) -> Result<(), DatabaseError> +where + C: ConnectionLike, +{ + if !keys.is_empty() { + cmd("DEL").arg(keys).query_async::<()>(connection).await?; + } + Ok(()) +} + +#[tracing::instrument(skip(connection, value))] +pub async fn lpush( + connection: &mut C, + key: &str, + value: D, +) -> Result<(), DatabaseError> +where + C: ConnectionLike, + D: ToRedisArgs + Send + Sync + Debug, +{ + cmd("LPUSH") + .arg(key) + .arg(value) + .query_async::<()>(connection) + .await?; + Ok(()) +} + +#[tracing::instrument(skip(connection))] +pub async fn incr( + connection: &mut C, + key: &str, +) -> Result, DatabaseError> +where + C: ConnectionLike, +{ + Ok(cmd("INCR").arg(key).query_async(connection).await?) +} diff --git a/apps/labrinth/src/database/redis/config.rs b/apps/labrinth/src/database/redis/config.rs new file mode 100644 index 0000000000..7c29c491f4 --- /dev/null +++ b/apps/labrinth/src/database/redis/config.rs @@ -0,0 +1,217 @@ +use std::str::FromStr; + +use crate::env::ENV; +use thiserror::Error; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RedisMode { + Standalone, + Cluster, +} + +#[derive(Debug, Error)] +#[error("invalid Redis mode; expected `standalone` or `cluster`")] +pub struct InvalidRedisMode; + +impl FromStr for RedisMode { + type Err = InvalidRedisMode; + + fn from_str(value: &str) -> Result { + match value { + "standalone" => Ok(Self::Standalone), + "cluster" => Ok(Self::Cluster), + _ => Err(InvalidRedisMode), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RedisConnectionType { + Pooled, + Multiplexed, +} + +#[derive(Debug, Error)] +#[error("invalid Redis connection type; expected `pooled` or `multiplexed`")] +pub struct InvalidRedisConnectionType; + +impl FromStr for RedisConnectionType { + type Err = InvalidRedisConnectionType; + + fn from_str(value: &str) -> Result { + match value { + "pooled" => Ok(Self::Pooled), + "multiplexed" => Ok(Self::Multiplexed), + _ => Err(InvalidRedisConnectionType), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct RedisPoolSize { + max: usize, + min: usize, +} + +impl RedisPoolSize { + fn new( + name: &'static str, + max: usize, + min: usize, + ) -> Result { + if max == 0 || min > max { + return Err(RedisConfigError::InvalidPoolSize { name, max, min }); + } + + Ok(Self { max, min }) + } + + pub(super) fn max(self) -> usize { + self.max + } + + pub(super) fn min(self) -> usize { + self.min + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum RedisBackendConfig { + StandalonePooled(RedisPoolSize), + ClusterPooled(RedisPoolSize), + ClusterMultiplexed, +} + +#[derive(Debug, Clone)] +pub(super) struct RedisConfig { + mode: RedisMode, + backend: RedisBackendConfig, + seed_urls: Vec, + wait_timeout_ms: u64, + blocking_pool_size: RedisPoolSize, +} + +#[derive(Debug, Error)] +pub(super) enum RedisConfigError { + #[error("`REDIS_URL` must contain at least one Redis URL")] + MissingUrl, + #[error("standalone Redis mode requires exactly one URL")] + MultipleStandaloneUrls, + #[error( + "unsupported Redis configuration: `{mode:?}` mode with `{connection_type:?}` connections" + )] + UnsupportedConnectionType { + mode: RedisMode, + connection_type: RedisConnectionType, + }, + #[error( + "invalid {name} Redis pool size: minimum {min} must not exceed nonzero maximum {max}" + )] + InvalidPoolSize { + name: &'static str, + max: usize, + min: usize, + }, +} + +impl RedisConfig { + pub(super) fn from_env() -> Result { + Self::new( + ENV.REDIS_MODE, + ENV.REDIS_CONNECTION_TYPE, + &ENV.REDIS_URL, + ENV.REDIS_WAIT_TIMEOUT_MS, + ( + ENV.REDIS_MAX_CONNECTIONS as usize, + ENV.REDIS_MIN_CONNECTIONS, + ), + ( + ENV.REDIS_CLUSTER_MAX_CONNECTIONS as usize, + ENV.REDIS_CLUSTER_MIN_CONNECTIONS, + ), + RedisPoolSize::new( + "blocking", + ENV.REDIS_BLOCKING_MAX_CONNECTIONS as usize, + 0, + )?, + ) + } + + fn new( + mode: RedisMode, + connection_type: RedisConnectionType, + raw_urls: &str, + wait_timeout_ms: u64, + standalone_pool_size: (usize, usize), + cluster_pool_size: (usize, usize), + blocking_pool_size: RedisPoolSize, + ) -> Result { + let seed_urls = raw_urls + .split(',') + .map(str::trim) + .filter(|url| !url.is_empty()) + .map(ToOwned::to_owned) + .collect::>(); + + if seed_urls.is_empty() { + return Err(RedisConfigError::MissingUrl); + } + + let backend = match (mode, connection_type) { + (RedisMode::Standalone, RedisConnectionType::Pooled) => { + if seed_urls.len() != 1 { + return Err(RedisConfigError::MultipleStandaloneUrls); + } + RedisBackendConfig::StandalonePooled(RedisPoolSize::new( + "standalone", + standalone_pool_size.0, + standalone_pool_size.1, + )?) + } + (RedisMode::Cluster, RedisConnectionType::Pooled) => { + RedisBackendConfig::ClusterPooled(RedisPoolSize::new( + "cluster", + cluster_pool_size.0, + cluster_pool_size.1, + )?) + } + (RedisMode::Cluster, RedisConnectionType::Multiplexed) => { + RedisBackendConfig::ClusterMultiplexed + } + (mode, connection_type) => { + return Err(RedisConfigError::UnsupportedConnectionType { + mode, + connection_type, + }); + } + }; + + Ok(Self { + mode, + backend, + seed_urls, + wait_timeout_ms, + blocking_pool_size, + }) + } + + pub(super) fn mode(&self) -> RedisMode { + self.mode + } + + pub(super) fn backend(&self) -> RedisBackendConfig { + self.backend + } + + pub(super) fn seed_urls(&self) -> &[String] { + &self.seed_urls + } + + pub(super) fn wait_timeout_ms(&self) -> u64 { + self.wait_timeout_ms + } + + pub(super) fn blocking_pool_size(&self) -> RedisPoolSize { + self.blocking_pool_size + } +} diff --git a/apps/labrinth/src/database/redis/connection.rs b/apps/labrinth/src/database/redis/connection.rs new file mode 100644 index 0000000000..61c56988fb --- /dev/null +++ b/apps/labrinth/src/database/redis/connection.rs @@ -0,0 +1,266 @@ +use std::time::Duration; + +use futures::future::try_join_all; +use prometheus::Registry; +use redis::aio::ConnectionLike; +use thiserror::Error; + +use super::config::{RedisBackendConfig, RedisConfig, RedisPoolSize}; +use super::metrics::{ + LogicalPoolStatus, LogicalPoolStatusProvider, register_command_pool_metrics, +}; + +const POOL_RETAIN_INTERVAL: Duration = Duration::from_secs(30); +const MAX_IDLE_CONNECTION_AGE: Duration = Duration::from_secs(5 * 60); +const MAX_STANDALONE_CONNECTION_AGE: Duration = Duration::from_secs(120); + +/// The primary backing "connection provider" for a Redis backend implementation. +#[derive(Clone)] +pub(super) enum RedisBackend { + StandalonePooled(deadpool_redis::Pool), + ClusterPooled(deadpool_redis::cluster::Pool), + ClusterMultiplexed(redis::cluster_async::ClusterConnection), +} + +#[derive(Debug, Error)] +pub(super) enum RedisBackendBuildError { + #[error("failed to configure Redis client: {0}")] + Redis(#[from] redis::RedisError), + #[error("failed to build Redis pool: {0}")] + PoolBuild(#[from] deadpool_redis::BuildError), + #[error("failed to establish initial Redis pool connections: {0}")] + Pool(#[from] deadpool_redis::PoolError), +} + +pub(super) struct RedisConnection { + inner: RedisConnectionInner, +} + +enum RedisConnectionInner { + StandalonePooled(deadpool_redis::Connection), + ClusterPooled(deadpool_redis::cluster::Connection), + ClusterMultiplexed(redis::cluster_async::ClusterConnection), +} + +impl RedisBackend { + pub(super) async fn new( + config: &RedisConfig, + ) -> Result { + match config.backend() { + RedisBackendConfig::StandalonePooled(pool_size) => { + Self::standalone_pooled(config, pool_size).await + } + RedisBackendConfig::ClusterPooled(pool_size) => { + Self::cluster_pooled(config, pool_size).await + } + RedisBackendConfig::ClusterMultiplexed => { + Self::cluster_multiplexed(config).await + } + } + } + + async fn standalone_pooled( + config: &RedisConfig, + pool_size: RedisPoolSize, + ) -> Result { + let connection_config = redis::AsyncConnectionConfig::new() + .set_connection_timeout(None) + .set_response_timeout(None); + let manager = deadpool_redis::Manager::new_with_config( + config.seed_urls()[0].clone(), + connection_config, + )?; + let pool = deadpool_redis::Pool::builder(manager) + .max_size(pool_size.max()) + .wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms()))) + .runtime(deadpool_redis::Runtime::Tokio1) + .build()?; + + warm_standalone_pool(&pool, pool_size.min()).await?; + retain_standalone_pool(pool.clone()); + + Ok(Self::StandalonePooled(pool)) + } + + async fn cluster_pooled( + config: &RedisConfig, + pool_size: RedisPoolSize, + ) -> Result { + let manager = deadpool_redis::cluster::Manager::new( + config.seed_urls().to_vec(), + false, + )?; + let pool = deadpool_redis::cluster::Pool::builder(manager) + .max_size(pool_size.max()) + .wait_timeout(Some(Duration::from_millis(config.wait_timeout_ms()))) + .runtime(deadpool_redis::Runtime::Tokio1) + .build()?; + + warm_cluster_pool(&pool, pool_size.min()).await?; + retain_cluster_pool(pool.clone()); + + Ok(Self::ClusterPooled(pool)) + } + + async fn cluster_multiplexed( + config: &RedisConfig, + ) -> Result { + let client = redis::cluster::ClusterClientBuilder::new( + config.seed_urls().iter().map(String::as_str), + ) + .build()?; + let connection = client.get_async_connection().await?; + + Ok(Self::ClusterMultiplexed(connection)) + } + + pub(super) async fn connect( + &self, + ) -> Result { + let inner = match self { + Self::StandalonePooled(pool) => { + RedisConnectionInner::StandalonePooled(pool.get().await?) + } + Self::ClusterPooled(pool) => { + RedisConnectionInner::ClusterPooled(pool.get().await?) + } + Self::ClusterMultiplexed(connection) => { + RedisConnectionInner::ClusterMultiplexed(connection.clone()) + } + }; + + Ok(RedisConnection { inner }) + } + + pub(super) fn register_metrics( + &self, + registry: &Registry, + ) -> Result<(), prometheus::Error> { + register_command_pool_metrics(registry, self.clone()) + } +} + +impl LogicalPoolStatusProvider for RedisBackend { + fn logical_pool_status(&self) -> LogicalPoolStatus { + match self { + Self::StandalonePooled(pool) => { + LogicalPoolStatus::from_deadpool(pool.status()) + } + Self::ClusterPooled(pool) => { + LogicalPoolStatus::from_deadpool(pool.status()) + } + Self::ClusterMultiplexed(_) => { + LogicalPoolStatus::shared_multiplexed() + } + } + } +} + +impl ConnectionLike for RedisConnectionInner { + fn req_packed_command<'a>( + &'a mut self, + cmd: &'a redis::Cmd, + ) -> redis::RedisFuture<'a, redis::Value> { + match self { + Self::StandalonePooled(connection) => { + connection.req_packed_command(cmd) + } + Self::ClusterPooled(connection) => { + connection.req_packed_command(cmd) + } + Self::ClusterMultiplexed(connection) => { + connection.req_packed_command(cmd) + } + } + } + + fn req_packed_commands<'a>( + &'a mut self, + cmd: &'a redis::Pipeline, + offset: usize, + count: usize, + ) -> redis::RedisFuture<'a, Vec> { + match self { + Self::StandalonePooled(connection) => { + connection.req_packed_commands(cmd, offset, count) + } + Self::ClusterPooled(connection) => { + connection.req_packed_commands(cmd, offset, count) + } + Self::ClusterMultiplexed(connection) => { + connection.req_packed_commands(cmd, offset, count) + } + } + } + + fn get_db(&self) -> i64 { + match self { + Self::StandalonePooled(connection) => connection.get_db(), + Self::ClusterPooled(connection) => connection.get_db(), + Self::ClusterMultiplexed(connection) => connection.get_db(), + } + } +} + +impl ConnectionLike for RedisConnection { + fn req_packed_command<'a>( + &'a mut self, + cmd: &'a redis::Cmd, + ) -> redis::RedisFuture<'a, redis::Value> { + self.inner.req_packed_command(cmd) + } + + fn req_packed_commands<'a>( + &'a mut self, + cmd: &'a redis::Pipeline, + offset: usize, + count: usize, + ) -> redis::RedisFuture<'a, Vec> { + self.inner.req_packed_commands(cmd, offset, count) + } + + fn get_db(&self) -> i64 { + self.inner.get_db() + } +} + +async fn warm_standalone_pool( + pool: &deadpool_redis::Pool, + min: usize, +) -> Result<(), deadpool_redis::PoolError> { + let connections = try_join_all((0..min).map(|_| pool.get())).await?; + drop(connections); + Ok(()) +} + +async fn warm_cluster_pool( + pool: &deadpool_redis::cluster::Pool, + min: usize, +) -> Result<(), deadpool_redis::PoolError> { + let connections = try_join_all((0..min).map(|_| pool.get())).await?; + drop(connections); + Ok(()) +} + +fn retain_standalone_pool(pool: deadpool_redis::Pool) { + tokio::spawn(async move { + loop { + tokio::time::sleep(POOL_RETAIN_INTERVAL).await; + pool.retain(|_, metrics| { + metrics.last_used() < MAX_IDLE_CONNECTION_AGE + && metrics.created.elapsed() < MAX_STANDALONE_CONNECTION_AGE + }); + } + }); +} + +fn retain_cluster_pool(pool: deadpool_redis::cluster::Pool) { + tokio::spawn(async move { + loop { + tokio::time::sleep(POOL_RETAIN_INTERVAL).await; + pool.retain(|_, metrics| { + metrics.last_used() < MAX_IDLE_CONNECTION_AGE + }); + } + }); +} diff --git a/apps/labrinth/src/database/redis/key.rs b/apps/labrinth/src/database/redis/key.rs new file mode 100644 index 0000000000..6181284cc4 --- /dev/null +++ b/apps/labrinth/src/database/redis/key.rs @@ -0,0 +1,72 @@ +use std::fmt::Display; +use std::sync::Arc; + +use super::RedisMode; + +#[derive(Debug, Clone)] +pub struct KeyBuilder { + meta_namespace: Arc, + mode: RedisMode, +} + +impl KeyBuilder { + pub fn new(meta_namespace: impl Into>, mode: RedisMode) -> Self { + Self { + meta_namespace: meta_namespace.into(), + mode, + } + } + + /// Build a key with the given namespace and logical key. The logical key is used as the key's slot tag. + pub fn entity(&self, namespace: &str, logical_key: impl Display) -> String { + let logical_key = logical_key.to_string(); + self.with_slot(namespace, &logical_key, &logical_key) + } + + /// Build a metadata key with the given namespace and logical key. The slot tag is fixed to `_metadata`. + pub fn metadata( + &self, + namespace: &str, + logical_key: impl Display, + ) -> String { + self.with_slot(namespace, logical_key, "_metadata") + } + + /// Build a key with the given namespace, logical key, and slot tag. + pub fn with_slot( + &self, + namespace: &str, + logical_key: impl Display, + slot_tag: impl Display, + ) -> String { + match self.mode { + RedisMode::Standalone => { + format!("{}_{}:{}", self.meta_namespace, namespace, logical_key) + } + RedisMode::Cluster => format!( + "{}_{}:{{{}}}:{}", + self.meta_namespace, + namespace, + escape_slot_tag(&slot_tag.to_string()), + logical_key + ), + } + } +} + +fn escape_slot_tag(value: &str) -> String { + if value.is_empty() { + return "%00".to_string(); + } + + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '%' => escaped.push_str("%25"), + '{' => escaped.push_str("%7B"), + '}' => escaped.push_str("%7D"), + _ => escaped.push(character), + } + } + escaped +} diff --git a/apps/labrinth/src/database/redis/metrics.rs b/apps/labrinth/src/database/redis/metrics.rs new file mode 100644 index 0000000000..d50e4cce47 --- /dev/null +++ b/apps/labrinth/src/database/redis/metrics.rs @@ -0,0 +1,161 @@ +use std::time::Duration; + +use prometheus::{IntGauge, Registry}; + +const METRICS_UPDATE_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone, Copy)] +pub(super) struct LogicalPoolStatus { + max_size: usize, + size: usize, + available: usize, + waiting: usize, +} + +impl LogicalPoolStatus { + pub(super) fn from_deadpool(status: deadpool_redis::Status) -> Self { + Self { + max_size: status.max_size, + size: status.size, + available: status.available, + waiting: status.waiting, + } + } + + pub(super) fn shared_multiplexed() -> Self { + Self { + max_size: 1, + size: 1, + available: 1, + waiting: 0, + } + } +} + +pub(super) trait LogicalPoolStatusProvider: + Clone + Send + 'static +{ + fn logical_pool_status(&self) -> LogicalPoolStatus; +} + +#[derive(Debug, Clone, Copy)] +enum RedisPoolMetricsKind { + Command, + Blocking, +} + +impl RedisPoolMetricsKind { + fn metric_prefix(self) -> &'static str { + match self { + Self::Command => "labrinth_redis_pool", + Self::Blocking => "labrinth_redis_blocking_pool", + } + } + + fn description(self) -> &'static str { + match self { + Self::Command => "Redis command pool", + Self::Blocking => "Redis blocking-command pool", + } + } +} + +#[derive(Clone)] +struct RedisPoolMetrics { + max_size: IntGauge, + size: IntGauge, + available: IntGauge, + waiting: IntGauge, +} + +impl RedisPoolMetrics { + fn register( + registry: &Registry, + kind: RedisPoolMetricsKind, + ) -> Result { + let prefix = kind.metric_prefix(); + let description = kind.description(); + let max_size = IntGauge::new( + format!("{prefix}_max_size"), + format!( + "Maximum logical connection count for the {description}; clustered logical connections may own multiple physical sockets" + ), + )?; + let size = IntGauge::new( + format!("{prefix}_size"), + format!( + "Current logical connection count for the {description}; clustered logical connections may own multiple physical sockets" + ), + )?; + let available = IntGauge::new( + format!("{prefix}_available"), + format!("Available logical connections in the {description}"), + )?; + let waiting = IntGauge::new( + format!("{prefix}_waiting"), + format!( + "Number of futures waiting for a logical connection from the {description}" + ), + )?; + + registry.register(Box::new(max_size.clone()))?; + registry.register(Box::new(size.clone()))?; + registry.register(Box::new(available.clone()))?; + registry.register(Box::new(waiting.clone()))?; + + Ok(Self { + max_size, + size, + available, + waiting, + }) + } + + fn set(&self, status: LogicalPoolStatus) { + self.max_size.set(status.max_size as i64); + self.size.set(status.size as i64); + self.available.set(status.available as i64); + self.waiting.set(status.waiting as i64); + } +} + +pub(super) fn register_command_pool_metrics

( + registry: &Registry, + provider: P, +) -> Result<(), prometheus::Error> +where + P: LogicalPoolStatusProvider, +{ + register_pool_metrics(registry, RedisPoolMetricsKind::Command, provider) +} + +pub(super) fn register_blocking_pool_metrics

( + registry: &Registry, + provider: P, +) -> Result<(), prometheus::Error> +where + P: LogicalPoolStatusProvider, +{ + register_pool_metrics(registry, RedisPoolMetricsKind::Blocking, provider) +} + +fn register_pool_metrics

( + registry: &Registry, + kind: RedisPoolMetricsKind, + provider: P, +) -> Result<(), prometheus::Error> +where + P: LogicalPoolStatusProvider, +{ + let metrics = RedisPoolMetrics::register(registry, kind)?; + metrics.set(provider.logical_pool_status()); + + tokio::spawn(async move { + loop { + tokio::time::sleep(METRICS_UPDATE_INTERVAL).await; + metrics.set(provider.logical_pool_status()); + } + }); + + Ok(()) +} diff --git a/apps/labrinth/src/database/redis/mod.rs b/apps/labrinth/src/database/redis/mod.rs index acab21af1e..3d2c2f36e8 100644 --- a/apps/labrinth/src/database/redis/mod.rs +++ b/apps/labrinth/src/database/redis/mod.rs @@ -1,299 +1,103 @@ -use crate::env::ENV; - -use super::models::DatabaseError; -use ariadne::ids::base62_impl::{parse_base62, to_base62}; -use chrono::{TimeZone, Utc}; -use dashmap::DashMap; -use deadpool_redis::Runtime; -use futures::TryStreamExt; -use futures::future::Either; -use futures::stream::{FuturesUnordered, StreamExt}; -use prometheus::{IntGauge, Registry}; -use redis::ToRedisArgs; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; -use std::borrow::Cow; -use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::future::Future; use std::hash::Hash; -use std::str::FromStr; use std::sync::Arc; -use std::time::Duration; -use thiserror::Error; -use tracing::{Instrument, info, info_span}; -use util::{cmd, redis_pipe}; - -pub mod util; - -const DEFAULT_EXPIRY: i64 = 60 * 60 * 12; // 12 hours -const ACTUAL_EXPIRY: i64 = 60 * 30; // 30 minutes -const VERSION_DEFAULT_EXPIRY: i64 = 60 * 60 * 48; // 48 hours -const VERSION_ACTUAL_EXPIRY: i64 = 60 * 60 * 24; // 24 hours - -// Bound how many commands we send in a single Redis pipeline. The multiplexed -// connection's BytesMut write buffer keeps its peak capacity for the life of -// the connection, so larger pipelines cause higher steady-state RSS. -const PIPELINE_CHUNK_SIZE: usize = 25; -// Bound how many keys we send in a single MGET. Each MGET response must fit -// into the connection's read buffer, which also retains its peak capacity. At -// ~1 MB per cached value, 32 keys caps any single response at ~32 MB. -const MGET_CHUNK_SIZE: usize = 32; -// How long a pooled Redis connection lives before being recycled, regardless -// of activity. Forced recycling is the only way to release the per-connection -// BytesMut peak capacity that builds up under steady load. -const REDIS_MAX_CONN_AGE: Duration = Duration::from_secs(120); - -#[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Codec { - Raw = 0, - Lz4 = 1, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EncodingFormat { - Json, - Postcard, -} - -#[derive(Debug, Error)] -#[error("invalid redis codec")] -pub struct InvalidCodec; - -#[derive(Debug, Error)] -#[error("invalid redis encoding format")] -pub struct InvalidEncodingFormat; - -impl TryFrom for Codec { - type Error = InvalidCodec; - - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(Self::Raw), - 1 => Ok(Self::Lz4), - _ => Err(InvalidCodec), - } - } -} - -impl FromStr for Codec { - type Err = InvalidCodec; - - fn from_str(value: &str) -> Result { - match value { - "lz4" => Ok(Self::Lz4), - _ => Err(InvalidCodec), - } - } -} -impl FromStr for EncodingFormat { - type Err = InvalidEncodingFormat; - - fn from_str(value: &str) -> Result { - match value { - "json" => Ok(Self::Json), - "postcard" => Ok(Self::Postcard), - _ => Err(InvalidEncodingFormat), - } - } -} - -fn encode_value(value: &T) -> Result, DatabaseError> { - let mut value = match ENV.REDIS_ENCODING_FORMAT { - EncodingFormat::Json => serde_json::to_vec(value)?, - EncodingFormat::Postcard => postcard::to_allocvec(value)?, - }; - - if ENV.REDIS_COMPRESSION_LEVEL > 0 - && ENV.REDIS_COMPRESSION_ALGORITHM == Codec::Lz4 - && value.len() >= ENV.REDIS_COMPRESSION_THRESHOLD_BYTES - { - let compressed = lz4_flex::block::compress_prepend_size(&value); - let savings_ratio = value.len().saturating_sub(compressed.len()) as f64 - / value.len().max(1) as f64 - * 100.0; - - if savings_ratio >= ENV.REDIS_COMPRESSION_MIN_SAVINGS_RATIO { - let mut encoded = Vec::with_capacity(compressed.len() + 1); - encoded.push(Codec::Lz4 as u8); - encoded.extend(compressed); - return Ok(encoded); - } - } - - let mut encoded = Vec::with_capacity(value.len() + 1); - encoded.push(Codec::Raw as u8); - encoded.append(&mut value); - Ok(encoded) -} - -fn decode_value(value: &[u8]) -> Option -where - T: for<'a> Deserialize<'a>, -{ - let (codec, value) = value.split_first()?; - let value = match Codec::try_from(*codec).ok()? { - Codec::Raw => Cow::Borrowed(value), - Codec::Lz4 => { - Cow::Owned(lz4_flex::block::decompress_size_prepended(value).ok()?) - } - }; +use dashmap::DashMap; +use prometheus::Registry; +use redis::ToRedisArgs; +use redis::aio::ConnectionLike; +use serde::Serialize; +use serde::de::DeserializeOwned; - match ENV.REDIS_ENCODING_FORMAT { - EncodingFormat::Json => serde_json::from_slice(&value).ok(), - EncodingFormat::Postcard => postcard::from_bytes(&value).ok(), - } -} +use crate::database::models::DatabaseError; +use crate::env::ENV; -fn cache_expiries(namespace: &str) -> (i64, i64) { - // Namespaces may embed a version suffix like `:v1`, so split it out. - match namespace.split_once(':').map(|t| t.0).unwrap_or(namespace) { - "versions" | "versions_files" => { - (VERSION_DEFAULT_EXPIRY, VERSION_ACTUAL_EXPIRY) - } - _ => (DEFAULT_EXPIRY, ACTUAL_EXPIRY), - } -} +mod blocking; +mod cache; +mod commands; +mod config; +mod connection; +mod key; +mod metrics; +mod pubsub; +mod util; + +use cache::{CacheManager, CacheSettings, ConnectionProvider}; +pub use cache::{Codec, EncodingFormat, RedisValue}; +use config::RedisConfig; +pub use config::{RedisConnectionType, RedisMode}; +use connection::RedisBackend; +pub use key::KeyBuilder; #[derive(Clone)] pub struct RedisPool { - pub url: String, - pub pool: deadpool_redis::Pool, - cache_list: Arc>, - meta_namespace: Arc, + backend: RedisBackend, + blocking: blocking::RedisBlockingPool, + cache: CacheManager, + config: RedisConfig, + key_builder: KeyBuilder, } pub struct RedisConnection { - pub connection: deadpool_redis::Connection, - meta_namespace: Arc, + inner: connection::RedisConnection, + key_builder: KeyBuilder, + settings: CacheSettings, } impl RedisPool { - // initiate a new redis pool - // testing pool uses a hashmap to mimic redis behaviour for very small data sizes (ie: tests) - // PANICS: production pool will panic if redis url is not set - pub fn new(meta_namespace: impl Into>) -> Self { - let wait_timeout = Duration::from_millis(ENV.REDIS_WAIT_TIMEOUT_MS); - - let url = &ENV.REDIS_URL; - // Redis 1.x enables async connection and response timeouts by default. - // Preserve the previous behavior because this pool still serves BRPOP - let connection_config = redis::AsyncConnectionConfig::new() - .set_connection_timeout(None) - .set_response_timeout(None); - let manager = deadpool_redis::Manager::new_with_config( - url.clone(), - connection_config, - ) - .expect("Error building Redis pool"); - let pool = deadpool_redis::Pool::builder(manager) - .max_size(ENV.REDIS_MAX_CONNECTIONS as usize) - .wait_timeout(Some(wait_timeout)) - .runtime(Runtime::Tokio1) - .build() - .expect("Redis connection failed"); - - let pool = RedisPool { - url: url.clone(), - pool, - cache_list: Arc::new(DashMap::with_capacity(2048)), - meta_namespace: meta_namespace.into(), - }; + pub async fn new(meta_namespace: impl Into>) -> Self { + let config = + RedisConfig::from_env().expect("invalid Redis configuration"); + let backend = RedisBackend::new(&config) + .await + .expect("failed to initialize Redis connections"); + let blocking = blocking::RedisBlockingPool::new(&config) + .await + .expect("failed to initialize blocking Redis connections"); + let key_builder = KeyBuilder::new(meta_namespace, config.mode()); + let cache = CacheManager::new( + key_builder.clone(), + CacheSettings { + encoding_format: ENV.REDIS_ENCODING_FORMAT, + compression_algorithm: ENV.REDIS_COMPRESSION_ALGORITHM, + compression_level: ENV.REDIS_COMPRESSION_LEVEL, + compression_threshold_bytes: ENV + .REDIS_COMPRESSION_THRESHOLD_BYTES, + compression_min_savings_ratio: ENV + .REDIS_COMPRESSION_MIN_SAVINGS_RATIO, + }, + ); - let redis_min_connections = ENV.REDIS_MIN_CONNECTIONS; - let spawn_min_connections = (0..redis_min_connections) - .map(|_| { - let pool = pool.clone(); - tokio::spawn(async move { pool.pool.get().await }) - }) - .collect::>(); - tokio::spawn({ - let pool = pool.clone(); - async move { - // collect the connections into a buffer while we're spawning them, - // to make sure that we're not `get`ing any connections we previously took - let _connections = - spawn_min_connections.try_collect::>().await; - info!( - pool_status = ?pool.pool.status(), - "Finished getting {redis_min_connections} initial Redis connections" - ); - } - }); + Self { + backend, + blocking, + cache, + config, + key_builder, + } + } - let interval = Duration::from_secs(30); - let max_idle = Duration::from_secs(5 * 60); // 5 minutes - let pool_ref = pool.clone(); - tokio::spawn(async move { - loop { - tokio::time::sleep(interval).await; - pool_ref.pool.retain(|_, metrics| { - // Drop connections that have been idle too long, OR that - // are older than REDIS_MAX_CONN_AGE regardless of use. - // The age-based recycle is what releases the per-connection - // BytesMut peak capacity under steady traffic. - metrics.last_used() < max_idle - && metrics.created.elapsed() < REDIS_MAX_CONN_AGE - }); - } - }); + pub fn key(&self) -> &KeyBuilder { + &self.key_builder + } - pool + pub async fn connect(&self) -> Result { + Ok(RedisConnection { + inner: self.backend.connect().await?, + key_builder: self.key_builder.clone(), + settings: self.cache.settings().clone(), + }) } pub async fn register_and_set_metrics( &self, registry: &Registry, ) -> Result<(), prometheus::Error> { - let redis_max_size = IntGauge::new( - "labrinth_redis_pool_max_size", - "Maximum size of Redis pool", - )?; - let redis_size = IntGauge::new( - "labrinth_redis_pool_size", - "Current size of Redis pool", - )?; - let redis_available = IntGauge::new( - "labrinth_redis_pool_available", - "Available connections in Redis pool", - )?; - let redis_waiting = IntGauge::new( - "labrinth_redis_pool_waiting", - "Number of futures waiting for a Redis connection", - )?; - - registry.register(Box::new(redis_max_size.clone()))?; - registry.register(Box::new(redis_size.clone()))?; - registry.register(Box::new(redis_available.clone()))?; - registry.register(Box::new(redis_waiting.clone()))?; - - let redis_pool_ref = self.pool.clone(); - tokio::spawn(async move { - loop { - let status = redis_pool_ref.status(); - redis_max_size.set(status.max_size as i64); - redis_size.set(status.size as i64); - redis_available.set(status.available as i64); - redis_waiting.set(status.waiting as i64); - - tokio::time::sleep(Duration::from_secs(5)).await; - } - }); - - Ok(()) + self.backend.register_metrics(registry)?; + self.blocking.register_metrics(registry) } - #[tracing::instrument(skip(self))] - pub async fn connect(&self) -> Result { - Ok(RedisConnection { - connection: self.pool.get().await?, - meta_namespace: self.meta_namespace.clone(), - }) - } - - #[tracing::instrument(skip(self, closure))] pub async fn get_cached_keys( &self, namespace: &str, @@ -313,21 +117,17 @@ impl RedisPool { + Serialize + Debug, { - Ok(self - .get_cached_keys_raw(namespace, keys, closure) - .await? - .into_iter() - .map(|x| x.1) - .collect()) + self.cache + .get_cached_keys(self, namespace, keys, closure) + .await } - #[tracing::instrument(skip(self, closure))] pub async fn get_cached_keys_raw( &self, namespace: &str, keys: &[K], closure: F, - ) -> Result, DatabaseError> + ) -> Result, DatabaseError> where F: FnOnce(Vec) -> Fut, Fut: Future, DatabaseError>>, @@ -341,23 +141,11 @@ impl RedisPool { + Serialize + Debug, { - self.get_cached_keys_raw_with_slug( - namespace, - None, - false, - keys, - |ids| async move { - Ok(closure(ids) - .await? - .into_iter() - .map(|(key, val)| (key, (None::, val))) - .collect()) - }, - ) - .await + self.cache + .get_cached_keys_raw(self, namespace, keys, closure) + .await } - #[tracing::instrument(skip(self, closure))] pub async fn get_cached_keys_with_slug( &self, namespace: &str, @@ -380,21 +168,18 @@ impl RedisPool { + Serialize, S: Display + Clone + DeserializeOwned + Serialize + Debug, { - Ok(self - .get_cached_keys_raw_with_slug( + self.cache + .get_cached_keys_with_slug( + self, namespace, - Some(slug_namespace), + slug_namespace, case_sensitive, keys, closure, ) - .await? - .into_iter() - .map(|x| x.1) - .collect()) + .await } - #[tracing::instrument(skip(self, closure))] pub async fn get_cached_keys_raw_with_slug( &self, namespace: &str, @@ -402,7 +187,7 @@ impl RedisPool { case_sensitive: bool, keys: &[I], closure: F, - ) -> Result, DatabaseError> + ) -> Result, DatabaseError> where F: FnOnce(Vec) -> Fut, Fut: Future, T)>, DatabaseError>>, @@ -417,553 +202,149 @@ impl RedisPool { + Serialize, S: Display + Clone + DeserializeOwned + Serialize + Debug, { - let ids = keys - .iter() - .map(|x| (x.to_string(), x.clone())) - .collect::>(); - - if ids.is_empty() { - return Ok(HashMap::new()); - } - - let get_cached_values = |ids: DashMap| { - async move { - let slug_ids = if let Some(slug_namespace) = slug_namespace { - async { - let mut connection = self.pool.get().await?; - - let args = ids - .iter() - .map(|x| { - format!( - "{}_{slug_namespace}:{}", - self.meta_namespace, - if case_sensitive { - x.value().to_string() - } else { - x.value().to_string().to_lowercase() - } - ) - }) - .collect::>(); - - let mut v = Vec::new(); - for chunk in args.chunks(MGET_CHUNK_SIZE) { - let part = cmd("MGET") - .arg(chunk) - .query_async::>>( - &mut connection, - ) - .await?; - v.extend(part.into_iter().flatten()); - } - Ok::<_, DatabaseError>(v) - } - .instrument(info_span!("get slug ids")) - .await? - } else { - Vec::new() - }; - - let mut connection = self.pool.get().await?; - let args = ids - .iter() - .map(|x| x.value().to_string()) - .chain(ids.iter().filter_map(|x| { - parse_base62(&x.value().to_string()) - .ok() - .map(|x| x.to_string()) - })) - .chain(slug_ids) - .map(|x| format!("{}_{namespace}:{x}", self.meta_namespace)) - .collect::>(); - - let mut cached_values = HashMap::new(); - for chunk in args.chunks(MGET_CHUNK_SIZE) { - let part = cmd("MGET") - .arg(chunk) - .query_async::>>>(&mut connection) - .await?; - cached_values.extend(part.into_iter().filter_map(|x| { - x.and_then(|val| { - decode_value::>(&val) - }) - .map(|val| (val.key.clone(), val)) - })); - } - - Ok::<_, DatabaseError>((cached_values, ids)) - } - .instrument(info_span!("get cached values")) - }; - - let (default_expiry, actual_expiry) = cache_expiries(namespace); - let current_time = Utc::now(); - let mut expired_values = HashMap::new(); - - let (cached_values_raw, ids) = get_cached_values(ids).await?; - let mut cached_values = cached_values_raw - .into_iter() - .filter_map(|(key, val)| { - if Utc.timestamp_opt(val.iat + actual_expiry, 0).unwrap() - < current_time - { - expired_values.insert(val.key.to_string(), val); - - None - } else { - let key_str = val.key.to_string(); - ids.remove(&key_str); - - if let Ok(value) = key_str.parse::() { - let base62 = to_base62(value); - ids.remove(&base62); - } - - if let Some(ref alias) = val.alias { - ids.remove(&alias.to_string()); - } - - Some((key, val)) - } - }) - .collect::>(); - - let subscribe_ids = DashMap::new(); - let mut cache_writers = HashMap::new(); - - if !ids.is_empty() { - let fetch_ids = - ids.iter().map(|x| x.key().clone()).collect::>(); - - fetch_ids.into_iter().for_each(|key| { - let ns_key_value = if case_sensitive { - key.to_lowercase() - } else { - key.clone() - }; - let namespaced_key = format!( - "{}_{namespace}:{ns_key_value}", - self.meta_namespace, - ); - let either = self.acquire_lock(namespaced_key); - - match either { - Either::Left(sentinel) => { - cache_writers.insert(key, sentinel); - } - - Either::Right(subscriber) => { - if let Some((key, raw_key)) = ids.remove(&key) { - if let Some(val) = expired_values.remove(&key) { - if let Some(ref alias) = val.alias { - ids.remove(&alias.to_string()); - } - - if let Ok(value) = - val.key.to_string().parse::() - { - let base62 = to_base62(value); - ids.remove(&base62); - } - - cached_values.insert(val.key.clone(), val); - } else { - subscribe_ids.insert(raw_key, subscriber); - } - } - } - } - }); - } - - let mut fetch_tasks = Vec::new(); - - if !ids.is_empty() { - fetch_tasks.push(Either::Left(async { - let fetch_ids = - ids.iter().map(|x| x.value().clone()).collect::>(); - - let vals = closure(fetch_ids).await?; - let mut return_values = HashMap::new(); - - let mut pipe = redis_pipe(); - let mut pipe_cmds: usize = 0; - let mut connection = self.pool.get().await?; - // Doesn't need to be atomic - - if !vals.is_empty() { - for (key, (slug, value)) in vals { - let value = RedisValue { - key: key.clone(), - iat: Utc::now().timestamp(), - val: value, - alias: slug.clone(), - }; - - pipe.set_ex( - format!( - "{}_{namespace}:{key}", - self.meta_namespace - ), - encode_value(&value)?, - default_expiry as u64, - ); - pipe_cmds += 1; - - if let Some(slug) = slug { - ids.remove(&slug.to_string()); - - if let Some(slug_namespace) = slug_namespace { - let actual_slug = if case_sensitive { - slug.to_string() - } else { - slug.to_string().to_lowercase() - }; - - pipe.set_ex( - format!( - "{}_{slug_namespace}:{}", - self.meta_namespace, actual_slug - ), - key.to_string(), - default_expiry as u64, - ); - pipe_cmds += 1; - } - } - - let key_str = key.to_string(); - ids.remove(&key_str); - - if let Ok(value) = key_str.parse::() { - let base62 = to_base62(value); - ids.remove(&base62); - } - - return_values.insert(key, value); - - if pipe_cmds >= PIPELINE_CHUNK_SIZE { - pipe.query_async::<()>(&mut connection).await?; - pipe = redis_pipe(); - pipe_cmds = 0; - } - } - } - - if pipe_cmds > 0 { - pipe.query_async::<()>(&mut connection).await?; - } - - drop(cache_writers); - - Result::<_, DatabaseError>::Ok(return_values) - })); - } - - if !subscribe_ids.is_empty() { - fetch_tasks.push(Either::Right(async move { - let mut futures = FuturesUnordered::new(); - let len = subscribe_ids.len(); - - for (key, subscriber) in subscribe_ids { - futures.push(async move { - ( - key, - subscriber - .wait_timeout(Duration::from_secs(5)) - .await, - ) - }); - } - - let fetch_ids = DashMap::with_capacity(len); - while let Some((key, result)) = futures.next().await { - result?; - fetch_ids.insert(key.to_string(), key); - } - - let (return_values, _) = get_cached_values(fetch_ids).await?; - Ok(return_values) - })); - } - - if !fetch_tasks.is_empty() { - for map in futures::future::try_join_all(fetch_tasks).await? { - for (key, value) in map { - cached_values.insert(key, value); - } - } - } - - Ok(cached_values.into_iter().map(|x| (x.0, x.1.val)).collect()) - } - - /// Acquire or create a cache lock onto the given key. - fn acquire_lock( - &self, - key: String, - ) -> Either, util::CacheSubscriber> { - let mut out_writer = None; - let subscriber = - self.cache_list.entry(key.clone()).or_insert_with(|| { - let (writer, subscriber) = util::cache(); - out_writer = Some(writer); - subscriber - }); - - match out_writer { - Some(writer) => Either::Left(LockSentinel { - pool: self, - key, - writer, - }), - None => Either::Right(subscriber.clone()), - } + self.cache + .get_cached_keys_raw_with_slug( + self, + namespace, + slug_namespace, + case_sensitive, + keys, + closure, + ) + .await } } -struct LockSentinel<'a> { - pool: &'a RedisPool, - key: String, - writer: util::CacheWriter, -} +impl ConnectionProvider for RedisPool { + type Connection = RedisConnection; -impl<'a> Drop for LockSentinel<'a> { - fn drop(&mut self) { - self.writer.write(); - self.pool.cache_list.remove(&self.key); + fn connect( + &self, + ) -> impl Future> + Send + { + RedisPool::connect(self) } } impl RedisConnection { - #[tracing::instrument(skip(self))] + pub fn keyspace(&self) -> &KeyBuilder { + &self.key_builder + } + pub async fn set( &mut self, - namespace: &str, - id: &str, + key: &str, data: D, expiry: Option, ) -> Result<(), DatabaseError> where D: ToRedisArgs + Send + Sync + Debug, { - let mut cmd = cmd("SET"); - cmd.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id)) - .arg(data) - .arg("EX") - .arg(expiry.unwrap_or(DEFAULT_EXPIRY)); - redis_execute::<()>(&mut cmd, &mut self.connection).await?; - Ok(()) + commands::set(&mut self.inner, key, data, expiry).await } - #[tracing::instrument(skip(self, id, data))] - pub async fn set_serialized( + pub async fn set_serialized( &mut self, - namespace: &str, - id: Id, + key: &str, data: D, expiry: Option, ) -> Result<(), DatabaseError> where - Id: Display, - D: serde::Serialize, + D: Serialize, { - self.set(namespace, &id.to_string(), encode_value(&data)?, expiry) - .await + commands::set_serialized( + &mut self.inner, + key, + data, + expiry, + &self.settings, + ) + .await } - #[tracing::instrument(skip(self))] pub async fn get( &mut self, - namespace: &str, - id: &str, + key: &str, ) -> Result, DatabaseError> { - let mut cmd = cmd("GET"); - redis_args( - &mut cmd, - vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)] - .as_slice(), - ); - let res = redis_execute(&mut cmd, &mut self.connection).await?; - Ok(res) + commands::get(&mut self.inner, key).await } - #[tracing::instrument(skip(self))] pub async fn get_many( &mut self, - namespace: &str, - ids: &[String], + keys: &[String], ) -> Result>>, DatabaseError> { - let mut cmd = cmd("MGET"); - redis_args( - &mut cmd, - ids.iter() - .map(|x| format!("{}_{}:{}", self.meta_namespace, namespace, x)) - .collect::>() - .as_slice(), - ); - let res = redis_execute(&mut cmd, &mut self.connection).await?; - Ok(res) + commands::get_many(&mut self.inner, keys).await } - #[tracing::instrument(skip(self))] pub async fn get_deserialized( &mut self, - namespace: &str, - id: &str, + key: &str, ) -> Result, DatabaseError> where R: for<'a> serde::Deserialize<'a>, { - let mut cmd = cmd("GET"); - redis_args( - &mut cmd, - vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)] - .as_slice(), - ); - let value: Option> = - redis_execute(&mut cmd, &mut self.connection).await?; - Ok(value.and_then(|value| decode_value(&value))) + commands::get_deserialized(&mut self.inner, key, &self.settings).await } - #[tracing::instrument(skip(self))] pub async fn get_many_deserialized( &mut self, - namespace: &str, - ids: &[String], + keys: &[String], ) -> Result>, DatabaseError> where R: for<'a> serde::Deserialize<'a>, { - Ok(self - .get_many(namespace, ids) - .await? - .into_iter() - .map(|value| value.and_then(|value| decode_value::(&value))) - .collect()) + commands::get_many_deserialized(&mut self.inner, keys, &self.settings) + .await } - #[tracing::instrument(skip(self, id))] - pub async fn delete( - &mut self, - namespace: &str, - id: T1, - ) -> Result<(), DatabaseError> - where - T1: Display, - { - let mut cmd = cmd("DEL"); - redis_args( - &mut cmd, - vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)] - .as_slice(), - ); - redis_execute::<()>(&mut cmd, &mut self.connection).await?; - Ok(()) + pub async fn delete(&mut self, key: &str) -> Result<(), DatabaseError> { + commands::delete(&mut self.inner, key).await } - #[tracing::instrument(skip(self, iter))] pub async fn delete_many( &mut self, - iter: impl IntoIterator)>, - ) -> Result<(), DatabaseError> { - let mut cmd = cmd("DEL"); - let mut any = false; - for (namespace, id) in iter { - if let Some(id) = id { - redis_args( - &mut cmd, - [format!("{}_{}:{}", self.meta_namespace, namespace, id)] - .as_slice(), - ); - any = true; - } - } - - if any { - redis_execute::<()>(&mut cmd, &mut self.connection).await?; - } - - Ok(()) - } - - #[tracing::instrument(skip(self, value))] - pub async fn lpush( - &mut self, - namespace: &str, - key: &str, - value: impl ToRedisArgs + Send + Sync + Debug, + keys: &[String], ) -> Result<(), DatabaseError> { - let key = format!("{}_{namespace}:{key}", self.meta_namespace); - cmd("LPUSH") - .arg(key) - .arg(value) - .query_async::<()>(&mut self.connection) - .await?; - Ok(()) + commands::delete_many(&mut self.inner, keys).await } - #[tracing::instrument(skip(self))] - pub async fn brpop( + pub async fn lpush( &mut self, - namespace: &str, key: &str, - timeout: Option, - ) -> Result; 2]>, DatabaseError> { - let key = format!("{}_{namespace}:{key}", self.meta_namespace); - // a timeout of 0 is infinite - let timeout = timeout.unwrap_or(0.0); - let values = cmd("BRPOP") - .arg(key) - .arg(timeout) - .query_async(&mut self.connection) - .await?; - Ok(values) + value: D, + ) -> Result<(), DatabaseError> + where + D: ToRedisArgs + Send + Sync + Debug, + { + commands::lpush(&mut self.inner, key, value).await } - #[tracing::instrument(skip(self))] pub async fn incr( &mut self, - namespace: &str, - id: &str, + key: &str, ) -> Result, DatabaseError> { - let key = format!("{}_{namespace}:{id}", self.meta_namespace); - let value = cmd("INCR") - .arg(key) - .query_async(&mut self.connection) - .await?; - Ok(value) + commands::incr(&mut self.inner, key).await } } -#[derive(Serialize, Deserialize)] -pub struct RedisValue { - key: K, - alias: Option, - iat: i64, - val: T, -} - -impl RedisValue { - pub fn value(&self) -> &T { - &self.val +impl ConnectionLike for RedisConnection { + fn req_packed_command<'a>( + &'a mut self, + command: &'a redis::Cmd, + ) -> redis::RedisFuture<'a, redis::Value> { + self.inner.req_packed_command(command) } -} -pub fn redis_args(cmd: &mut util::InstrumentedCmd, args: &[String]) { - for arg in args { - cmd.arg(arg); + fn req_packed_commands<'a>( + &'a mut self, + pipeline: &'a redis::Pipeline, + offset: usize, + count: usize, + ) -> redis::RedisFuture<'a, Vec> { + self.inner.req_packed_commands(pipeline, offset, count) } -} -pub async fn redis_execute( - cmd: &mut util::InstrumentedCmd, - redis: &mut deadpool_redis::Connection, -) -> Result -where - T: redis::FromRedisValue, -{ - let res = cmd.query_async::(redis).await?; - Ok(res) + fn get_db(&self) -> i64 { + self.inner.get_db() + } } diff --git a/apps/labrinth/src/database/redis/pubsub.rs b/apps/labrinth/src/database/redis/pubsub.rs new file mode 100644 index 0000000000..aed90d871c --- /dev/null +++ b/apps/labrinth/src/database/redis/pubsub.rs @@ -0,0 +1,129 @@ +use std::time::Duration; + +use futures::StreamExt; +use redis::ToRedisArgs; +use tokio::sync::mpsc; +use tracing::{info, warn}; + +use crate::database::models::DatabaseError; + +use super::RedisPool; + +const PUBSUB_BUFFER_SIZE: usize = 1024; +const INITIAL_RECONNECT_BACKOFF: Duration = Duration::from_millis(250); +const MAX_RECONNECT_BACKOFF: Duration = Duration::from_secs(30); + +enum SubscriptionOutcome { + SubscriberClosed, + Disconnected, +} + +impl RedisPool { + pub(crate) fn subscribe( + &self, + channel: &'static str, + ) -> mpsc::Receiver> { + let seed_urls = self.config.seed_urls().to_vec(); + let (sender, receiver) = mpsc::channel(PUBSUB_BUFFER_SIZE); + tokio::spawn(run_subscription(seed_urls, channel, sender)); + receiver + } + + pub async fn publish( + &self, + channel: &str, + message: M, + ) -> Result<(), DatabaseError> + where + M: ToRedisArgs + Send + Sync, + { + let mut connection = self.connect().await?; + let _: usize = redis::cmd("PUBLISH") + .arg(channel) + .arg(message) + .query_async(&mut connection) + .await?; + Ok(()) + } +} + +async fn run_subscription( + seed_urls: Vec, + channel: &'static str, + sender: mpsc::Sender>, +) { + let mut next_seed = 0; + let mut reconnect_backoff = INITIAL_RECONNECT_BACKOFF; + + loop { + let mut connected = false; + + for _ in 0..seed_urls.len() { + let seed_url = &seed_urls[next_seed]; + next_seed = (next_seed + 1) % seed_urls.len(); + + match forward_from_seed(seed_url, channel, &sender).await { + Ok(SubscriptionOutcome::SubscriberClosed) => return, + Ok(SubscriptionOutcome::Disconnected) => { + warn!(channel, "Redis Pub/Sub connection disconnected"); + connected = true; + break; + } + Err(error) => { + warn!( + ?error, + channel, + "Failed to establish Redis Pub/Sub subscription" + ); + } + } + } + + if sender.is_closed() { + return; + } + + let delay = if connected { + INITIAL_RECONNECT_BACKOFF + } else { + reconnect_backoff + }; + tokio::time::sleep(delay).await; + + reconnect_backoff = if connected { + INITIAL_RECONNECT_BACKOFF + } else { + reconnect_backoff + .saturating_mul(2) + .min(MAX_RECONNECT_BACKOFF) + }; + } +} + +async fn forward_from_seed( + seed_url: &str, + channel: &'static str, + sender: &mpsc::Sender>, +) -> redis::RedisResult { + let client = redis::Client::open(seed_url)?; + let mut pubsub = client.get_async_pubsub().await?; + pubsub.subscribe(channel).await?; + info!(channel, "Established Redis Pub/Sub subscription"); + + let mut stream = pubsub.into_on_message(); + while let Some(message) = stream.next().await { + if message.get_channel_name() != channel { + continue; + } + + if sender + .send(message.get_payload_bytes().to_vec()) + .await + .is_err() + { + return Ok(SubscriptionOutcome::SubscriberClosed); + } + } + + Ok(SubscriptionOutcome::Disconnected) +} diff --git a/apps/labrinth/src/database/redis/util.rs b/apps/labrinth/src/database/redis/util.rs index 8993708112..badf2f32e1 100644 --- a/apps/labrinth/src/database/redis/util.rs +++ b/apps/labrinth/src/database/redis/util.rs @@ -2,7 +2,6 @@ use std::fmt::Debug; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use derive_more::{Deref, DerefMut}; use redis::{FromRedisValue, RedisResult, ToRedisArgs}; use tokio::sync::Notify; use tokio::time::{Duration, timeout}; @@ -10,37 +9,6 @@ use tracing::{Instrument, info_span}; use crate::database::models::DatabaseError; -pub fn redis_pipe() -> InstrumentedPipeline { - InstrumentedPipeline { - inner: redis::pipe(), - } -} - -#[derive(Clone, Deref, DerefMut)] -pub struct InstrumentedPipeline { - #[deref] - #[deref_mut] - inner: redis::Pipeline, -} - -impl InstrumentedPipeline { - pub fn atomic(&mut self) -> &mut Self { - self.inner.atomic(); - self - } - - #[inline] - pub async fn query_async( - &self, - con: &mut impl redis::aio::ConnectionLike, - ) -> RedisResult { - self.inner - .query_async(con) - .instrument(info_span!("pipeline.query_async")) - .await - } -} - pub fn cmd(name: &str) -> InstrumentedCmd { InstrumentedCmd { inner: redis::cmd(name), diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index 2fc836e7ce..53e097c343 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -129,6 +129,8 @@ vars! { LABRINTH_EXTERNAL_NOTIFICATION_KEY: String = ""; RATE_LIMIT_IGNORE_KEY: String = ""; DATABASE_URL: String = "postgresql://labrinth:labrinth@localhost/labrinth"; + REDIS_MODE: crate::database::redis::RedisMode = crate::database::redis::RedisMode::Standalone; + REDIS_CONNECTION_TYPE: crate::database::redis::RedisConnectionType = crate::database::redis::RedisConnectionType::Pooled; REDIS_URL: String = "redis://localhost"; KAFKA_BOOTSTRAP_SERVERS: StringCsv = StringCsv(vec!["localhost:19092".into()]); KAFKA_CLIENT_ID: String = "labrinth"; @@ -289,6 +291,9 @@ vars! { REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64; REDIS_MAX_CONNECTIONS: u32 = 10000u32; REDIS_MIN_CONNECTIONS: usize = 0usize; + REDIS_CLUSTER_MAX_CONNECTIONS: u32 = 16u32; + REDIS_CLUSTER_MIN_CONNECTIONS: usize = 0usize; + REDIS_BLOCKING_MAX_CONNECTIONS: u32 = 8u32; REDIS_ENCODING_FORMAT: crate::database::redis::EncodingFormat = crate::database::redis::EncodingFormat::Json; REDIS_COMPRESSION_LEVEL: i32 = 0i32; REDIS_COMPRESSION_ALGORITHM: crate::database::redis::Codec = crate::database::redis::Codec::Lz4; diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index e05e536fd2..8ca9ba36d9 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -26,7 +26,7 @@ use crate::util::archon::ArchonClient; use crate::util::http::HttpClient; use crate::util::ratelimit::{AsyncRateLimiter, GCRAParameters}; use crate::util::tiltify::TiltifyClient; -use sync::friends::handle_pubsub; +use sync::friends::{FRIENDS_CHANNEL_NAME, handle_pubsub}; use url::Url; use webauthn_rs::{Webauthn, WebauthnBuilder}; @@ -290,11 +290,10 @@ pub fn app_setup( { let pool = pool.clone(); - let redis_client = redis::Client::open(redis_pool.url.clone()).unwrap(); + let pubsub_messages = redis_pool.subscribe(FRIENDS_CHANNEL_NAME); let sockets = active_sockets.clone(); actix_rt::spawn(async move { - let pubsub = redis_client.get_async_pubsub().await.unwrap(); - handle_pubsub(pubsub, pool, sockets).await; + handle_pubsub(pubsub_messages, pool, sockets).await; }); } diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index 9cb158bdab..9ce222be33 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -110,7 +110,7 @@ async fn app() -> std::io::Result<()> { .expect("Database connection failed"); // Redis connector - let redis_pool = RedisPool::new(""); + let redis_pool = RedisPool::new("").await; let storage_backend = ENV.STORAGE_BACKEND; let file_host: Arc = match storage_backend { diff --git a/apps/labrinth/src/models/exp/project.rs b/apps/labrinth/src/models/exp/project.rs index 68d4fbb613..a19e2ee91e 100644 --- a/apps/labrinth/src/models/exp/project.rs +++ b/apps/labrinth/src/models/exp/project.rs @@ -257,14 +257,16 @@ pub async fn fetch_query_context( { HashMap::new() } else { + let ping_keys = minecraft_java_server_pings + .iter() + .map(|project_id| { + redis + .keyspace() + .entity(server_ping::REDIS_NAMESPACE, project_id) + }) + .collect::>(); redis - .get_many_deserialized::( - server_ping::REDIS_NAMESPACE, - &minecraft_java_server_pings - .iter() - .map(ToString::to_string) - .collect::>(), - ) + .get_many_deserialized::(&ping_keys) .await? .into_iter() .enumerate() @@ -280,14 +282,16 @@ pub async fn fetch_query_context( let minecraft_server_analytics = if minecraft_server_analytics.is_empty() { HashMap::new() } else { + let analytics_keys = minecraft_server_analytics + .iter() + .map(|project_id| { + redis + .keyspace() + .entity(MINECRAFT_SERVER_ANALYTICS, project_id) + }) + .collect::>(); redis - .get_many_deserialized::( - MINECRAFT_SERVER_ANALYTICS, - &minecraft_server_analytics - .iter() - .map(ToString::to_string) - .collect::>(), - ) + .get_many_deserialized::(&analytics_keys) .await? .into_iter() .enumerate() diff --git a/apps/labrinth/src/queue/analytics/cache.rs b/apps/labrinth/src/queue/analytics/cache.rs index e4a5a00ad2..1a3bc1e20e 100644 --- a/apps/labrinth/src/queue/analytics/cache.rs +++ b/apps/labrinth/src/queue/analytics/cache.rs @@ -116,13 +116,11 @@ pub async fn cache_analytics( }; debug!("Caching analytics for {project_id}: {analytics:?}"); + let key = redis + .keyspace() + .entity(MINECRAFT_SERVER_ANALYTICS, project_id); redis - .set_serialized( - MINECRAFT_SERVER_ANALYTICS, - project_id.to_string(), - analytics, - None, - ) + .set_serialized(&key, analytics, None) .await .wrap_err_with(|| { eyre!("failed to set analytics for project '{project_id}'") diff --git a/apps/labrinth/src/queue/analytics/mod.rs b/apps/labrinth/src/queue/analytics/mod.rs index 090145765f..4249bcc9e9 100644 --- a/apps/labrinth/src/queue/analytics/mod.rs +++ b/apps/labrinth/src/queue/analytics/mod.rs @@ -133,29 +133,25 @@ impl AnalyticsQueue { raw_plays.insert(index, play); } - let mut redis = - redis.pool.get().await.map_err(DatabaseError::RedisPool)?; + let redis_keys = plays_keys + .iter() + .map(|key| { + let logical_key = format!("{}-{}", key.0, key.1); + redis.key().with_slot( + MINECRAFT_SERVER_PLAYS_NAMESPACE, + &logical_key, + &logical_key, + ) + }) + .collect::>(); + let mut redis_connection = redis.connect().await?; let results = cmd("MGET") - .arg( - plays_keys - .iter() - .map(|x| { - format!( - "{}:{}-{}", - MINECRAFT_SERVER_PLAYS_NAMESPACE, x.0, x.1 - ) - }) - .collect::>(), - ) - .query_async::>>(&mut redis) + .arg(&redis_keys) + .query_async::>>(&mut redis_connection) .await .map_err(DatabaseError::CacheError)?; - - let mut pipe = redis::pipe(); for (idx, count) in results.into_iter().enumerate() { - let key = &plays_keys[idx]; - let new_count = if let Some(count) = count { if count >= MINECRAFT_SERVER_PLAYS_LIMIT { raw_plays.remove(&idx); @@ -166,19 +162,14 @@ impl AnalyticsQueue { 1 }; - pipe.atomic().set_ex( - format!( - "{}:{}-{}", - MINECRAFT_SERVER_PLAYS_NAMESPACE, key.0, key.1 - ), - new_count, - MINECRAFT_SERVER_PLAYS_EXPIRY, - ); - } - if !pipe.is_empty() { - pipe.query_async::<()>(&mut *redis) - .await - .map_err(DatabaseError::CacheError)?; + let key = &redis_keys[idx]; + redis_connection + .set( + key, + new_count, + Some(MINECRAFT_SERVER_PLAYS_EXPIRY as i64), + ) + .await?; } let mut plays = client @@ -201,24 +192,25 @@ impl AnalyticsQueue { raw_views.push((views, true)); } - let mut redis = - redis.pool.get().await.map_err(DatabaseError::RedisPool)?; + let redis_keys = views_keys + .iter() + .map(|key| { + let logical_key = format!("{}-{}", key.0, key.1); + redis.key().with_slot( + VIEWS_NAMESPACE, + &logical_key, + &logical_key, + ) + }) + .collect::>(); + let mut redis_connection = redis.connect().await?; let results = cmd("MGET") - .arg( - views_keys - .iter() - .map(|x| format!("{}:{}-{}", VIEWS_NAMESPACE, x.0, x.1)) - .collect::>(), - ) - .query_async::>>(&mut redis) + .arg(&redis_keys) + .query_async::>>(&mut redis_connection) .await .map_err(DatabaseError::CacheError)?; - - let mut pipe = redis::pipe(); for (idx, count) in results.into_iter().enumerate() { - let key = &views_keys[idx]; - let new_count = if let Some((views, monetized)) = raw_views.get_mut(idx) { if let Some(count) = count { @@ -239,16 +231,10 @@ impl AnalyticsQueue { 1 }; - pipe.atomic().set_ex( - format!("{}:{}-{}", VIEWS_NAMESPACE, key.0, key.1), - new_count, - 6 * 60 * 60, - ); - } - if !pipe.is_empty() { - pipe.query_async::<()>(&mut *redis) - .await - .map_err(DatabaseError::CacheError)?; + let key = &redis_keys[idx]; + redis_connection + .set(key, new_count, Some(6 * 60 * 60)) + .await?; } let mut views = client.insert::("views").await?; @@ -278,26 +264,25 @@ impl AnalyticsQueue { raw_downloads.insert(index, download); } - let mut redis = - redis.pool.get().await.map_err(DatabaseError::RedisPool)?; + let redis_keys = downloads_keys + .iter() + .map(|key| { + let logical_key = format!("{}-{}", key.0, key.1); + redis.key().with_slot( + DOWNLOADS_NAMESPACE, + &logical_key, + &logical_key, + ) + }) + .collect::>(); + let mut redis_connection = redis.connect().await?; let results = cmd("MGET") - .arg( - downloads_keys - .iter() - .map(|x| { - format!("{}:{}-{}", DOWNLOADS_NAMESPACE, x.0, x.1) - }) - .collect::>(), - ) - .query_async::>>(&mut redis) + .arg(&redis_keys) + .query_async::>>(&mut redis_connection) .await .map_err(DatabaseError::CacheError)?; - - let mut pipe = redis::pipe(); for (idx, count) in results.into_iter().enumerate() { - let key = &downloads_keys[idx]; - let new_count = if let Some(count) = count { if count > 5 { raw_downloads.remove(&idx); @@ -309,16 +294,10 @@ impl AnalyticsQueue { 1 }; - pipe.atomic().set_ex( - format!("{}:{}-{}", DOWNLOADS_NAMESPACE, key.0, key.1), - new_count, - 6 * 60 * 60, - ); - } - if !pipe.is_empty() { - pipe.query_async::<()>(&mut *redis) - .await - .map_err(DatabaseError::CacheError)?; + let key = &redis_keys[idx]; + redis_connection + .set(key, new_count, Some(6 * 60 * 60)) + .await?; } let mut transaction = pool.begin().await?; diff --git a/apps/labrinth/src/queue/server_ping.rs b/apps/labrinth/src/queue/server_ping.rs index 46dbd67179..b6bc7602e7 100644 --- a/apps/labrinth/src/queue/server_ping.rs +++ b/apps/labrinth/src/queue/server_ping.rs @@ -111,6 +111,12 @@ impl ServerPingQueue { for (project_id, ping) in &pings { let data = ping.data.as_ref(); + let ping_key = + self.redis.key().entity(REDIS_NAMESPACE, project_id); + let failure_key = self + .redis + .key() + .entity(REDIS_FAILURE_NAMESPACE, project_id); let row = ServerPingRecord { recorded: ping.when.timestamp_nanos_opt().unwrap() @@ -134,13 +140,13 @@ impl ServerPingQueue { // ping succeeded; immediately update its online status in redis redis - .set_serialized(REDIS_NAMESPACE, project_id, ping, None) + .set_serialized(&ping_key, ping, None) .await .wrap_err("failed to set redis key")?; updated_project = true; redis - .delete(REDIS_FAILURE_NAMESPACE, project_id) + .delete(&failure_key) .await .wrap_err("failed to delete failure count")?; } else { @@ -148,7 +154,7 @@ impl ServerPingQueue { // otherwise, just add to the fail counter let failure_count = redis - .incr(REDIS_FAILURE_NAMESPACE, &project_id.to_string()) + .incr(&failure_key) .await .wrap_err("failed to increment failure count")?; @@ -156,12 +162,7 @@ impl ServerPingQueue { && count >= ENV.SERVER_PING_MAX_FAIL_COUNT { redis - .set_serialized( - REDIS_NAMESPACE, - project_id, - ping, - None, - ) + .set_serialized(&ping_key, ping, None) .await .wrap_err( "failed to set failed ping record in redis", @@ -246,14 +247,17 @@ impl ServerPingQueue { // and if we do miss an entry that we shouldn't, we just ping it again let all_project_ids = all_server_projects .iter() - .map(|row| ProjectId::from(DBProjectId(row.id)).to_string()) + .map(|row| ProjectId::from(DBProjectId(row.id))) + .collect::>(); + let ping_keys = all_project_ids + .iter() + .map(|project_id| { + self.redis.key().entity(REDIS_NAMESPACE, project_id) + }) .collect::>(); let all_server_last_pings = redis - .get_many_deserialized::( - REDIS_NAMESPACE, - &all_project_ids, - ) + .get_many_deserialized::(&ping_keys) .await .wrap_err("failed to fetch server project last pings")?; diff --git a/apps/labrinth/src/routes/internal/campaign.rs b/apps/labrinth/src/routes/internal/campaign.rs index 24d629874a..848b1900a4 100644 --- a/apps/labrinth/src/routes/internal/campaign.rs +++ b/apps/labrinth/src/routes/internal/campaign.rs @@ -143,7 +143,7 @@ impl CampaignDonation { } } -/// Receive a Tiltify webhook. +/// Receive a Tiltify webhook. #[utoipa::path( context_path = "/campaign", tag = "campaigns", @@ -307,7 +307,7 @@ fn verify_tiltify_webhook_signature( Ok(()) } -/// Get Pride campaign data. +/// Get Pride campaign data. #[utoipa::path( context_path = "/campaign", tag = "campaigns", @@ -324,12 +324,12 @@ pub async fn pride_26( .connect() .await .wrap_internal_err("connecting to redis")?; + let cache_key = redis + .key() + .entity(CAMPAIGN_INFO_CACHE_NAMESPACE, campaign_id); let cached = redis_connection - .get_deserialized::( - CAMPAIGN_INFO_CACHE_NAMESPACE, - campaign_id, - ) + .get_deserialized::(&cache_key) .await .wrap_internal_err("getting cached campaign info")?; @@ -383,8 +383,7 @@ pub async fn pride_26( redis_connection .set_serialized( - CAMPAIGN_INFO_CACHE_NAMESPACE, - campaign_id, + &cache_key, &campaign_info, Some(CAMPAIGN_INFO_CACHE_TTL_SECONDS), ) diff --git a/apps/labrinth/src/routes/internal/flows.rs b/apps/labrinth/src/routes/internal/flows.rs index c81c66a32c..5171742c75 100644 --- a/apps/labrinth/src/routes/internal/flows.rs +++ b/apps/labrinth/src/routes/internal/flows.rs @@ -1440,7 +1440,7 @@ struct NewOAuthAccount { pub sign_up_newsletter: bool, } -/// Create account with OAuth. +/// Create account with OAuth. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -1527,7 +1527,7 @@ struct DiscordCommunityHandoffPayload { nonce: String, } -/// Link Discord community. +/// Link Discord community. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -1604,7 +1604,7 @@ pub async fn discord_community_link( Ok(web::Json(DiscordCommunityLinkResponse { url })) } -/// Remove an auth provider. +/// Remove an auth provider. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -1947,7 +1947,7 @@ impl ReadyAccountRegisterFlow { } } -/// Validate password account creation. +/// Validate password account creation. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -1975,7 +1975,7 @@ pub async fn validate_create_account_with_password( Ok(()) } -/// Create account with a password. +/// Create account with a password. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2024,7 +2024,7 @@ pub struct Login { pub challenge: String, } -/// Log in with a password. +/// Log in with a password. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2128,13 +2128,13 @@ async fn validate_2fa_code( const TOTP_NAMESPACE: &str = "used_totp:v1"; let mut conn = redis.connect().await?; + let logical_key = format!("{}-{}", input, user_id.0); + let key = redis + .key() + .with_slot(TOTP_NAMESPACE, &logical_key, &logical_key); // Check if TOTP has already been used - if conn - .get(TOTP_NAMESPACE, &format!("{}-{}", input, user_id.0)) - .await? - .is_some() - { + if conn.get(&key).await?.is_some() { return Err(AuthenticationError::InvalidCredentials); } @@ -2142,13 +2142,7 @@ async fn validate_2fa_code( .check_current(input.as_str()) .map_err(|_| AuthenticationError::InvalidCredentials)? { - conn.set( - TOTP_NAMESPACE, - &format!("{}-{}", input, user_id.0), - "", - Some(60), - ) - .await?; + conn.set(&key, "", Some(60)).await?; Ok(true) } else if allow_backup { @@ -2185,7 +2179,7 @@ async fn validate_2fa_code( } } -/// Complete login with 2FA. +/// Complete login with 2FA. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2245,7 +2239,7 @@ pub async fn login_2fa( } } -/// Start 2FA setup. +/// Start 2FA setup. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2296,7 +2290,7 @@ pub async fn begin_2fa_flow( } } -/// Finish 2FA setup. +/// Finish 2FA setup. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2431,7 +2425,7 @@ pub struct Remove2FA { pub code: String, } -/// Remove 2FA. +/// Remove 2FA. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2531,7 +2525,7 @@ pub struct ResetPassword { pub challenge: String, } -/// Start password reset. +/// Start password reset. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2637,7 +2631,7 @@ pub struct ChangePassword { pub new_password: Option, } -/// Change password. +/// Change password. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2803,7 +2797,7 @@ pub struct SetEmail { pub email: String, } -/// Set email address. +/// Set email address. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -2925,7 +2919,7 @@ pub async fn set_email( Ok(HttpResponse::Ok().finish()) } -/// Resend verification email. +/// Resend verification email. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3000,7 +2994,7 @@ pub struct VerifyEmail { pub flow: String, } -/// Verify email address. +/// Verify email address. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3066,7 +3060,7 @@ pub async fn verify_email( } } -/// Subscribe to the newsletter. +/// Subscribe to the newsletter. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3115,7 +3109,7 @@ pub async fn subscribe_newsletter( Ok(HttpResponse::NoContent().finish()) } -/// Get newsletter subscription status. +/// Get newsletter subscription status. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3165,7 +3159,7 @@ pub struct RegisterPasskeyResponse { pub flow: String, } -/// Start passkey registration. +/// Start passkey registration. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3265,7 +3259,7 @@ pub struct PasskeyResponse { pub last_used: Option>, } -/// Finish passkey registration. +/// Finish passkey registration. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3374,7 +3368,7 @@ pub struct AuthenticatePasskeyResponse { pub flow: String, } -/// Start passkey authentication. +/// Start passkey authentication. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3417,7 +3411,7 @@ pub struct AuthenticatePasskeyFinish { pub credential: PublicKeyCredential, } -/// Finish passkey authentication. +/// Finish passkey authentication. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3534,7 +3528,7 @@ pub async fn authenticate_passkey_finish( } } -/// List passkeys. +/// List passkeys. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3584,7 +3578,7 @@ pub struct RenamePasskey { pub name: String, } -/// Rename a passkey. +/// Rename a passkey. #[utoipa::path( context_path = "/auth", tag = "auth", @@ -3639,7 +3633,7 @@ pub async fn rename_passkey( Ok(HttpResponse::NoContent().finish()) } -/// Delete a passkey. +/// Delete a passkey. #[utoipa::path( context_path = "/auth", tag = "auth", diff --git a/apps/labrinth/src/routes/internal/gotenberg.rs b/apps/labrinth/src/routes/internal/gotenberg.rs index 71d39dedb4..f422f72df7 100644 --- a/apps/labrinth/src/routes/internal/gotenberg.rs +++ b/apps/labrinth/src/routes/internal/gotenberg.rs @@ -13,7 +13,7 @@ use crate::routes::ApiError; use crate::util::error::Context; use crate::util::gotenberg::{ GeneratedPdfType, MODRINTH_GENERATED_PDF_TYPE, MODRINTH_PAYMENT_ID, - PAYMENT_STATEMENTS_NAMESPACE, + payment_statement_key, }; use crate::util::guards::internal_network_guard; @@ -63,6 +63,7 @@ pub async fn success_callback( .0 .0; + let response_key = payment_statement_key(redis.get_ref(), &payout_id); let mut redis = redis .connect() .await @@ -77,11 +78,7 @@ pub async fn success_callback( .wrap_internal_err("failed to serialize Redis document response")?; redis - .lpush( - PAYMENT_STATEMENTS_NAMESPACE, - &payout_id.to_string(), - &redis_msg, - ) + .lpush(&response_key, &redis_msg) .await .wrap_internal_err("failed to send document over Redis")?; @@ -134,6 +131,7 @@ pub async fn error_callback( .wrap_request_err("no payout ID for document")? .0 .0; + let response_key = payment_statement_key(redis.get_ref(), &payout_id); let mut redis = redis .connect() .await @@ -146,11 +144,7 @@ pub async fn error_callback( .wrap_internal_err("failed to serialize Redis error response")?; redis - .lpush( - PAYMENT_STATEMENTS_NAMESPACE, - &payout_id.to_string(), - &redis_msg, - ) + .lpush(&response_key, &redis_msg) .await .wrap_internal_err("failed to send error over Redis")?; diff --git a/apps/labrinth/src/routes/internal/statuses.rs b/apps/labrinth/src/routes/internal/statuses.rs index 6d3b652d80..6545251017 100644 --- a/apps/labrinth/src/routes/internal/statuses.rs +++ b/apps/labrinth/src/routes/internal/statuses.rs @@ -28,7 +28,6 @@ use chrono::Utc; use either::Either; use futures_util::future::select; use futures_util::{StreamExt, TryStreamExt}; -use redis::AsyncCommands; use serde::Deserialize; use std::pin::pin; use std::sync::atomic::Ordering; @@ -393,13 +392,7 @@ pub async fn broadcast_friends_message( redis: &RedisPool, message: RedisFriendsMessage, ) -> Result<(), crate::database::models::DatabaseError> { - let _: () = redis - .pool - .get() - .await? - .publish(FRIENDS_CHANNEL_NAME, message) - .await?; - Ok(()) + redis.publish(FRIENDS_CHANNEL_NAME, message).await } pub async fn broadcast_to_local_friends( diff --git a/apps/labrinth/src/routes/v3/content/mod.rs b/apps/labrinth/src/routes/v3/content/mod.rs index 297511fd80..0d234df906 100644 --- a/apps/labrinth/src/routes/v3/content/mod.rs +++ b/apps/labrinth/src/routes/v3/content/mod.rs @@ -295,11 +295,13 @@ async fn increment_content_resolve_cache_heat( return None; } }; + let key = redis.keyspace().with_slot( + CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE, + heat_key, + heat_key, + ); - let count = match redis - .incr(CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE, heat_key) - .await - { + let count = match redis.incr(&key).await { Ok(Some(count)) => count, Ok(None) => 1, Err(error) => { @@ -312,8 +314,7 @@ async fn increment_content_resolve_cache_heat( if let Err(error) = redis .set( - CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE, - heat_key, + &key, &count.to_string(), Some(CONTENT_RESOLVE_CACHE_HEAT_WINDOW_SECONDS), ) @@ -338,11 +339,13 @@ async fn get_cached_resolve_content_plan( return None; } }; + let key = redis.keyspace().with_slot( + CONTENT_RESOLVE_CACHE_NAMESPACE, + cache_key, + cache_key, + ); - match redis - .get_deserialized(CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key) - .await - { + match redis.get_deserialized(&key).await { Ok(cached) => cached, Err(error) => { tracing::warn!("failed to read content resolve cache: {error}"); @@ -366,14 +369,14 @@ async fn set_cached_resolve_content_plan( return; } }; + let key = redis.keyspace().with_slot( + CONTENT_RESOLVE_CACHE_NAMESPACE, + cache_key, + cache_key, + ); if let Err(error) = redis - .set_serialized( - CONTENT_RESOLVE_CACHE_NAMESPACE, - cache_key, - cached, - Some(expiry_seconds), - ) + .set_serialized(&key, cached, Some(expiry_seconds)) .await { tracing::warn!("failed to write content resolve cache: {error}"); diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index ac003ed4e5..f2025bf130 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -150,14 +150,16 @@ async fn hydrate_search_results( HashMap::new() } else { let mut redis = redis_pool.connect().await?; + let ping_keys = project_ids + .iter() + .map(|project_id| { + redis_pool + .key() + .entity(server_ping::REDIS_NAMESPACE, project_id) + }) + .collect::>(); let ping_results = redis - .get_many_deserialized::( - server_ping::REDIS_NAMESPACE, - &project_ids - .iter() - .map(ToString::to_string) - .collect::>(), - ) + .get_many_deserialized::(&ping_keys) .await?; ping_results diff --git a/apps/labrinth/src/sync/friends.rs b/apps/labrinth/src/sync/friends.rs index 35f7141e66..e8e5fb47d4 100644 --- a/apps/labrinth/src/sync/friends.rs +++ b/apps/labrinth/src/sync/friends.rs @@ -10,10 +10,9 @@ use actix_web::web::Data; use ariadne::ids::UserId; use ariadne::networking::message::ServerToClientMessage; use ariadne::users::UserStatus; -use redis::aio::PubSub; use redis::{RedisWrite, ToRedisArgs, ToSingleRedisArg}; use serde::{Deserialize, Serialize}; -use tokio_stream::StreamExt; +use tokio::sync::mpsc; pub const FRIENDS_CHANNEL_NAME: &str = "friends:v1"; @@ -47,17 +46,12 @@ impl ToRedisArgs for RedisFriendsMessage { impl ToSingleRedisArg for RedisFriendsMessage {} pub async fn handle_pubsub( - mut pubsub: PubSub, + mut messages: mpsc::Receiver>, pool: PgPool, sockets: Data, ) { - pubsub.subscribe(FRIENDS_CHANNEL_NAME).await.unwrap(); - let mut stream = pubsub.into_on_message(); - while let Some(message) = stream.next().await { - if message.get_channel_name() != FRIENDS_CHANNEL_NAME { - continue; - } - let payload = postcard::from_bytes(message.get_payload_bytes()); + while let Some(message) = messages.recv().await { + let payload = postcard::from_bytes::(&message); let pool = pool.clone(); let sockets = sockets.clone(); diff --git a/apps/labrinth/src/sync/status.rs b/apps/labrinth/src/sync/status.rs index 0d7892a57a..3e27094e0f 100644 --- a/apps/labrinth/src/sync/status.rs +++ b/apps/labrinth/src/sync/status.rs @@ -5,6 +5,7 @@ use ariadne::users::UserStatus; use redis::AsyncCommands; const EXPIRY_TIME_SECONDS: i64 = 60; +const USER_STATUS_NAMESPACE: &str = "user_status:v1"; pub async fn get_user_status( user: UserId, @@ -15,9 +16,9 @@ pub async fn get_user_status( return Some(friend_status); } - if let Ok(mut conn) = redis.pool.get().await - && let Ok(mut statuses) = - conn.sscan::<_, Vec>(get_field_name(user)).await + let key = get_key(redis, user); + if let Ok(mut conn) = redis.connect().await + && let Ok(mut statuses) = conn.sscan::<_, Vec>(&key).await && let Some(Ok(status)) = statuses.next_item().await { return postcard::from_bytes::(&status).ok(); @@ -35,18 +36,18 @@ pub async fn replace_user_status( return Ok(()); }; - if let Ok(mut conn) = redis.pool.get().await { - let field_name = get_field_name(user); + if let Ok(mut conn) = redis.connect().await { + let key = get_key(redis, user); let mut pipe = redis::pipe(); pipe.atomic(); if let Some(status) = old_status { - pipe.srem(&field_name, postcard::to_allocvec(status).unwrap()) + pipe.srem(&key, postcard::to_allocvec(status).unwrap()) .ignore(); } if let Some(status) = new_status { - pipe.sadd(&field_name, postcard::to_allocvec(status).unwrap()) + pipe.sadd(&key, postcard::to_allocvec(status).unwrap()) .ignore(); - pipe.expire(&field_name, EXPIRY_TIME_SECONDS).ignore(); + pipe.expire(&key, EXPIRY_TIME_SECONDS).ignore(); } return pipe.query_async(&mut conn).await; } @@ -58,12 +59,13 @@ pub async fn push_back_user_expiry( user: UserId, redis: &RedisPool, ) -> Result<(), redis::RedisError> { - if let Ok(mut conn) = redis.pool.get().await { - return conn.expire(get_field_name(user), EXPIRY_TIME_SECONDS).await; + if let Ok(mut conn) = redis.connect().await { + let key = get_key(redis, user); + return conn.expire(&key, EXPIRY_TIME_SECONDS).await; } Ok(()) } -fn get_field_name(user: UserId) -> String { - format!("user_status:v1:{user}") +fn get_key(redis: &RedisPool, user: UserId) -> String { + redis.key().entity(USER_STATUS_NAMESPACE, user) } diff --git a/apps/labrinth/src/test/database.rs b/apps/labrinth/src/test/database.rs index 47d54dc19e..a4e6ff11d6 100644 --- a/apps/labrinth/src/test/database.rs +++ b/apps/labrinth/src/test/database.rs @@ -90,7 +90,7 @@ impl TemporaryDatabase { println!("Migrations complete"); // Gets new Redis pool - let redis_pool = RedisPool::new(temp_database_name.clone()); + let redis_pool = RedisPool::new(temp_database_name.clone()).await; // Create search backend let search_backend = search::backend(Some(temp_database_name.clone())); @@ -192,7 +192,7 @@ impl TemporaryDatabase { pool: pool.clone(), ro_pool: ReadOnlyPgPool::from(pool.clone()), database_name: TEMPLATE_DATABASE_NAME.to_string(), - redis_pool: RedisPool::new(name.clone()), + redis_pool: RedisPool::new(name.clone()).await, search_backend: Arc::from(search::backend(Some( name.clone(), ))), diff --git a/apps/labrinth/src/util/gotenberg.rs b/apps/labrinth/src/util/gotenberg.rs index 75e74fe65b..834bc50278 100644 --- a/apps/labrinth/src/util/gotenberg.rs +++ b/apps/labrinth/src/util/gotenberg.rs @@ -15,6 +15,16 @@ pub const MODRINTH_GENERATED_PDF_TYPE: HeaderName = pub const MODRINTH_PAYMENT_ID: HeaderName = HeaderName::from_static("modrinth-payment-id"); pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements:v1"; +const REDIS_TIMEOUT_MARGIN_MS: u64 = 250; + +pub(crate) fn payment_statement_key( + redis: &RedisPool, + payment_id: &PayoutId, +) -> String { + redis + .key() + .with_slot(PAYMENT_STATEMENTS_NAMESPACE, payment_id, payment_id) +} #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PaymentStatement { @@ -180,23 +190,19 @@ impl GotenbergClient { &self, statement: &PaymentStatement, ) -> Result { - let mut redis = self - .redis - .connect() - .await - .wrap_internal_err("failed to get Redis connection")?; - self.generate_payment_statement(statement).await?; let timeout_ms = ENV.GOTENBERG_TIMEOUT; + let redis_timeout_ms = + timeout_ms.saturating_sub(REDIS_TIMEOUT_MARGIN_MS).max(1); + let client_timeout_ms = timeout_ms.max(redis_timeout_ms + 1); + let response_key = + payment_statement_key(&self.redis, &statement.payment_id); let [_key, document] = tokio::time::timeout( - Duration::from_millis(timeout_ms), - redis.brpop( - PAYMENT_STATEMENTS_NAMESPACE, - &statement.payment_id.to_string(), - None, - ), + Duration::from_millis(client_timeout_ms), + self.redis + .brpop(&response_key, Duration::from_millis(redis_timeout_ms)), ) .await .wrap_internal_err("Gotenberg document generation timed out")? diff --git a/apps/labrinth/src/util/mod.rs b/apps/labrinth/src/util/mod.rs index 87676d448f..cc8d1ea032 100644 --- a/apps/labrinth/src/util/mod.rs +++ b/apps/labrinth/src/util/mod.rs @@ -15,7 +15,6 @@ pub mod img; pub mod ip; pub mod kafka; pub mod ratelimit; -pub mod redis; pub mod routes; pub mod sentry; pub mod tags; diff --git a/apps/labrinth/src/util/ratelimit.rs b/apps/labrinth/src/util/ratelimit.rs index 067af8b498..4031ce0ad9 100644 --- a/apps/labrinth/src/util/ratelimit.rs +++ b/apps/labrinth/src/util/ratelimit.rs @@ -68,9 +68,10 @@ impl AsyncRateLimiter { // Get current time in nanoseconds since UNIX epoch let now = Utc::now().timestamp_nanos_opt().unwrap_or(0); + let key = conn.keyspace().with_slot(RATE_LIMIT_NAMESPACE, key, key); // Get the current TAT from Redis (if it exists) - let tat_str = conn.get(RATE_LIMIT_NAMESPACE, key).await.ok().flatten(); + let tat_str = conn.get(&key).await.ok().flatten(); // Parse the TAT or use current time if not found let current_tat = match tat_str { @@ -102,12 +103,7 @@ impl AsyncRateLimiter { let new_tat = std::cmp::max(current_tat + increment, now); let _ = conn - .set( - RATE_LIMIT_NAMESPACE, - key, - &new_tat.to_string(), - Some(RATE_LIMIT_EXPIRY), - ) + .set(&key, &new_tat.to_string(), Some(RATE_LIMIT_EXPIRY)) .await; let remaining_capacity = diff --git a/apps/labrinth/src/util/redis.rs b/apps/labrinth/src/util/redis.rs deleted file mode 100644 index b3f34ee2b2..0000000000 --- a/apps/labrinth/src/util/redis.rs +++ /dev/null @@ -1,18 +0,0 @@ -use redis::Cmd; - -pub fn redis_args(cmd: &mut Cmd, args: &[String]) { - for arg in args { - cmd.arg(arg); - } -} - -pub async fn redis_execute( - cmd: &mut Cmd, - redis: &mut deadpool_redis::Connection, -) -> Result -where - T: redis::FromRedisValue, -{ - let res = cmd.query_async::(redis).await?; - Ok(res) -} diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index fd5046cb99..6ac52cbf24 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -60,24 +60,28 @@ async fn test_get_project() { // Confirm that the request was cached let mut redis_pool = test_env.db.redis_pool.connect().await.unwrap(); + let slug_key = redis_pool + .keyspace() + .entity(PROJECTS_SLUGS_NAMESPACE, alpha_project_slug); assert_eq!( redis_pool - .get(PROJECTS_SLUGS_NAMESPACE, alpha_project_slug) + .get(&slug_key) .await .unwrap() .and_then(|x| x.parse::().ok()), Some(parse_base62(alpha_project_id).unwrap() as i64) ); + let project_key = redis_pool.keyspace().entity( + PROJECTS_NAMESPACE, + parse_base62(alpha_project_id).unwrap(), + ); let cached_project: RedisValue< ProjectQueryResult, DBProjectId, String, > = redis_pool - .get_deserialized( - PROJECTS_NAMESPACE, - &parse_base62(alpha_project_id).unwrap().to_string(), - ) + .get_deserialized(&project_key) .await .unwrap() .unwrap(); @@ -283,9 +287,14 @@ async fn test_add_remove_project() { // Confirm that the project is gone from the cache let mut redis_pool = test_env.db.redis_pool.connect().await.unwrap(); + let slug_key = redis_pool + .keyspace() + .entity(PROJECTS_SLUGS_NAMESPACE, "demo"); + let id_key = + redis_pool.keyspace().entity(PROJECTS_SLUGS_NAMESPACE, &id); assert_eq!( redis_pool - .get(PROJECTS_SLUGS_NAMESPACE, "demo") + .get(&slug_key) .await .unwrap() .and_then(|x| x.parse::().ok()), @@ -293,7 +302,7 @@ async fn test_add_remove_project() { ); assert_eq!( redis_pool - .get(PROJECTS_SLUGS_NAMESPACE, &id) + .get(&id_key) .await .unwrap() .and_then(|x| x.parse::().ok()), diff --git a/apps/labrinth/tests/v2/project.rs b/apps/labrinth/tests/v2/project.rs index b640b0bc3f..9a83ca76cd 100644 --- a/apps/labrinth/tests/v2/project.rs +++ b/apps/labrinth/tests/v2/project.rs @@ -268,9 +268,14 @@ async fn test_add_remove_project() { // Confirm that the project is gone from the cache let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap(); + let slug_key = redis_conn + .keyspace() + .entity(PROJECTS_SLUGS_NAMESPACE, "demo"); + let id_key = + redis_conn.keyspace().entity(PROJECTS_SLUGS_NAMESPACE, &id); assert_eq!( redis_conn - .get(PROJECTS_SLUGS_NAMESPACE, "demo") + .get(&slug_key) .await .unwrap() .map(|x| x.parse::().unwrap()), @@ -278,7 +283,7 @@ async fn test_add_remove_project() { ); assert_eq!( redis_conn - .get(PROJECTS_SLUGS_NAMESPACE, &id) + .get(&id_key) .await .unwrap() .map(|x| x.parse::().unwrap()), diff --git a/apps/labrinth/tests/version.rs b/apps/labrinth/tests/version.rs index 951b408cd6..ebbdf00ec9 100644 --- a/apps/labrinth/tests/version.rs +++ b/apps/labrinth/tests/version.rs @@ -51,15 +51,16 @@ async fn test_get_version() { assert_eq!(&version.id.to_string(), alpha_version_id); let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap(); + let version_key = redis_conn.keyspace().entity( + VERSIONS_NAMESPACE, + parse_base62(alpha_version_id).unwrap(), + ); let cached_version: RedisValue< VersionQueryResult, DBVersionId, String, > = redis_conn - .get_deserialized( - VERSIONS_NAMESPACE, - &parse_base62(alpha_version_id).unwrap().to_string(), - ) + .get_deserialized(&version_key) .await .unwrap() .unwrap(); diff --git a/docker-compose.yml b/docker-compose.yml index 38227c4337..def52d2862 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,6 +64,255 @@ services: interval: 3s timeout: 5s retries: 3 + redis-cluster-0: + profiles: + - clustered-redis + image: redis:alpine + container_name: labrinth-redis-cluster-0 + restart: on-failure + networks: + - redis-cluster-mesh + ports: + - '127.0.0.1:7000:7000' + command: + [ + 'redis-server', + '--port', + '7000', + '--cluster-enabled', + 'yes', + '--cluster-config-file', + 'nodes.conf', + '--cluster-node-timeout', + '5000', + '--appendonly', + 'no', + '--save', + '', + ] + tmpfs: + - /data + healthcheck: + test: ['CMD', 'redis-cli', '-p', '7000', 'PING'] + interval: 3s + timeout: 5s + retries: 10 + redis-cluster-1: + profiles: + - clustered-redis + image: redis:alpine + container_name: labrinth-redis-cluster-1 + restart: on-failure + networks: + - redis-cluster-mesh + ports: + - '127.0.0.1:7001:7001' + command: + [ + 'redis-server', + '--port', + '7001', + '--cluster-enabled', + 'yes', + '--cluster-config-file', + 'nodes.conf', + '--cluster-node-timeout', + '5000', + '--appendonly', + 'no', + '--save', + '', + ] + tmpfs: + - /data + healthcheck: + test: ['CMD', 'redis-cli', '-p', '7001', 'PING'] + interval: 3s + timeout: 5s + retries: 10 + redis-cluster-2: + profiles: + - clustered-redis + image: redis:alpine + container_name: labrinth-redis-cluster-2 + restart: on-failure + networks: + - redis-cluster-mesh + ports: + - '127.0.0.1:7002:7002' + command: + [ + 'redis-server', + '--port', + '7002', + '--cluster-enabled', + 'yes', + '--cluster-config-file', + 'nodes.conf', + '--cluster-node-timeout', + '5000', + '--appendonly', + 'no', + '--save', + '', + ] + tmpfs: + - /data + healthcheck: + test: ['CMD', 'redis-cli', '-p', '7002', 'PING'] + interval: 3s + timeout: 5s + retries: 10 + redis-cluster-3: + profiles: + - clustered-redis + image: redis:alpine + container_name: labrinth-redis-cluster-3 + restart: on-failure + networks: + - redis-cluster-mesh + ports: + - '127.0.0.1:7003:7003' + command: + [ + 'redis-server', + '--port', + '7003', + '--cluster-enabled', + 'yes', + '--cluster-config-file', + 'nodes.conf', + '--cluster-node-timeout', + '5000', + '--appendonly', + 'no', + '--save', + '', + ] + tmpfs: + - /data + healthcheck: + test: ['CMD', 'redis-cli', '-p', '7003', 'PING'] + interval: 3s + timeout: 5s + retries: 10 + redis-cluster-4: + profiles: + - clustered-redis + image: redis:alpine + container_name: labrinth-redis-cluster-4 + restart: on-failure + networks: + - redis-cluster-mesh + ports: + - '127.0.0.1:7004:7004' + command: + [ + 'redis-server', + '--port', + '7004', + '--cluster-enabled', + 'yes', + '--cluster-config-file', + 'nodes.conf', + '--cluster-node-timeout', + '5000', + '--appendonly', + 'no', + '--save', + '', + ] + tmpfs: + - /data + healthcheck: + test: ['CMD', 'redis-cli', '-p', '7004', 'PING'] + interval: 3s + timeout: 5s + retries: 10 + redis-cluster-5: + profiles: + - clustered-redis + image: redis:alpine + container_name: labrinth-redis-cluster-5 + restart: on-failure + networks: + - redis-cluster-mesh + ports: + - '127.0.0.1:7005:7005' + command: + [ + 'redis-server', + '--port', + '7005', + '--cluster-enabled', + 'yes', + '--cluster-config-file', + 'nodes.conf', + '--cluster-node-timeout', + '5000', + '--appendonly', + 'no', + '--save', + '', + ] + tmpfs: + - /data + healthcheck: + test: ['CMD', 'redis-cli', '-p', '7005', 'PING'] + interval: 3s + timeout: 5s + retries: 10 + redis-cluster-creator: + profiles: + - clustered-redis + image: redis:alpine + container_name: labrinth-redis-cluster-creator + networks: + - redis-cluster-mesh + depends_on: + redis-cluster-0: + condition: service_healthy + redis-cluster-1: + condition: service_healthy + redis-cluster-2: + condition: service_healthy + redis-cluster-3: + condition: service_healthy + redis-cluster-4: + condition: service_healthy + redis-cluster-5: + condition: service_healthy + entrypoint: ['sh', '-c'] + command: + - | + cluster_info="$$(redis-cli -h redis-cluster-0 -p 7000 cluster info 2>/dev/null || true)" + case "$$cluster_info" in + *"cluster_state:ok"*) exit 0 ;; + esac + exec redis-cli --cluster create \ + redis-cluster-0:7000 \ + redis-cluster-1:7001 \ + redis-cluster-2:7002 \ + redis-cluster-3:7003 \ + redis-cluster-4:7004 \ + redis-cluster-5:7005 \ + --cluster-replicas 1 \ + --cluster-yes + redis-insight: + profiles: + - clustered-redis + image: redis/redisinsight:latest + container_name: labrinth-redis-insight + restart: on-failure + networks: + - redis-cluster-mesh + depends_on: + redis-cluster-creator: + condition: service_completed_successfully + ports: + - '127.0.0.1:5540:5540' + volumes: + - redis-insight-data:/data clickhouse: image: clickhouse/clickhouse-server container_name: labrinth-clickhouse @@ -234,6 +483,8 @@ services: networks: meilisearch-mesh: driver: bridge + redis-cluster-mesh: + driver: bridge volumes: typesense-data: meilisearch-data: @@ -244,5 +495,6 @@ volumes: elasticsearch-certs: db-data: redis-data: + redis-insight-data: redpanda-data: labrinth-cdn-data: diff --git a/turbo.jsonc b/turbo.jsonc index f1ad4c6ec4..ba7d7b2fc4 100644 --- a/turbo.jsonc +++ b/turbo.jsonc @@ -83,6 +83,7 @@ "env": [ "SQLX_OFFLINE", "DATABASE_URL", + "REDIS_*", "CARGO_*", "RUST_*", "RUSTFLAGS", From 5187431798e62c95083e3fe026af6b31503b430d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sat, 18 Jul 2026 23:33:06 -0400 Subject: [PATCH 03/12] chore: cleanup --- .../database/models/analytics_event_item.rs | 4 ++-- .../src/database/models/categories.rs | 17 +++++++-------- .../src/database/models/collection_item.rs | 2 +- .../labrinth/src/database/models/flow_item.rs | 6 +++--- .../src/database/models/image_item.rs | 2 +- .../database/models/legacy_loader_fields.rs | 2 +- .../src/database/models/loader_fields.rs | 21 +++++++++---------- .../database/models/moderation_note_item.rs | 14 ++++++------- .../src/database/models/notification_item.rs | 13 ++++-------- .../models/notifications_template_item.rs | 21 +++++++------------ .../models/notifications_type_item.rs | 8 ++----- .../src/database/models/organization_item.rs | 5 ++--- apps/labrinth/src/database/models/pat_item.rs | 10 ++++----- .../src/database/models/product_item.rs | 4 ++-- .../src/database/models/project_item.rs | 16 +++++--------- .../src/database/models/session_item.rs | 15 +++++-------- .../labrinth/src/database/models/team_item.rs | 2 +- .../labrinth/src/database/models/user_item.rs | 11 +++++----- .../src/database/models/version_item.rs | 11 ++++------ apps/labrinth/src/database/redis/mod.rs | 2 +- apps/labrinth/src/models/exp/project.rs | 8 ++----- apps/labrinth/src/queue/analytics/cache.rs | 4 +--- apps/labrinth/src/routes/v3/content/mod.rs | 6 +++--- apps/labrinth/src/util/ratelimit.rs | 2 +- apps/labrinth/tests/project.rs | 12 +++++------ apps/labrinth/tests/v2/project.rs | 8 +++---- apps/labrinth/tests/version.rs | 2 +- 27 files changed, 91 insertions(+), 137 deletions(-) diff --git a/apps/labrinth/src/database/models/analytics_event_item.rs b/apps/labrinth/src/database/models/analytics_event_item.rs index 913872ff1f..ab8c4b9e2e 100644 --- a/apps/labrinth/src/database/models/analytics_event_item.rs +++ b/apps/labrinth/src/database/models/analytics_event_item.rs @@ -87,7 +87,7 @@ impl DBAnalyticsEvent { ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; let key = redis - .keyspace() + .key() .metadata(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY); if let Some(events) = redis.get_deserialized(&key).await? { @@ -123,7 +123,7 @@ impl DBAnalyticsEvent { pub async fn clear_cache(redis: &RedisPool) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; let key = redis - .keyspace() + .key() .metadata(ANALYTICS_EVENTS_NAMESPACE, ANALYTICS_EVENTS_ALL_KEY); redis.delete(&key).await?; Ok(()) diff --git a/apps/labrinth/src/database/models/categories.rs b/apps/labrinth/src/database/models/categories.rs index 60e6e5f988..9e3012e99a 100644 --- a/apps/labrinth/src/database/models/categories.rs +++ b/apps/labrinth/src/database/models/categories.rs @@ -95,7 +95,7 @@ impl Category { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(TAGS_NAMESPACE, "category"); + let key = redis.key().metadata(TAGS_NAMESPACE, "category"); let res: Option> = redis.get_deserialized(&key).await?; @@ -125,7 +125,7 @@ impl Category { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(TAGS_NAMESPACE, "category"); + let key = redis.key().metadata(TAGS_NAMESPACE, "category"); redis.set_serialized(&key, &result, None).await?; @@ -163,8 +163,7 @@ impl LinkPlatform { { { let mut redis = redis.connect().await?; - let key = - redis.keyspace().metadata(TAGS_NAMESPACE, "link_platform"); + let key = redis.key().metadata(TAGS_NAMESPACE, "link_platform"); let res: Option> = redis.get_deserialized(&key).await?; @@ -189,7 +188,7 @@ impl LinkPlatform { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(TAGS_NAMESPACE, "link_platform"); + let key = redis.key().metadata(TAGS_NAMESPACE, "link_platform"); redis.set_serialized(&key, &result, None).await?; @@ -227,7 +226,7 @@ impl ReportType { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(TAGS_NAMESPACE, "report_type"); + let key = redis.key().metadata(TAGS_NAMESPACE, "report_type"); let res: Option> = redis.get_deserialized(&key).await?; @@ -247,7 +246,7 @@ impl ReportType { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(TAGS_NAMESPACE, "report_type"); + let key = redis.key().metadata(TAGS_NAMESPACE, "report_type"); redis.set_serialized(&key, &result, None).await?; @@ -285,7 +284,7 @@ impl ProjectType { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(TAGS_NAMESPACE, "project_type"); + let key = redis.key().metadata(TAGS_NAMESPACE, "project_type"); let res: Option> = redis.get_deserialized(&key).await?; @@ -305,7 +304,7 @@ impl ProjectType { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(TAGS_NAMESPACE, "project_type"); + let key = redis.key().metadata(TAGS_NAMESPACE, "project_type"); redis.set_serialized(&key, &result, None).await?; diff --git a/apps/labrinth/src/database/models/collection_item.rs b/apps/labrinth/src/database/models/collection_item.rs index 0fad263ca9..ae325b0a8a 100644 --- a/apps/labrinth/src/database/models/collection_item.rs +++ b/apps/labrinth/src/database/models/collection_item.rs @@ -217,7 +217,7 @@ impl DBCollection { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(COLLECTIONS_NAMESPACE, id.0); + let key = redis.key().entity(COLLECTIONS_NAMESPACE, id.0); redis.delete(&key).await?; Ok(()) diff --git a/apps/labrinth/src/database/models/flow_item.rs b/apps/labrinth/src/database/models/flow_item.rs index c44855d486..17d7b894e4 100644 --- a/apps/labrinth/src/database/models/flow_item.rs +++ b/apps/labrinth/src/database/models/flow_item.rs @@ -75,7 +75,7 @@ impl DBFlow { state: &str, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(FLOWS_NAMESPACE, state); + let key = redis.key().entity(FLOWS_NAMESPACE, state); redis .set_serialized(&key, &self, Some(expires.num_seconds())) @@ -103,7 +103,7 @@ impl DBFlow { redis: &RedisPool, ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(FLOWS_NAMESPACE, id); + let key = redis.key().entity(FLOWS_NAMESPACE, id); redis.get_deserialized(&key).await } @@ -129,7 +129,7 @@ impl DBFlow { redis: &RedisPool, ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(FLOWS_NAMESPACE, id); + let key = redis.key().entity(FLOWS_NAMESPACE, id); redis.delete(&key).await?; Ok(Some(())) diff --git a/apps/labrinth/src/database/models/image_item.rs b/apps/labrinth/src/database/models/image_item.rs index a44f275912..bca6399f3b 100644 --- a/apps/labrinth/src/database/models/image_item.rs +++ b/apps/labrinth/src/database/models/image_item.rs @@ -229,7 +229,7 @@ impl DBImage { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(IMAGES_NAMESPACE, id.0); + let key = redis.key().entity(IMAGES_NAMESPACE, id.0); redis.delete(&key).await?; Ok(()) diff --git a/apps/labrinth/src/database/models/legacy_loader_fields.rs b/apps/labrinth/src/database/models/legacy_loader_fields.rs index a0ec65a20c..89ccfd71b0 100644 --- a/apps/labrinth/src/database/models/legacy_loader_fields.rs +++ b/apps/labrinth/src/database/models/legacy_loader_fields.rs @@ -220,7 +220,7 @@ impl<'a> MinecraftGameVersionBuilder<'a> { .await?; let mut conn = redis.connect().await?; - let key = conn.keyspace().entity( + let key = conn.key().entity( crate::database::models::loader_fields::LOADER_FIELD_ENUM_VALUES_NAMESPACE, game_versions_enum.id.0, ); diff --git a/apps/labrinth/src/database/models/loader_fields.rs b/apps/labrinth/src/database/models/loader_fields.rs index 09b82eaa9b..1f011d8872 100644 --- a/apps/labrinth/src/database/models/loader_fields.rs +++ b/apps/labrinth/src/database/models/loader_fields.rs @@ -54,7 +54,7 @@ impl Game { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(GAMES_LIST_NAMESPACE, "games"); + let key = redis.key().metadata(GAMES_LIST_NAMESPACE, "games"); let cached_games: Option> = redis.get_deserialized(&key).await?; if let Some(cached_games) = cached_games { @@ -79,7 +79,7 @@ impl Game { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(GAMES_LIST_NAMESPACE, "games"); + let key = redis.key().metadata(GAMES_LIST_NAMESPACE, "games"); redis.set_serialized(&key, &result, None).await?; @@ -108,7 +108,7 @@ impl Loader { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(LOADER_ID, name); + let key = redis.key().metadata(LOADER_ID, name); let cached_id: Option = redis.get_deserialized(&key).await?; if let Some(cached_id) = cached_id { return Ok(Some(LoaderId(cached_id))); @@ -128,7 +128,7 @@ impl Loader { if let Some(result) = result { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(LOADER_ID, name); + let key = redis.key().metadata(LOADER_ID, name); redis.set_serialized(&key, &result.0, None).await?; } @@ -144,7 +144,7 @@ impl Loader { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(LOADERS_LIST_NAMESPACE, "all"); + let key = redis.key().metadata(LOADERS_LIST_NAMESPACE, "all"); let cached_loaders: Option> = redis.get_deserialized(&key).await?; if let Some(cached_loaders) = cached_loaders { @@ -185,7 +185,7 @@ impl Loader { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(LOADERS_LIST_NAMESPACE, "all"); + let key = redis.key().metadata(LOADERS_LIST_NAMESPACE, "all"); redis.set_serialized(&key, &result, None).await?; @@ -457,8 +457,7 @@ impl LoaderField { { { let mut redis = redis.connect().await?; - let key = - redis.keyspace().metadata(LOADER_FIELDS_NAMESPACE_ALL, ""); + let key = redis.key().metadata(LOADER_FIELDS_NAMESPACE_ALL, ""); let cached_fields: Option> = redis.get_deserialized(&key).await?; @@ -492,7 +491,7 @@ impl LoaderField { .collect(); let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(LOADER_FIELDS_NAMESPACE_ALL, ""); + let key = redis.key().metadata(LOADER_FIELDS_NAMESPACE_ALL, ""); redis.set_serialized(&key, &result, None).await?; @@ -511,7 +510,7 @@ impl LoaderFieldEnum { { let mut redis = redis.connect().await?; let key = redis - .keyspace() + .key() .metadata(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name); let cached_enum = redis.get_deserialized(&key).await?; @@ -540,7 +539,7 @@ impl LoaderFieldEnum { let mut redis = redis.connect().await?; let key = redis - .keyspace() + .key() .metadata(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name); redis.set_serialized(&key, &result, None).await?; diff --git a/apps/labrinth/src/database/models/moderation_note_item.rs b/apps/labrinth/src/database/models/moderation_note_item.rs index 678a5de616..656d8cffc8 100644 --- a/apps/labrinth/src/database/models/moderation_note_item.rs +++ b/apps/labrinth/src/database/models/moderation_note_item.rs @@ -37,9 +37,7 @@ impl DBModerationNote { let keys = user_ids .iter() .map(|id| { - redis - .keyspace() - .entity(MODERATION_NOTES_USERS_NAMESPACE, id.0) + redis.key().entity(MODERATION_NOTES_USERS_NAMESPACE, id.0) }) .collect::>(); redis.get_many_deserialized::(&keys).await? @@ -85,7 +83,7 @@ impl DBModerationNote { if let Some(user_id) = note.user_id { let key = redis - .keyspace() + .key() .entity(MODERATION_NOTES_USERS_NAMESPACE, user_id.0); redis.set_serialized(&key, ¬e, None).await?; notes.insert(user_id, note); @@ -122,7 +120,7 @@ impl DBModerationNote { .iter() .map(|id| { redis - .keyspace() + .key() .entity(MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, id.0) }) .collect::>(); @@ -168,7 +166,7 @@ impl DBModerationNote { }; if let Some(organization_id) = note.organization_id { - let key = redis.keyspace().entity( + let key = redis.key().entity( MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, organization_id.0, ); @@ -279,7 +277,7 @@ impl DBModerationNote { ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; let key = redis - .keyspace() + .key() .entity(MODERATION_NOTES_USERS_NAMESPACE, user_id.0); redis.delete(&key).await } @@ -289,7 +287,7 @@ impl DBModerationNote { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity( + let key = redis.key().entity( MODERATION_NOTES_ORGANIZATIONS_NAMESPACE, organization_id.0, ); diff --git a/apps/labrinth/src/database/models/notification_item.rs b/apps/labrinth/src/database/models/notification_item.rs index b567c4cad7..7a6e921ebc 100644 --- a/apps/labrinth/src/database/models/notification_item.rs +++ b/apps/labrinth/src/database/models/notification_item.rs @@ -433,9 +433,8 @@ impl DBNotification { { { let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0); + let key = + redis.key().entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0); let cached_notifications: Option> = redis.get_deserialized(&key).await?; @@ -490,9 +489,7 @@ impl DBNotification { .await?; let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0); + let key = redis.key().entity(USER_NOTIFICATIONS_NAMESPACE, user_id.0); redis.set_serialized(&key, &db_notifications, None).await?; @@ -635,9 +632,7 @@ impl DBNotification { let mut redis = redis.connect().await?; let keys = user_ids .into_iter() - .map(|id| { - redis.keyspace().entity(USER_NOTIFICATIONS_NAMESPACE, id.0) - }) + .map(|id| redis.key().entity(USER_NOTIFICATIONS_NAMESPACE, id.0)) .collect::>(); redis.delete_many(&keys).await?; diff --git a/apps/labrinth/src/database/models/notifications_template_item.rs b/apps/labrinth/src/database/models/notifications_template_item.rs index 2fef6ec4f8..2c8f160cf0 100644 --- a/apps/labrinth/src/database/models/notifications_template_item.rs +++ b/apps/labrinth/src/database/models/notifications_template_item.rs @@ -55,9 +55,8 @@ impl NotificationTemplate { ) -> Result, DatabaseError> { { let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .metadata(TEMPLATES_NAMESPACE, channel.as_str()); + let key = + redis.key().metadata(TEMPLATES_NAMESPACE, channel.as_str()); let maybe_cached_templates = redis.get_deserialized(&key).await?; @@ -79,9 +78,7 @@ impl NotificationTemplate { let templates = results.into_iter().map(Into::into).collect(); let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .metadata(TEMPLATES_NAMESPACE, channel.as_str()); + let key = redis.key().metadata(TEMPLATES_NAMESPACE, channel.as_str()); redis .set_serialized(&key, &templates, Some(TEMPLATES_CACHE_EXPIRY)) @@ -95,9 +92,7 @@ impl NotificationTemplate { redis: &RedisPool, ) -> Result, DatabaseError> { let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id); + let key = redis.key().metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id); redis.get_deserialized(&key).await } @@ -107,9 +102,7 @@ impl NotificationTemplate { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id); + let key = redis.key().metadata(TEMPLATES_HTML_DATA_NAMESPACE, self.id); redis .set_serialized(&key, &data, Some(HTML_DATA_CACHE_EXPIRY)) .await @@ -131,7 +124,7 @@ where let mut redis_conn = redis.connect().await?; let redis_key = redis_conn - .keyspace() + .key() .metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key); if let Some(body) = redis_conn.get_deserialized::(&redis_key).await? @@ -144,7 +137,7 @@ where let cached = HtmlBody { html: get().await? }; let mut redis_conn = redis.connect().await?; let redis_key = redis_conn - .keyspace() + .key() .metadata(TEMPLATES_DYNAMIC_HTML_NAMESPACE, key); redis_conn diff --git a/apps/labrinth/src/database/models/notifications_type_item.rs b/apps/labrinth/src/database/models/notifications_type_item.rs index b4ba2cd2cc..77e40db0a7 100644 --- a/apps/labrinth/src/database/models/notifications_type_item.rs +++ b/apps/labrinth/src/database/models/notifications_type_item.rs @@ -41,9 +41,7 @@ impl NotificationTypeItem { { { let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .metadata(NOTIFICATION_TYPES_NAMESPACE, "all"); + let key = redis.key().metadata(NOTIFICATION_TYPES_NAMESPACE, "all"); let cached_types = redis.get_deserialized(&key).await?; @@ -62,9 +60,7 @@ impl NotificationTypeItem { let types = results.into_iter().map(Into::into).collect(); let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .metadata(NOTIFICATION_TYPES_NAMESPACE, "all"); + let key = redis.key().metadata(NOTIFICATION_TYPES_NAMESPACE, "all"); redis.set_serialized(&key, &types, None).await?; diff --git a/apps/labrinth/src/database/models/organization_item.rs b/apps/labrinth/src/database/models/organization_item.rs index 0538b2bc13..15a00998d0 100644 --- a/apps/labrinth/src/database/models/organization_item.rs +++ b/apps/labrinth/src/database/models/organization_item.rs @@ -256,11 +256,10 @@ impl DBOrganization { redis: &RedisPool, ) -> Result<(), super::DatabaseError> { let mut redis = redis.connect().await?; - let mut keys = - vec![redis.keyspace().entity(ORGANIZATIONS_NAMESPACE, id.0)]; + let mut keys = vec![redis.key().entity(ORGANIZATIONS_NAMESPACE, id.0)]; if let Some(slug) = slug { keys.push( - redis.keyspace().entity( + redis.key().entity( ORGANIZATIONS_TITLES_NAMESPACE, slug.to_lowercase(), ), diff --git a/apps/labrinth/src/database/models/pat_item.rs b/apps/labrinth/src/database/models/pat_item.rs index 7373bd67fa..417fa82ff1 100644 --- a/apps/labrinth/src/database/models/pat_item.rs +++ b/apps/labrinth/src/database/models/pat_item.rs @@ -159,7 +159,7 @@ impl DBPersonalAccessToken { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(PATS_USERS_NAMESPACE, user_id.0); + let key = redis.key().entity(PATS_USERS_NAMESPACE, user_id.0); let res = redis.get_deserialized::>(&key).await?; @@ -183,7 +183,7 @@ impl DBPersonalAccessToken { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(PATS_USERS_NAMESPACE, user_id.0); + let key = redis.key().entity(PATS_USERS_NAMESPACE, user_id.0); redis.set_serialized(&key, &db_pats, None).await?; Ok(db_pats) @@ -203,12 +203,12 @@ impl DBPersonalAccessToken { .into_iter() .flat_map(|(id, token, user_id)| { [ - id.map(|id| redis.keyspace().entity(PATS_NAMESPACE, id.0)), + id.map(|id| redis.key().entity(PATS_NAMESPACE, id.0)), token.map(|token| { - redis.keyspace().entity(PATS_TOKENS_NAMESPACE, token) + redis.key().entity(PATS_TOKENS_NAMESPACE, token) }), user_id.map(|user_id| { - redis.keyspace().entity(PATS_USERS_NAMESPACE, user_id.0) + redis.key().entity(PATS_USERS_NAMESPACE, user_id.0) }), ] .into_iter() diff --git a/apps/labrinth/src/database/models/product_item.rs b/apps/labrinth/src/database/models/product_item.rs index 5f7b220e1f..a8bafc41ff 100644 --- a/apps/labrinth/src/database/models/product_item.rs +++ b/apps/labrinth/src/database/models/product_item.rs @@ -152,7 +152,7 @@ impl QueryProductWithPrices { { { let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(PRODUCTS_NAMESPACE, "all"); + let key = redis.key().metadata(PRODUCTS_NAMESPACE, "all"); let res: Option> = redis.get_deserialized(&key).await?; @@ -194,7 +194,7 @@ impl QueryProductWithPrices { .collect::>(); let mut redis = redis.connect().await?; - let key = redis.keyspace().metadata(PRODUCTS_NAMESPACE, "all"); + let key = redis.key().metadata(PRODUCTS_NAMESPACE, "all"); redis.set_serialized(&key, &products, None).await?; diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs index 2f8790fdf2..9c4c3f1b14 100644 --- a/apps/labrinth/src/database/models/project_item.rs +++ b/apps/labrinth/src/database/models/project_item.rs @@ -974,9 +974,7 @@ impl DBProject { { let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0); + let key = redis.key().entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0); let dependencies = redis.get_deserialized::(&key).await?; @@ -1011,9 +1009,7 @@ impl DBProject { .await?; let mut redis = redis.connect().await?; - let key = redis - .keyspace() - .entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0); + let key = redis.key().entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0); redis.set_serialized(&key, &dependencies, None).await?; Ok(dependencies) @@ -1026,19 +1022,17 @@ impl DBProject { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - let mut keys = vec![redis.keyspace().entity(PROJECTS_NAMESPACE, id.0)]; + let mut keys = vec![redis.key().entity(PROJECTS_NAMESPACE, id.0)]; if let Some(slug) = slug { keys.push( redis - .keyspace() + .key() .entity(PROJECTS_SLUGS_NAMESPACE, slug.to_lowercase()), ); } if clear_dependencies.unwrap_or(false) { keys.push( - redis - .keyspace() - .entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0), + redis.key().entity(PROJECTS_DEPENDENCIES_NAMESPACE, id.0), ); } diff --git a/apps/labrinth/src/database/models/session_item.rs b/apps/labrinth/src/database/models/session_item.rs index daf9493fe1..5769ff4986 100644 --- a/apps/labrinth/src/database/models/session_item.rs +++ b/apps/labrinth/src/database/models/session_item.rs @@ -224,8 +224,7 @@ impl DBSession { { { let mut redis = redis.connect().await?; - let key = - redis.keyspace().entity(SESSIONS_USERS_NAMESPACE, user_id.0); + let key = redis.key().entity(SESSIONS_USERS_NAMESPACE, user_id.0); let res = redis.get_deserialized::>(&key).await?; @@ -250,7 +249,7 @@ impl DBSession { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(SESSIONS_USERS_NAMESPACE, user_id.0); + let key = redis.key().entity(SESSIONS_USERS_NAMESPACE, user_id.0); redis.set_serialized(&key, &db_sessions, None).await?; @@ -275,16 +274,12 @@ impl DBSession { .into_iter() .flat_map(|(id, session, user_id)| { [ - id.map(|id| { - redis.keyspace().entity(SESSIONS_NAMESPACE, id.0) - }), + id.map(|id| redis.key().entity(SESSIONS_NAMESPACE, id.0)), session.map(|session| { - redis.keyspace().entity(SESSIONS_IDS_NAMESPACE, session) + redis.key().entity(SESSIONS_IDS_NAMESPACE, session) }), user_id.map(|user_id| { - redis - .keyspace() - .entity(SESSIONS_USERS_NAMESPACE, user_id.0) + redis.key().entity(SESSIONS_USERS_NAMESPACE, user_id.0) }), ] .into_iter() diff --git a/apps/labrinth/src/database/models/team_item.rs b/apps/labrinth/src/database/models/team_item.rs index 9748f3db62..4220456247 100644 --- a/apps/labrinth/src/database/models/team_item.rs +++ b/apps/labrinth/src/database/models/team_item.rs @@ -265,7 +265,7 @@ impl DBTeamMember { redis: &RedisPool, ) -> Result<(), super::DatabaseError> { let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(TEAMS_NAMESPACE, id.0); + let key = redis.key().entity(TEAMS_NAMESPACE, id.0); redis.delete(&key).await?; Ok(()) } diff --git a/apps/labrinth/src/database/models/user_item.rs b/apps/labrinth/src/database/models/user_item.rs index b3e4a585f8..6daef10670 100644 --- a/apps/labrinth/src/database/models/user_item.rs +++ b/apps/labrinth/src/database/models/user_item.rs @@ -389,8 +389,7 @@ impl DBUser { { let mut redis = redis.connect().await?; - let key = - redis.keyspace().entity(USERS_PROJECTS_NAMESPACE, user_id.0); + let key = redis.key().entity(USERS_PROJECTS_NAMESPACE, user_id.0); let cached_projects = redis.get_deserialized::>(&key).await?; @@ -415,7 +414,7 @@ impl DBUser { .await?; let mut redis = redis.connect().await?; - let key = redis.keyspace().entity(USERS_PROJECTS_NAMESPACE, user_id.0); + let key = redis.key().entity(USERS_PROJECTS_NAMESPACE, user_id.0); redis.set_serialized(&key, &db_projects, None).await?; @@ -552,9 +551,9 @@ impl DBUser { .iter() .flat_map(|(id, username)| { [ - Some(redis.keyspace().entity(USERS_NAMESPACE, id.0)), + Some(redis.key().entity(USERS_NAMESPACE, id.0)), username.as_ref().map(|username| { - redis.keyspace().entity( + redis.key().entity( USER_USERNAMES_NAMESPACE, username.to_lowercase(), ) @@ -576,7 +575,7 @@ impl DBUser { let mut redis = redis.connect().await?; let keys = user_ids .iter() - .map(|id| redis.keyspace().entity(USERS_PROJECTS_NAMESPACE, id.0)) + .map(|id| redis.key().entity(USERS_PROJECTS_NAMESPACE, id.0)) .collect::>(); redis.delete_many(&keys).await?; diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs index 5317bf4b81..77917211b9 100644 --- a/apps/labrinth/src/database/models/version_item.rs +++ b/apps/labrinth/src/database/models/version_item.rs @@ -1050,14 +1050,11 @@ impl DBVersion { redis: &RedisPool, ) -> Result<(), DatabaseError> { let mut redis = redis.connect().await?; - let mut keys = vec![ - redis - .keyspace() - .entity(VERSIONS_NAMESPACE, version.inner.id.0), - ]; + let mut keys = + vec![redis.key().entity(VERSIONS_NAMESPACE, version.inner.id.0)]; keys.extend(version.files.iter().flat_map(|file| { file.hashes.iter().map(|(algorithm, hash)| { - redis.keyspace().entity( + redis.key().entity( VERSION_FILES_NAMESPACE, format!("{algorithm}_{hash}"), ) @@ -1075,7 +1072,7 @@ impl DBVersion { let mut redis = redis.connect().await?; let keys = version_ids .iter() - .map(|id| redis.keyspace().entity(VERSIONS_NAMESPACE, id.0)) + .map(|id| redis.key().entity(VERSIONS_NAMESPACE, id.0)) .collect::>(); redis.delete_many(&keys).await?; diff --git a/apps/labrinth/src/database/redis/mod.rs b/apps/labrinth/src/database/redis/mod.rs index 3d2c2f36e8..0bef753ae9 100644 --- a/apps/labrinth/src/database/redis/mod.rs +++ b/apps/labrinth/src/database/redis/mod.rs @@ -227,7 +227,7 @@ impl ConnectionProvider for RedisPool { } impl RedisConnection { - pub fn keyspace(&self) -> &KeyBuilder { + pub fn key(&self) -> &KeyBuilder { &self.key_builder } diff --git a/apps/labrinth/src/models/exp/project.rs b/apps/labrinth/src/models/exp/project.rs index a19e2ee91e..bdc1f55f17 100644 --- a/apps/labrinth/src/models/exp/project.rs +++ b/apps/labrinth/src/models/exp/project.rs @@ -260,9 +260,7 @@ pub async fn fetch_query_context( let ping_keys = minecraft_java_server_pings .iter() .map(|project_id| { - redis - .keyspace() - .entity(server_ping::REDIS_NAMESPACE, project_id) + redis.key().entity(server_ping::REDIS_NAMESPACE, project_id) }) .collect::>(); redis @@ -285,9 +283,7 @@ pub async fn fetch_query_context( let analytics_keys = minecraft_server_analytics .iter() .map(|project_id| { - redis - .keyspace() - .entity(MINECRAFT_SERVER_ANALYTICS, project_id) + redis.key().entity(MINECRAFT_SERVER_ANALYTICS, project_id) }) .collect::>(); redis diff --git a/apps/labrinth/src/queue/analytics/cache.rs b/apps/labrinth/src/queue/analytics/cache.rs index 1a3bc1e20e..bd0bc4fd7c 100644 --- a/apps/labrinth/src/queue/analytics/cache.rs +++ b/apps/labrinth/src/queue/analytics/cache.rs @@ -116,9 +116,7 @@ pub async fn cache_analytics( }; debug!("Caching analytics for {project_id}: {analytics:?}"); - let key = redis - .keyspace() - .entity(MINECRAFT_SERVER_ANALYTICS, project_id); + let key = redis.key().entity(MINECRAFT_SERVER_ANALYTICS, project_id); redis .set_serialized(&key, analytics, None) .await diff --git a/apps/labrinth/src/routes/v3/content/mod.rs b/apps/labrinth/src/routes/v3/content/mod.rs index 0d234df906..7b7807308f 100644 --- a/apps/labrinth/src/routes/v3/content/mod.rs +++ b/apps/labrinth/src/routes/v3/content/mod.rs @@ -295,7 +295,7 @@ async fn increment_content_resolve_cache_heat( return None; } }; - let key = redis.keyspace().with_slot( + let key = redis.key().with_slot( CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE, heat_key, heat_key, @@ -339,7 +339,7 @@ async fn get_cached_resolve_content_plan( return None; } }; - let key = redis.keyspace().with_slot( + let key = redis.key().with_slot( CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key, cache_key, @@ -369,7 +369,7 @@ async fn set_cached_resolve_content_plan( return; } }; - let key = redis.keyspace().with_slot( + let key = redis.key().with_slot( CONTENT_RESOLVE_CACHE_NAMESPACE, cache_key, cache_key, diff --git a/apps/labrinth/src/util/ratelimit.rs b/apps/labrinth/src/util/ratelimit.rs index 4031ce0ad9..5a54f8965c 100644 --- a/apps/labrinth/src/util/ratelimit.rs +++ b/apps/labrinth/src/util/ratelimit.rs @@ -68,7 +68,7 @@ impl AsyncRateLimiter { // Get current time in nanoseconds since UNIX epoch let now = Utc::now().timestamp_nanos_opt().unwrap_or(0); - let key = conn.keyspace().with_slot(RATE_LIMIT_NAMESPACE, key, key); + let key = conn.key().with_slot(RATE_LIMIT_NAMESPACE, key, key); // Get the current TAT from Redis (if it exists) let tat_str = conn.get(&key).await.ok().flatten(); diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 6ac52cbf24..de36143de9 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -61,7 +61,7 @@ async fn test_get_project() { // Confirm that the request was cached let mut redis_pool = test_env.db.redis_pool.connect().await.unwrap(); let slug_key = redis_pool - .keyspace() + .key() .entity(PROJECTS_SLUGS_NAMESPACE, alpha_project_slug); assert_eq!( redis_pool @@ -72,7 +72,7 @@ async fn test_get_project() { Some(parse_base62(alpha_project_id).unwrap() as i64) ); - let project_key = redis_pool.keyspace().entity( + let project_key = redis_pool.key().entity( PROJECTS_NAMESPACE, parse_base62(alpha_project_id).unwrap(), ); @@ -287,11 +287,9 @@ async fn test_add_remove_project() { // Confirm that the project is gone from the cache let mut redis_pool = test_env.db.redis_pool.connect().await.unwrap(); - let slug_key = redis_pool - .keyspace() - .entity(PROJECTS_SLUGS_NAMESPACE, "demo"); - let id_key = - redis_pool.keyspace().entity(PROJECTS_SLUGS_NAMESPACE, &id); + let slug_key = + redis_pool.key().entity(PROJECTS_SLUGS_NAMESPACE, "demo"); + let id_key = redis_pool.key().entity(PROJECTS_SLUGS_NAMESPACE, &id); assert_eq!( redis_pool .get(&slug_key) diff --git a/apps/labrinth/tests/v2/project.rs b/apps/labrinth/tests/v2/project.rs index 9a83ca76cd..36bb380323 100644 --- a/apps/labrinth/tests/v2/project.rs +++ b/apps/labrinth/tests/v2/project.rs @@ -268,11 +268,9 @@ async fn test_add_remove_project() { // Confirm that the project is gone from the cache let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap(); - let slug_key = redis_conn - .keyspace() - .entity(PROJECTS_SLUGS_NAMESPACE, "demo"); - let id_key = - redis_conn.keyspace().entity(PROJECTS_SLUGS_NAMESPACE, &id); + let slug_key = + redis_conn.key().entity(PROJECTS_SLUGS_NAMESPACE, "demo"); + let id_key = redis_conn.key().entity(PROJECTS_SLUGS_NAMESPACE, &id); assert_eq!( redis_conn .get(&slug_key) diff --git a/apps/labrinth/tests/version.rs b/apps/labrinth/tests/version.rs index ebbdf00ec9..8014febce8 100644 --- a/apps/labrinth/tests/version.rs +++ b/apps/labrinth/tests/version.rs @@ -51,7 +51,7 @@ async fn test_get_version() { assert_eq!(&version.id.to_string(), alpha_version_id); let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap(); - let version_key = redis_conn.keyspace().entity( + let version_key = redis_conn.key().entity( VERSIONS_NAMESPACE, parse_base62(alpha_version_id).unwrap(), ); From c198d4a076cfc16b1571f40e94f9f8971208e2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 01:17:31 -0400 Subject: [PATCH 04/12] feat(labrinth): cache locking --- .github/workflows/turbo-ci.yml | 19 + Cargo.lock | 2 + apps/labrinth/.env.docker-compose | 3 + apps/labrinth/.env.local | 3 + apps/labrinth/Cargo.toml | 2 + apps/labrinth/src/database/redis/cache.rs | 844 +++++++++++++---- .../src/database/redis/cache/locking.rs | 165 ++++ .../redis/cache/locking/distributed.rs | 350 +++++++ .../src/database/redis/cache/locking/local.rs | 226 +++++ apps/labrinth/src/database/redis/commands.rs | 117 ++- apps/labrinth/src/database/redis/config.rs | 77 +- apps/labrinth/src/database/redis/key.rs | 22 + apps/labrinth/src/database/redis/mod.rs | 20 +- apps/labrinth/src/database/redis/pubsub.rs | 11 + apps/labrinth/src/database/redis/util.rs | 95 +- apps/labrinth/src/env.rs | 1 + apps/labrinth/src/queue/analytics/mod.rs | 23 +- apps/labrinth/tests/redis.rs | 854 ++++++++++++++++++ 18 files changed, 2525 insertions(+), 309 deletions(-) create mode 100644 apps/labrinth/src/database/redis/cache/locking.rs create mode 100644 apps/labrinth/src/database/redis/cache/locking/distributed.rs create mode 100644 apps/labrinth/src/database/redis/cache/locking/local.rs create mode 100644 apps/labrinth/tests/redis.rs diff --git a/.github/workflows/turbo-ci.yml b/.github/workflows/turbo-ci.yml index a3533e0358..e5baaa800e 100644 --- a/.github/workflows/turbo-ci.yml +++ b/.github/workflows/turbo-ci.yml @@ -82,9 +82,12 @@ jobs: SCCACHE_DIR: ${{ needs.skip-if-clean.outputs.internal == 'true' && '/mnt/sccache' || '' }} SCCACHE_CACHE_SIZE: ${{ needs.skip-if-clean.outputs.internal == 'true' && '10G' || '' }} RUSTC_WRAPPER: ${{ needs.skip-if-clean.outputs.internal == 'true' && 'sccache' || '' }} + # Redis config for tests REDIS_MODE: cluster REDIS_CONNECTION_TYPE: multiplexed REDIS_URL: redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005 + # Avoid stack overflows in tests + RUST_MIN_STACK: 16777216 steps: - name: Check out code @@ -200,6 +203,22 @@ jobs: - name: Lint and test run: pnpm run ci + # Test different Redis connection modes + + - name: Test pooled Redis cluster backend + if: steps.check-labrinth.outputs.needs_services == 'true' + env: + REDIS_CONNECTION_TYPE: pooled + run: cargo nextest run --package labrinth --test redis --no-fail-fast + + - name: Test pooled standalone Redis backend + if: steps.check-labrinth.outputs.needs_services == 'true' + env: + REDIS_MODE: standalone + REDIS_CONNECTION_TYPE: pooled + REDIS_URL: redis://127.0.0.1:6379 + run: cargo nextest run --package labrinth --test redis --no-fail-fast + - name: Verify intl:extract has been run run: | pnpm turbo run intl:extract --force diff --git a/Cargo.lock b/Cargo.lock index 82040aa753..9334184de9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5499,6 +5499,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-stream", + "tokio-util", "totp-rs", "tracing", "tracing-actix-web", @@ -11278,6 +11279,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", + "futures-util", "pin-project-lite", "slab", "tokio", diff --git a/apps/labrinth/.env.docker-compose b/apps/labrinth/.env.docker-compose index f806260962..f9c726f479 100644 --- a/apps/labrinth/.env.docker-compose +++ b/apps/labrinth/.env.docker-compose @@ -31,6 +31,9 @@ TYPESENSE_INDEX_PREFIX=labrinth REDIS_MODE=standalone REDIS_CONNECTION_TYPE=pooled +# Cache fill coordination: `local` per process or a tokenized `distributed` Redis lease +# Set consistently across all replicas; Redis failures never fall back to `local` +REDIS_CACHE_LOCKING_STRATEGY=local REDIS_URL=redis://labrinth-redis REDIS_MIN_CONNECTIONS=0 REDIS_MAX_CONNECTIONS=10000 diff --git a/apps/labrinth/.env.local b/apps/labrinth/.env.local index 2ebc8fad8b..0ae8823928 100644 --- a/apps/labrinth/.env.local +++ b/apps/labrinth/.env.local @@ -49,6 +49,9 @@ TYPESENSE_INDEX_PREFIX=labrinth REDIS_MODE=standalone REDIS_CONNECTION_TYPE=pooled +# Cache fill coordination: `local` per process or a tokenized `distributed` Redis lease +# Set consistently across all replicas; Redis failures never fall back to `local` +REDIS_CACHE_LOCKING_STRATEGY=local REDIS_URL=redis://localhost REDIS_MIN_CONNECTIONS=0 REDIS_MAX_CONNECTIONS=10000 diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index c7f6e9f4e8..2a8fa1007a 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -131,6 +131,7 @@ strum = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "sync"] } tokio-stream = { workspace = true } +tokio-util = { workspace = true, features = ["rt"] } totp-rs = { workspace = true, features = ["gen_secret"] } tracing = { workspace = true } tracing-actix-web = { workspace = true } @@ -152,6 +153,7 @@ zxcvbn = { workspace = true } [dev-dependencies] labrinth = { path = ".", features = ["test"] } +tokio = { workspace = true, features = ["test-util"] } [build-dependencies] chrono = { workspace = true } diff --git a/apps/labrinth/src/database/redis/cache.rs b/apps/labrinth/src/database/redis/cache.rs index d97ca7f130..9b75704629 100644 --- a/apps/labrinth/src/database/redis/cache.rs +++ b/apps/labrinth/src/database/redis/cache.rs @@ -4,27 +4,34 @@ use std::fmt::{Debug, Display}; use std::future::Future; use std::hash::Hash; use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; use ariadne::ids::base62_impl::{parse_base62, to_base62}; use chrono::{TimeZone, Utc}; use dashmap::DashMap; -use futures::future::Either; use futures::stream::{FuturesUnordered, StreamExt}; use redis::aio::ConnectionLike; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use thiserror::Error; +use tokio::time::{Instant, timeout_at}; use tracing::{Instrument, info_span}; use crate::database::models::DatabaseError; use super::commands; +use super::config::CacheLockingStrategy; +use super::connection::RedisBackend; use super::key::KeyBuilder; -use super::util; + +mod locking; + +use locking::{ + LockAcquisition, LockCoordinator, LockWaiter, OwnedLockGuard, WAIT_TIMEOUT, + normalize_key, +}; const ACTUAL_EXPIRY: i64 = 60 * 30; +const FILL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); const VERSION_DEFAULT_EXPIRY: i64 = 60 * 60 * 48; const VERSION_ACTUAL_EXPIRY: i64 = 60 * 60 * 24; @@ -158,15 +165,20 @@ impl CacheSettings { pub struct CacheManager { key_builder: KeyBuilder, settings: CacheSettings, - cache_list: Arc>, + locking: LockCoordinator, } impl CacheManager { - pub fn new(key_builder: KeyBuilder, settings: CacheSettings) -> Self { + pub fn new( + key_builder: KeyBuilder, + settings: CacheSettings, + locking_strategy: CacheLockingStrategy, + backend: RedisBackend, + ) -> Self { Self { + locking: LockCoordinator::new(locking_strategy, backend), key_builder, settings, - cache_list: Arc::new(DashMap::with_capacity(2048)), } } @@ -174,7 +186,7 @@ impl CacheManager { &self.settings } - #[tracing::instrument(skip(self, provider, closure))] + #[tracing::instrument(skip(self, provider, keys, closure))] pub async fn get_cached_keys( &self, provider: &P, @@ -203,7 +215,7 @@ impl CacheManager { .collect()) } - #[tracing::instrument(skip(self, provider, closure))] + #[tracing::instrument(skip(self, provider, keys, closure))] pub async fn get_cached_keys_raw( &self, provider: &P, @@ -242,7 +254,7 @@ impl CacheManager { .await } - #[tracing::instrument(skip(self, provider, closure))] + #[tracing::instrument(skip(self, provider, keys, closure))] pub async fn get_cached_keys_with_slug( &self, provider: &P, @@ -281,7 +293,7 @@ impl CacheManager { .collect()) } - #[tracing::instrument(skip(self, provider, closure))] + #[tracing::instrument(skip(self, provider, keys, closure))] pub async fn get_cached_keys_raw_with_slug( &self, provider: &P, @@ -308,140 +320,222 @@ impl CacheManager { { let ids = keys .iter() - .map(|key| (key.to_string(), key.clone())) + .map(|key| { + (normalize_key(&key.to_string(), case_sensitive), key.clone()) + }) .collect::>(); if ids.is_empty() { return Ok(HashMap::new()); } - let get_cached_values = |ids: DashMap| { - async move { - let slug_ids = if let Some(slug_namespace) = slug_namespace { - async { - let keys = ids - .iter() - .map(|entry| { - let logical_key = if case_sensitive { - entry.value().to_string() - } else { - entry.value().to_string().to_lowercase() - }; - self.key_builder - .entity(slug_namespace, logical_key) - }) - .collect::>(); - let mut connection = provider.connect().await?; - Ok::<_, DatabaseError>( - commands::get_many_strings(&mut connection, &keys) + let get_cached_values = + |ids: DashMap, + deadline: Option, + locks_released: usize, + locks_waiting: usize| { + async move { + let slug_ids = if let Some(slug_namespace) = slug_namespace + { + async { + let keys = ids + .iter() + .map(|entry| { + let logical_key = normalize_key( + &entry.value().to_string(), + case_sensitive, + ); + self.key_builder + .entity(slug_namespace, logical_key) + }) + .collect::>(); + let mut connection = connect_before_deadline( + provider, + deadline, + self.locking.strategy(), + locks_released, + locks_waiting, + ) + .await?; + Ok::<_, DatabaseError>( + query_before_deadline( + deadline, + commands::get_many_strings( + &mut connection, + &keys, + ), + self.locking.strategy(), + locks_released, + locks_waiting, + ) .await? .into_iter() .flatten() .collect::>(), - ) - } - .instrument(info_span!("get slug ids")) - .await? - } else { - Vec::new() - }; + ) + } + .instrument(info_span!("get slug ids")) + .await? + } else { + Vec::new() + }; - let keys = ids - .iter() - .map(|entry| entry.value().to_string()) - .chain(ids.iter().filter_map(|entry| { - parse_base62(&entry.value().to_string()) - .ok() - .map(|value| value.to_string()) - })) - .chain(slug_ids) - .map(|key| self.key_builder.entity(namespace, key)) - .collect::>(); + let keys = ids + .iter() + .map(|entry| entry.value().to_string()) + .chain(ids.iter().filter_map(|entry| { + parse_base62(&entry.value().to_string()) + .ok() + .map(|value| value.to_string()) + })) + .chain(slug_ids) + .map(|key| self.key_builder.entity(namespace, key)) + .collect::>(); - let mut connection = provider.connect().await?; - let mut cached_values = HashMap::new(); - for value in commands::get_many(&mut connection, &keys).await? { - if let Some(value) = value.and_then(|value| { - self.settings - .decode_value::>(&value) - }) { - cached_values.insert(value.key.clone(), value); + let mut connection = connect_before_deadline( + provider, + deadline, + self.locking.strategy(), + locks_released, + locks_waiting, + ) + .await?; + let mut cached_values = HashMap::new(); + for value in query_before_deadline( + deadline, + commands::get_many(&mut connection, &keys), + self.locking.strategy(), + locks_released, + locks_waiting, + ) + .await? + { + if let Some(value) = value.and_then(|value| { + self.settings + .decode_value::>(&value) + }) { + cached_values.insert(value.key.clone(), value); + } } - } - Ok::<_, DatabaseError>((cached_values, ids)) - } - .instrument(info_span!("get cached values")) - }; + Ok::<_, DatabaseError>((cached_values, ids)) + } + .instrument(info_span!("get cached values")) + }; let (default_expiry, actual_expiry) = cache_expiries(namespace); let current_time = Utc::now(); let mut expired_values = HashMap::new(); + let mut expired_identities = HashMap::new(); + let deadline = Instant::now() + WAIT_TIMEOUT; - let (cached_values_raw, ids) = get_cached_values(ids).await?; + let (cached_values_raw, ids) = + get_cached_values(ids, Some(deadline), 0, 0).await?; let mut cached_values = cached_values_raw .into_iter() .filter_map(|(key, value)| { if Utc.timestamp_opt(value.iat + actual_expiry, 0).unwrap() < current_time { - expired_values.insert(value.key.to_string(), value); + let canonical_key = value.key.to_string(); + for identity in value_identities(&value, case_sensitive) { + expired_identities + .insert(identity, canonical_key.clone()); + } + expired_values.insert(canonical_key, value); None } else { - remove_resolved_ids(&ids, &value); + remove_resolved_ids(&ids, &value, case_sensitive); Some((key, value)) } }) .collect::>(); - let subscribe_ids = DashMap::new(); - let mut cache_writers = HashMap::new(); + let mut waiters = Vec::new(); + let mut owned_locks = HashMap::new(); + let mut has_contention = false; if !ids.is_empty() { let fetch_ids = ids .iter() .map(|entry| entry.key().clone()) .collect::>(); + let locks_total = fetch_ids.len(); for key in fetch_ids { - let lock_key = self.key_builder.entity( - namespace, - if case_sensitive { - key.to_lowercase() - } else { - key.clone() - }, - ); + if !ids.contains_key(&key) { + continue; + } - match self.acquire_lock(lock_key) { - Either::Left(sentinel) => { - cache_writers.insert(key, sentinel); + let lock_key = self.key_builder.entity(namespace, &key); + let acquisition = + match self.locking.acquire(lock_key, deadline).await { + Ok(acquisition) => acquisition, + Err(error) => { + let error = map_lock_operation_error( + error, + self.locking.strategy(), + 0, + locks_total, + ); + release_owned_locks(owned_locks, deadline).await; + return Err(error); + } + }; + + match acquisition { + LockAcquisition::Owned(guard) => { + owned_locks.insert(key, guard); } - Either::Right(subscriber) => { - if let Some((key, raw_key)) = ids.remove(&key) { - if let Some(value) = expired_values.remove(&key) { - remove_resolved_ids(&ids, &value); - cached_values.insert(value.key.clone(), value); - } else { - subscribe_ids.insert(raw_key, subscriber); - } + LockAcquisition::Waiting(waiter) => { + has_contention = true; + if let Some(canonical_key) = + expired_identities.get(&key).cloned() + && let Some(value) = + expired_values.remove(&canonical_key) + { + remove_resolved_ids(&ids, &value, case_sensitive); + expired_identities.retain(|_, canonical| { + canonical != &canonical_key + }); + cached_values.insert(value.key.clone(), value); + } else if let Some((_, raw_key)) = ids.remove(&key) { + waiters.push((raw_key, waiter)); } } } } } - let mut fetch_tasks = Vec::new(); - - if !ids.is_empty() { - fetch_tasks.push(Either::Left(async { + let is_contended = has_contention; + let fill_result = if !ids.is_empty() { + async { let fetch_ids = ids .iter() .map(|entry| entry.value().clone()) .collect::>(); - let values = closure(fetch_ids).await?; + let fill_deadline = if is_contended { + deadline + } else { + Instant::now() + FILL_TIMEOUT + }; + let values = timeout_at(fill_deadline, closure(fetch_ids)) + .await + .map_err(|_| { + if is_contended { + lock_timeout_error( + self.locking.strategy(), + 0, + waiters.len(), + ) + } else { + DatabaseError::Internal(eyre::eyre!( + "cache fill timed out after 60 seconds" + )) + } + })??; let mut return_values = HashMap::new(); - let mut connection = provider.connect().await?; + let mut encoded_values = Vec::with_capacity(values.len()); for (key, (slug, value)) in values { let value = RedisValue { @@ -450,110 +544,156 @@ impl CacheManager { val: value, alias: slug.clone(), }; - let redis_key = - self.key_builder.entity(namespace, key.to_string()); - commands::set( - &mut connection, - &redis_key, - self.settings.encode_value(&value)?, - Some(default_expiry), - ) - .await?; + let encoded = self.settings.encode_value(&value)?; + encoded_values.push((key, slug, value, encoded)); + } - if let Some(slug) = slug { - ids.remove(&slug.to_string()); - if let Some(slug_namespace) = slug_namespace { - let actual_slug = if case_sensitive { - slug.to_string() - } else { - slug.to_string().to_lowercase() - }; + let publication_deadline = if is_contended { + deadline + } else { + Instant::now() + WAIT_TIMEOUT + }; + let mut connection = connect_before_deadline( + provider, + Some(publication_deadline), + self.locking.strategy(), + 0, + waiters.len(), + ) + .await?; + let mut ownership_valid = true; + for lock in owned_locks.values() { + match lock + .validate_with_connection( + &mut connection, + publication_deadline, + ) + .await + { + Ok(true) => {} + Ok(false) => { + ownership_valid = false; + break; + } + Err(_) => { + ownership_valid = false; + break; + } + } + } + + if ownership_valid { + for (key, slug, _, encoded) in &encoded_values { + let redis_key = + self.key_builder.entity(namespace, key.to_string()); + publish_cache_value( + &mut connection, + self.locking.strategy(), + &redis_key, + encoded, + default_expiry, + publication_deadline, + waiters.len(), + ) + .await?; + if let Some(slug) = slug + && let Some(slug_namespace) = slug_namespace + { + let canonical_key = key.to_string(); + let actual_slug = normalize_key( + &slug.to_string(), + case_sensitive, + ); let slug_key = self .key_builder .entity(slug_namespace, actual_slug); - commands::set( + publish_cache_value( &mut connection, + self.locking.strategy(), &slug_key, - key.to_string(), - Some(default_expiry), + canonical_key.as_bytes(), + default_expiry, + publication_deadline, + waiters.len(), ) .await?; } } + } - let key_string = key.to_string(); - ids.remove(&key_string); - if let Ok(value) = key_string.parse::() { - ids.remove(&to_base62(value)); - } + for (key, _, value, _) in encoded_values { + remove_resolved_ids(&ids, &value, case_sensitive); return_values.insert(key, value); } - drop(cache_writers); Result::<_, DatabaseError>::Ok(return_values) - })); - } + } + .await + } else { + Ok(HashMap::new()) + }; - if !subscribe_ids.is_empty() { - fetch_tasks.push(Either::Right(async move { - let mut futures = FuturesUnordered::new(); - let len = subscribe_ids.len(); - - for (key, subscriber) in subscribe_ids { - futures.push(async move { - ( - key, - subscriber - .wait_timeout(Duration::from_secs(5)) - .await, - ) - }); - } + let release_deadline = if is_contended { + deadline + } else { + Instant::now() + WAIT_TIMEOUT + }; + release_owned_locks(owned_locks, release_deadline).await; - let fetch_ids = DashMap::with_capacity(len); - while let Some((key, result)) = futures.next().await { - result?; - fetch_ids.insert(key.to_string(), key); + let operation_result = match fill_result { + Ok(mut values) => { + if waiters.is_empty() { + Ok(values) + } else { + let total = waiters.len(); + match wait_for_locks( + self.locking.strategy(), + waiters, + deadline, + ) + .await + { + Ok(released_ids) => { + let fetch_ids = released_ids + .into_iter() + .map(|key| { + ( + normalize_key( + &key.to_string(), + case_sensitive, + ), + key, + ) + }) + .collect::>(); + match get_cached_values( + fetch_ids, + Some(deadline), + total, + total, + ) + .await + { + Ok((released_values, _)) => { + values.extend(released_values); + Ok(values) + } + Err(error) => Err(error), + } + } + Err(error) => Err(error), + } } - - let (return_values, _) = get_cached_values(fetch_ids).await?; - Ok(return_values) - })); - } - - if !fetch_tasks.is_empty() { - for values in futures::future::try_join_all(fetch_tasks).await? { - cached_values.extend(values); } - } + Err(error) => Err(error), + }; + cached_values.extend(operation_result?); Ok(cached_values .into_iter() .map(|(key, value)| (key, value.val)) .collect()) } - - fn acquire_lock( - &self, - key: String, - ) -> Either { - let mut writer = None; - let subscriber = - self.cache_list.entry(key.clone()).or_insert_with(|| { - let (new_writer, subscriber) = util::cache(); - writer = Some(new_writer); - subscriber - }); - - match writer { - Some(writer) => Either::Left(LockSentinel { - cache_list: self.cache_list.clone(), - key, - writer, - }), - None => Either::Right(subscriber.clone()), - } - } } fn cache_expiries(namespace: &str) -> (i64, i64) { @@ -572,30 +712,203 @@ fn cache_expiries(namespace: &str) -> (i64, i64) { fn remove_resolved_ids( ids: &DashMap, value: &RedisValue, + case_sensitive: bool, ) where K: Display, S: Display, { - let key = value.key.to_string(); - ids.remove(&key); - if let Ok(value) = key.parse::() { - ids.remove(&to_base62(value)); + for identity in value_identities(value, case_sensitive) { + ids.remove(&normalize_key(&identity, case_sensitive)); + } +} + +fn value_identities( + value: &RedisValue, + case_sensitive: bool, +) -> Vec +where + K: Display, + S: Display, +{ + let mut identities = Vec::with_capacity(5); + let canonical_key = value.key.to_string(); + push_identity(&mut identities, canonical_key.clone()); + push_identity( + &mut identities, + normalize_key(&canonical_key, case_sensitive), + ); + if let Ok(decimal_id) = canonical_key.parse::() { + let base62_id = to_base62(decimal_id); + push_identity(&mut identities, base62_id.clone()); + push_identity( + &mut identities, + normalize_key(&base62_id, case_sensitive), + ); + } else if let Ok(decimal_id) = parse_base62(&canonical_key) { + push_identity(&mut identities, decimal_id.to_string()); } if let Some(alias) = &value.alias { - ids.remove(&alias.to_string()); + let alias = alias.to_string(); + push_identity(&mut identities, alias.clone()); + push_identity(&mut identities, normalize_key(&alias, case_sensitive)); + } + identities +} + +fn push_identity(identities: &mut Vec, identity: String) { + if !identities.contains(&identity) { + identities.push(identity); + } +} + +async fn connect_before_deadline

( + provider: &P, + deadline: Option, + strategy: CacheLockingStrategy, + locks_released: usize, + locks_waiting: usize, +) -> Result +where + P: ConnectionProvider, +{ + if let Some(deadline) = deadline { + timeout_at(deadline, provider.connect()) + .await + .map_err(|_| { + lock_timeout_error(strategy, locks_released, locks_waiting) + })? + } else { + provider.connect().await + } +} + +async fn query_before_deadline( + deadline: Option, + query: F, + strategy: CacheLockingStrategy, + locks_released: usize, + locks_waiting: usize, +) -> Result +where + F: Future>, +{ + if let Some(deadline) = deadline { + timeout_at(deadline, query).await.map_err(|_| { + lock_timeout_error(strategy, locks_released, locks_waiting) + })? + } else { + query.await + } +} + +async fn publish_cache_value( + connection: &mut C, + strategy: CacheLockingStrategy, + key: &str, + data: &[u8], + expiry: i64, + deadline: Instant, + locks_waiting: usize, +) -> Result<(), DatabaseError> +where + C: ConnectionLike, +{ + query_before_deadline( + Some(deadline), + commands::set(connection, key, data, Some(expiry)), + strategy, + 0, + locks_waiting, + ) + .await +} + +async fn wait_for_locks( + strategy: CacheLockingStrategy, + waiters: Vec<(I, LockWaiter)>, + deadline: Instant, +) -> Result, DatabaseError> { + let total = waiters.len(); + let mut released = Vec::with_capacity(total); + let mut futures = FuturesUnordered::new(); + for (key, waiter) in waiters { + futures.push(async move { + let result = waiter.wait(deadline).await; + (key, result) + }); + } + + while let Some((key, result)) = futures.next().await { + match result { + Ok(()) => { + released.push(key); + } + Err(error) + if is_lock_timeout(&error) || Instant::now() >= deadline => + { + return Err(lock_timeout_error( + strategy, + released.len(), + total, + )); + } + Err(error) => return Err(error), + } } + Ok(released) } -struct LockSentinel { - cache_list: Arc>, - key: String, - writer: util::CacheWriter, +fn is_lock_timeout(error: &DatabaseError) -> bool { + matches!( + error, + DatabaseError::CacheTimeout { .. } + | DatabaseError::LocalCacheTimeout { .. } + ) } -impl Drop for LockSentinel { - fn drop(&mut self) { - self.writer.write(); - self.cache_list.remove(&self.key); +fn map_lock_operation_error( + error: DatabaseError, + strategy: CacheLockingStrategy, + locks_released: usize, + locks_waiting: usize, +) -> DatabaseError { + if is_lock_timeout(&error) { + lock_timeout_error(strategy, locks_released, locks_waiting) + } else { + error + } +} + +fn lock_timeout_error( + strategy: CacheLockingStrategy, + locks_released: usize, + locks_waiting: usize, +) -> DatabaseError { + match strategy { + CacheLockingStrategy::Local => DatabaseError::LocalCacheTimeout { + released: locks_released, + total: locks_waiting, + }, + CacheLockingStrategy::Distributed => DatabaseError::CacheTimeout { + locks_released, + locks_waiting, + time_spent_pool_wait_ms: 0, + time_spent_total_ms: WAIT_TIMEOUT.as_millis() as u64, + }, + } +} + +async fn release_owned_locks( + owned_locks: HashMap, + deadline: Instant, +) { + for guard in owned_locks.into_values() { + if let Err(error) = guard.release(deadline).await { + tracing::warn!( + error = ?error, + "failed to explicitly release cache lock", + ); + } } } @@ -612,3 +925,166 @@ impl RedisValue { &self.val } } + +#[cfg(test)] +mod tests { + use std::future::{Future, pending}; + use std::time::Duration; + + use dashmap::DashMap; + use redis::aio::ConnectionLike; + use tokio::time::{Instant, advance}; + + use super::{ + ConnectionProvider, RedisValue, connect_before_deadline, + query_before_deadline, remove_resolved_ids, value_identities, + }; + use crate::database::models::DatabaseError; + use crate::database::redis::CacheLockingStrategy; + + struct NeverConnection; + + impl ConnectionLike for NeverConnection { + fn req_packed_command<'a>( + &'a mut self, + _: &'a redis::Cmd, + ) -> redis::RedisFuture<'a, redis::Value> { + Box::pin(pending()) + } + + fn req_packed_commands<'a>( + &'a mut self, + _: &'a redis::Pipeline, + _: usize, + _: usize, + ) -> redis::RedisFuture<'a, Vec> { + Box::pin(pending()) + } + + fn get_db(&self) -> i64 { + 0 + } + } + + struct PendingProvider; + + impl ConnectionProvider for PendingProvider { + type Connection = NeverConnection; + + fn connect( + &self, + ) -> impl Future> + Send + { + pending() + } + } + + #[tokio::test(start_paused = true)] + async fn five_second_deadline_includes_hung_pool_acquisition() { + let deadline = Instant::now() + Duration::from_secs(5); + let task = tokio::spawn(async move { + connect_before_deadline( + &PendingProvider, + Some(deadline), + CacheLockingStrategy::Local, + 0, + 1, + ) + .await + }); + + advance(Duration::from_millis(4_999)).await; + assert!(!task.is_finished()); + advance(Duration::from_millis(1)).await; + assert!(matches!( + task.await.unwrap(), + Err(DatabaseError::LocalCacheTimeout { + released: 0, + total: 1, + }) + )); + } + + #[tokio::test(start_paused = true)] + async fn five_second_deadline_includes_hung_redis_query() { + let deadline = Instant::now() + Duration::from_secs(5); + let task = tokio::spawn(query_before_deadline( + Some(deadline), + pending::>(), + CacheLockingStrategy::Distributed, + 0, + 1, + )); + + advance(Duration::from_millis(4_999)).await; + assert!(!task.is_finished()); + advance(Duration::from_millis(1)).await; + assert!(matches!( + task.await.unwrap(), + Err(DatabaseError::CacheTimeout { + locks_released: 0, + locks_waiting: 1, + .. + }) + )); + } + + #[test] + fn stale_identity_index_includes_equivalent_request_forms() { + let value = RedisValue { + key: 1234_u64, + alias: Some("ExampleSlug"), + iat: 0, + val: (), + }; + let identities = value_identities(&value, false); + + assert!(identities.contains(&"1234".to_string())); + assert!( + identities.contains(&ariadne::ids::base62_impl::to_base62(1234)) + ); + assert!(identities.contains(&"ExampleSlug".to_string())); + assert!(identities.contains(&"exampleslug".to_string())); + } + + #[test] + fn resolving_stale_value_removes_every_equivalent_request_form() { + let value = RedisValue { + key: 1234_u64, + alias: Some("ExampleSlug"), + iat: 0, + val: (), + }; + let ids = DashMap::new(); + ids.insert("1234".to_string(), ()); + ids.insert( + ariadne::ids::base62_impl::to_base62(1234).to_lowercase(), + (), + ); + ids.insert("exampleslug".to_string(), ()); + + remove_resolved_ids(&ids, &value, false); + + assert!(ids.is_empty()); + } + + #[test] + fn case_sensitive_alias_resolution_preserves_other_case() { + let value = RedisValue { + key: 1234_u64, + alias: Some("mra_PATValue"), + iat: 0, + val: (), + }; + let identities = value_identities(&value, true); + let ids = DashMap::new(); + ids.insert("mra_PATValue".to_string(), ()); + ids.insert("mra_patvalue".to_string(), ()); + + remove_resolved_ids(&ids, &value, true); + + assert!(identities.contains(&"mra_PATValue".to_string())); + assert!(!identities.contains(&"mra_patvalue".to_string())); + assert!(ids.contains_key("mra_patvalue")); + } +} diff --git a/apps/labrinth/src/database/redis/cache/locking.rs b/apps/labrinth/src/database/redis/cache/locking.rs new file mode 100644 index 0000000000..0785486091 --- /dev/null +++ b/apps/labrinth/src/database/redis/cache/locking.rs @@ -0,0 +1,165 @@ +mod distributed; +mod local; + +use std::time::Duration; + +use redis::aio::ConnectionLike; +use tokio::time::Instant; + +use crate::database::models::DatabaseError; + +use super::super::config::CacheLockingStrategy; +use super::super::connection::RedisBackend; + +use self::distributed::{ + DistributedLockGuard, DistributedLockManager, DistributedLockWaiter, +}; +use self::local::{ + LocalLockAcquisition, LocalLockGuard, LocalLockManager, LocalLockWaiter, +}; + +pub(super) const WAIT_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone, Copy)] +pub(super) struct LockTiming { + pub(super) lease: Duration, + pub(super) renewal: Duration, + pub(super) poll_min: Duration, + pub(super) poll_max: Duration, +} + +impl Default for LockTiming { + fn default() -> Self { + Self { + lease: Duration::from_secs(5), + renewal: Duration::from_secs(2), + poll_min: Duration::from_millis(50), + poll_max: Duration::from_millis(500), + } + } +} + +#[derive(Clone)] +pub(super) enum LockCoordinator { + Local(LocalLockManager), + Distributed(DistributedLockManager), +} + +impl LockCoordinator { + pub(super) fn new( + strategy: CacheLockingStrategy, + backend: RedisBackend, + ) -> Self { + match strategy { + CacheLockingStrategy::Local => Self::Local(LocalLockManager::new()), + CacheLockingStrategy::Distributed => Self::Distributed( + DistributedLockManager::new(backend, LockTiming::default()), + ), + } + } + + pub(super) fn strategy(&self) -> CacheLockingStrategy { + match self { + Self::Local(_) => CacheLockingStrategy::Local, + Self::Distributed(_) => CacheLockingStrategy::Distributed, + } + } + + pub(super) async fn acquire( + &self, + key: String, + deadline: Instant, + ) -> Result { + match self { + Self::Local(manager) => Ok(manager.acquire(key).into()), + Self::Distributed(manager) => manager.acquire(key, deadline).await, + } + } +} + +pub(super) enum LockAcquisition { + Owned(OwnedLockGuard), + Waiting(LockWaiter), +} + +impl From for LockAcquisition { + fn from(acquisition: LocalLockAcquisition) -> Self { + match acquisition { + LocalLockAcquisition::Owned(guard) => { + Self::Owned(OwnedLockGuard::Local(guard)) + } + LocalLockAcquisition::Waiting(waiter) => { + Self::Waiting(LockWaiter::Local(waiter)) + } + } + } +} + +pub(super) enum OwnedLockGuard { + Local(LocalLockGuard), + Distributed(DistributedLockGuard), +} + +impl OwnedLockGuard { + pub(super) async fn validate_with_connection( + &self, + connection: &mut C, + deadline: Instant, + ) -> Result + where + C: ConnectionLike, + { + match self { + Self::Local(_) => Ok(true), + Self::Distributed(guard) => { + guard.validate_with_connection(connection, deadline).await + } + } + } + + pub(super) async fn release( + self, + deadline: Instant, + ) -> Result { + match self { + Self::Local(guard) => { + guard.release(); + Ok(ReleaseOutcome::Released) + } + Self::Distributed(guard) => guard.release(deadline).await, + } + } +} + +pub(super) enum LockWaiter { + Local(LocalLockWaiter), + Distributed(DistributedLockWaiter), +} + +impl LockWaiter { + pub(super) async fn wait( + self, + deadline: Instant, + ) -> Result<(), DatabaseError> { + match self { + Self::Local(waiter) => waiter.wait(deadline).await, + Self::Distributed(waiter) => waiter.wait(deadline).await, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ReleaseOutcome { + Released, + NotOwner, +} + +/// Normalize only the requested lookup form's case. Raw IDs and aliases remain +/// distinct lock identities and may therefore fill concurrently. +pub(super) fn normalize_key(key: &str, case_sensitive: bool) -> String { + if case_sensitive { + key.to_owned() + } else { + key.to_lowercase() + } +} diff --git a/apps/labrinth/src/database/redis/cache/locking/distributed.rs b/apps/labrinth/src/database/redis/cache/locking/distributed.rs new file mode 100644 index 0000000000..b5ccfea686 --- /dev/null +++ b/apps/labrinth/src/database/redis/cache/locking/distributed.rs @@ -0,0 +1,350 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use redis::aio::ConnectionLike; +use tokio::runtime::Handle; +use tokio::time::{Instant, sleep, timeout_at}; +use tokio_util::sync::CancellationToken; +use tracing::warn; + +use crate::database::models::DatabaseError; + +use super::super::super::commands; +use super::super::super::connection::RedisBackend; +use super::{ + LockAcquisition, LockTiming, LockWaiter, OwnedLockGuard, ReleaseOutcome, +}; + +#[derive(Clone)] +pub struct DistributedLockManager { + backend: RedisBackend, + timing: LockTiming, +} + +impl DistributedLockManager { + pub(super) fn new(backend: RedisBackend, timing: LockTiming) -> Self { + Self { backend, timing } + } + + pub(super) async fn acquire( + &self, + key: String, + deadline: Instant, + ) -> Result { + let lock_key = cache_lock_key(&key); + let acquired = self.try_acquire(&lock_key, deadline).await?; + + if !acquired { + return Ok(LockAcquisition::Waiting(LockWaiter::Distributed( + DistributedLockWaiter { + manager: self.clone(), + lock_key, + }, + ))); + } + + let state = Arc::new(LeaseState::new()); + let renewal_cancellation_token = CancellationToken::new(); + self.spawn_renewal( + lock_key.clone(), + state.clone(), + renewal_cancellation_token.clone(), + ); + Ok(LockAcquisition::Owned(OwnedLockGuard::Distributed( + DistributedLockGuard { + manager: self.clone(), + lock_key, + state, + renewal_cancellation_token, + cleanup_complete: false, + }, + ))) + } + + async fn connect( + &self, + deadline: Instant, + ) -> Result + { + timeout_at(deadline, self.backend.connect()) + .await + .map_err(|_| lock_timeout())? + .map_err(Into::into) + } + + async fn try_acquire( + &self, + lock_key: &str, + deadline: Instant, + ) -> Result { + let mut connection = self.connect(deadline).await?; + timeout_at( + deadline, + commands::acquire_lock( + &mut connection, + lock_key, + duration_millis(self.timing.lease), + ), + ) + .await + .map_err(|_| lock_timeout())? + } + + async fn renew( + &self, + lock_key: &str, + deadline: Instant, + ) -> Result { + let mut connection = self.connect(deadline).await?; + timeout_at( + deadline, + commands::renew_lock( + &mut connection, + lock_key, + duration_millis(self.timing.lease), + ), + ) + .await + .map_err(|_| lock_timeout())? + } + + async fn renew_with_connection( + &self, + connection: &mut C, + lock_key: &str, + deadline: Instant, + ) -> Result + where + C: ConnectionLike, + { + timeout_at( + deadline, + commands::renew_lock( + connection, + lock_key, + duration_millis(self.timing.lease), + ), + ) + .await + .map_err(|_| lock_timeout())? + } + + async fn release( + &self, + lock_key: &str, + deadline: Instant, + ) -> Result { + let mut connection = self.connect(deadline).await?; + timeout_at(deadline, commands::release_lock(&mut connection, lock_key)) + .await + .map_err(|_| lock_timeout())? + } + + async fn exists( + &self, + lock_key: &str, + deadline: Instant, + ) -> Result { + let mut connection = self.connect(deadline).await?; + timeout_at(deadline, commands::lock_exists(&mut connection, lock_key)) + .await + .map_err(|_| lock_timeout())? + } + + fn spawn_renewal( + &self, + lock_key: String, + state: Arc, + cancellation_token: CancellationToken, + ) { + let manager = self.clone(); + let _ = tokio::spawn(async move { + loop { + let result = tokio::select! { + biased; + _ = cancellation_token.cancelled() => break, + result = async { + sleep(manager.timing.renewal).await; + let deadline = Instant::now() + manager.timing.renewal; + manager.renew(&lock_key, deadline).await + } => result, + }; + + match result { + Ok(true) => {} + Ok(false) => { + state.mark_lost(); + break; + } + Err(_) => { + state.mark_lost(); + warn!("failed to renew distributed cache lease"); + break; + } + } + } + }); + } + + fn spawn_cleanup(&self, lock_key: String) { + let Ok(handle) = Handle::try_current() else { + return; + }; + let manager = self.clone(); + handle.spawn(async move { + let deadline = Instant::now() + manager.timing.renewal; + if manager.release(&lock_key, deadline).await.is_err() { + warn!("failed to clean up distributed cache lease"); + } + }); + } +} + +pub struct DistributedLockGuard { + manager: DistributedLockManager, + lock_key: String, + state: Arc, + renewal_cancellation_token: CancellationToken, + cleanup_complete: bool, +} + +impl DistributedLockGuard { + pub(super) async fn validate_with_connection( + &self, + connection: &mut C, + deadline: Instant, + ) -> Result + where + C: ConnectionLike, + { + if !self.state.owned.load(Ordering::Acquire) { + return Ok(false); + } + + let result = self + .manager + .renew_with_connection(connection, &self.lock_key, deadline) + .await; + self.handle_validation_result(result) + } + + fn handle_validation_result( + &self, + result: Result, + ) -> Result { + match result { + Ok(true) => Ok(self.state.owned.load(Ordering::Acquire)), + Ok(false) => { + self.state.mark_lost(); + Ok(false) + } + Err(error) => { + self.state.mark_lost(); + Err(error) + } + } + } + + pub(super) async fn release( + mut self, + deadline: Instant, + ) -> Result { + self.stop_renewal(); + match self.manager.release(&self.lock_key, deadline).await { + Ok(true) => { + self.cleanup_complete = true; + self.state.owned.store(false, Ordering::Release); + Ok(ReleaseOutcome::Released) + } + Ok(false) => { + self.cleanup_complete = true; + self.state.mark_lost(); + Ok(ReleaseOutcome::NotOwner) + } + Err(error) => Err(error), + } + } + + fn stop_renewal(&self) { + self.renewal_cancellation_token.cancel(); + } +} + +impl Drop for DistributedLockGuard { + fn drop(&mut self) { + self.stop_renewal(); + if self.cleanup_complete { + return; + } + + self.manager.spawn_cleanup(self.lock_key.clone()); + } +} + +pub struct DistributedLockWaiter { + manager: DistributedLockManager, + lock_key: String, +} + +impl DistributedLockWaiter { + pub(super) async fn wait( + self, + deadline: Instant, + ) -> Result<(), DatabaseError> { + let mut attempt = 0; + loop { + if !self.manager.exists(&self.lock_key, deadline).await? { + return Ok(()); + } + + let delay = poll_delay(self.manager.timing, attempt); + + timeout_at(deadline, sleep(delay)) + .await + .map_err(|_| lock_timeout())?; + attempt = attempt.saturating_add(1); + } + } +} + +struct LeaseState { + owned: AtomicBool, +} + +impl LeaseState { + fn new() -> Self { + Self { + owned: AtomicBool::new(true), + } + } + + fn mark_lost(&self) { + self.owned.store(false, Ordering::Release); + } +} + +fn cache_lock_key(key: &str) -> String { + format!("{key}/lock") +} + +fn poll_delay(timing: LockTiming, attempt: u32) -> Duration { + // With the defaults, delays are 50, 100, 200, 250 then 500 ms per poll. + timing + .poll_min + .saturating_mul(2_u32.saturating_pow(attempt)) + .min(timing.poll_max) +} + +fn duration_millis(duration: Duration) -> u64 { + duration.as_millis().min(u64::MAX as u128) as u64 +} + +fn lock_timeout() -> DatabaseError { + DatabaseError::CacheTimeout { + locks_released: 0, + locks_waiting: 1, + time_spent_pool_wait_ms: 0, + time_spent_total_ms: 0, + } +} diff --git a/apps/labrinth/src/database/redis/cache/locking/local.rs b/apps/labrinth/src/database/redis/cache/locking/local.rs new file mode 100644 index 0000000000..7bdada8713 --- /dev/null +++ b/apps/labrinth/src/database/redis/cache/locking/local.rs @@ -0,0 +1,226 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use dashmap::DashMap; +use dashmap::mapref::entry::Entry; +use tokio::sync::Notify; +use tokio::time::{Instant, timeout_at}; + +use crate::database::models::DatabaseError; + +#[derive(Clone)] +pub(in crate::database::redis::cache) struct LocalLockManager { + locks: Arc>>, +} + +impl LocalLockManager { + pub(super) fn new() -> Self { + Self { + locks: Arc::new(DashMap::with_capacity(2048)), + } + } + + pub(super) fn acquire(&self, key: String) -> LocalLockAcquisition { + match self.locks.entry(key.clone()) { + Entry::Occupied(entry) => { + LocalLockAcquisition::Waiting(LocalLockWaiter { + state: entry.get().clone(), + }) + } + Entry::Vacant(entry) => { + let state = Arc::new(LocalLockState::new()); + entry.insert(state.clone()); + LocalLockAcquisition::Owned(LocalLockGuard { + locks: self.locks.clone(), + key, + state, + released: false, + }) + } + } + } +} + +pub(super) enum LocalLockAcquisition { + Owned(LocalLockGuard), + Waiting(LocalLockWaiter), +} + +pub(in crate::database::redis::cache) struct LocalLockGuard { + locks: Arc>>, + key: String, + state: Arc, + released: bool, +} + +impl LocalLockGuard { + pub(super) fn release(mut self) { + self.release_inner(); + } + + fn release_inner(&mut self) { + if self.released { + return; + } + + self.released = true; + self.locks + .remove_if(&self.key, |_, state| Arc::ptr_eq(state, &self.state)); + self.state.released.store(true, Ordering::Release); + self.state.notify.notify_waiters(); + } +} + +impl Drop for LocalLockGuard { + fn drop(&mut self) { + self.release_inner(); + } +} + +pub(in crate::database::redis::cache) struct LocalLockWaiter { + state: Arc, +} + +impl LocalLockWaiter { + pub(super) async fn wait( + self, + deadline: Instant, + ) -> Result<(), DatabaseError> { + loop { + if self.state.released.load(Ordering::Acquire) { + return Ok(()); + } + + let notified = self.state.notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.state.released.load(Ordering::Acquire) { + return Ok(()); + } + + timeout_at(deadline, notified) + .await + .map_err(|_| lock_timeout())?; + } + } +} + +struct LocalLockState { + released: AtomicBool, + notify: Notify, +} + +impl LocalLockState { + fn new() -> Self { + Self { + released: AtomicBool::new(false), + notify: Notify::new(), + } + } +} + +fn lock_timeout() -> DatabaseError { + DatabaseError::LocalCacheTimeout { + released: 0, + total: 1, + } +} + +#[cfg(test)] +mod tests { + use std::future::pending; + use std::time::Duration; + + use tokio::time::Instant; + + use super::{LocalLockAcquisition, LocalLockManager}; + + #[tokio::test] + async fn waiter_is_released_when_owner_releases() { + let manager = LocalLockManager::new(); + let owner = match manager.acquire("project".to_string()) { + LocalLockAcquisition::Owned(owner) => owner, + LocalLockAcquisition::Waiting(_) => { + panic!("first acquisition waited") + } + }; + let waiter = match manager.acquire("project".to_string()) { + LocalLockAcquisition::Waiting(waiter) => waiter, + LocalLockAcquisition::Owned(_) => { + panic!("second acquisition owned") + } + }; + + owner.release(); + waiter + .wait(Instant::now() + Duration::from_secs(1)) + .await + .unwrap(); + } + + #[tokio::test] + async fn dropping_owner_releases_waiters() { + let manager = LocalLockManager::new(); + let owner = match manager.acquire("project".to_string()) { + LocalLockAcquisition::Owned(owner) => owner, + LocalLockAcquisition::Waiting(_) => { + panic!("first acquisition waited") + } + }; + let waiter = match manager.acquire("project".to_string()) { + LocalLockAcquisition::Waiting(waiter) => waiter, + LocalLockAcquisition::Owned(_) => { + panic!("second acquisition owned") + } + }; + + drop(owner); + waiter + .wait(Instant::now() + Duration::from_secs(1)) + .await + .unwrap(); + } + + #[tokio::test] + async fn aborting_owner_task_releases_waiters() { + let manager = LocalLockManager::new(); + let owner = match manager.acquire("project".to_string()) { + LocalLockAcquisition::Owned(owner) => owner, + LocalLockAcquisition::Waiting(_) => { + panic!("first acquisition waited") + } + }; + let waiter = match manager.acquire("project".to_string()) { + LocalLockAcquisition::Waiting(waiter) => waiter, + LocalLockAcquisition::Owned(_) => { + panic!("second acquisition owned") + } + }; + let task = tokio::spawn(async move { + let _owner = owner; + pending::<()>().await; + }); + + task.abort(); + let _ = task.await; + waiter + .wait(Instant::now() + Duration::from_secs(1)) + .await + .unwrap(); + } + + #[test] + fn independent_managers_do_not_coordinate() { + let first = LocalLockManager::new(); + let second = LocalLockManager::new(); + + assert!(matches!( + first.acquire("project".to_string()), + LocalLockAcquisition::Owned(_) + )); + assert!(matches!( + second.acquire("project".to_string()), + LocalLockAcquisition::Owned(_) + )); + } +} diff --git a/apps/labrinth/src/database/redis/commands.rs b/apps/labrinth/src/database/redis/commands.rs index 102475e170..c1cfbda4e7 100644 --- a/apps/labrinth/src/database/redis/commands.rs +++ b/apps/labrinth/src/database/redis/commands.rs @@ -11,7 +11,7 @@ use super::util::cmd; pub const DEFAULT_EXPIRY: i64 = 60 * 60 * 12; pub const MGET_CHUNK_SIZE: usize = 32; -#[tracing::instrument(skip(connection, data))] +#[tracing::instrument(skip_all)] pub async fn set( connection: &mut C, key: &str, @@ -32,7 +32,7 @@ where Ok(()) } -#[tracing::instrument(skip(connection, data, settings))] +#[tracing::instrument(skip_all)] pub async fn set_serialized( connection: &mut C, key: &str, @@ -47,7 +47,7 @@ where set(connection, key, settings.encode_value(&data)?, expiry).await } -#[tracing::instrument(skip(connection))] +#[tracing::instrument(skip_all)] pub async fn get( connection: &mut C, key: &str, @@ -61,7 +61,7 @@ where /// Issues ordinary `MGET` commands in bounded chunks. Cluster routing and /// result ordering remain redis-rs's responsibility; multiple chunks are not /// an atomic snapshot. -#[tracing::instrument(skip(connection, keys))] +#[tracing::instrument(skip_all)] pub async fn get_many( connection: &mut C, keys: &[String], @@ -72,7 +72,7 @@ where get_many_as(connection, keys).await } -#[tracing::instrument(skip(connection, keys))] +#[tracing::instrument(skip_all)] pub async fn get_many_strings( connection: &mut C, keys: &[String], @@ -83,7 +83,7 @@ where get_many_as(connection, keys).await } -async fn get_many_as( +pub(super) async fn get_many_as( connection: &mut C, keys: &[String], ) -> Result>, DatabaseError> @@ -102,7 +102,7 @@ where Ok(values) } -#[tracing::instrument(skip(connection, settings))] +#[tracing::instrument(skip_all)] pub async fn get_deserialized( connection: &mut C, key: &str, @@ -117,7 +117,7 @@ where Ok(value.and_then(|value| settings.decode_value(&value))) } -#[tracing::instrument(skip(connection, keys, settings))] +#[tracing::instrument(skip_all)] pub async fn get_many_deserialized( connection: &mut C, keys: &[String], @@ -134,7 +134,7 @@ where .collect()) } -#[tracing::instrument(skip(connection))] +#[tracing::instrument(skip_all)] pub async fn delete( connection: &mut C, key: &str, @@ -146,7 +146,7 @@ where Ok(()) } -#[tracing::instrument(skip(connection, keys))] +#[tracing::instrument(skip_all)] pub async fn delete_many( connection: &mut C, keys: &[String], @@ -160,7 +160,7 @@ where Ok(()) } -#[tracing::instrument(skip(connection, value))] +#[tracing::instrument(skip_all)] pub async fn lpush( connection: &mut C, key: &str, @@ -178,7 +178,7 @@ where Ok(()) } -#[tracing::instrument(skip(connection))] +#[tracing::instrument(skip_all)] pub async fn incr( connection: &mut C, key: &str, @@ -188,3 +188,96 @@ where { Ok(cmd("INCR").arg(key).query_async(connection).await?) } + +#[tracing::instrument( + name = "redis.cache_lock.acquire", + skip_all, + fields(command = "SET {} {} NX PX {}") +)] +pub(super) async fn acquire_lock( + connection: &mut C, + key: &str, + lease_ms: u64, +) -> Result +where + C: ConnectionLike, +{ + let response = acquire_lock_command(key, lease_ms) + .query_async::>(connection) + .await?; + Ok(response.is_some()) +} + +fn acquire_lock_command(key: &str, lease_ms: u64) -> redis::Cmd { + let mut command = redis::cmd("SET"); + command.arg(key).arg("").arg("NX").arg("PX").arg(lease_ms); + command +} + +#[tracing::instrument( + name = "redis.cache_lock.renew", + skip_all, + fields(command = "PEXPIRE {} {}") +)] +pub(super) async fn renew_lock( + connection: &mut C, + key: &str, + lease_ms: u64, +) -> Result +where + C: ConnectionLike, +{ + let renewed = renew_lock_command(key, lease_ms) + .query_async::(connection) + .await?; + Ok(renewed == 1) +} + +fn renew_lock_command(key: &str, lease_ms: u64) -> redis::Cmd { + let mut command = redis::cmd("PEXPIRE"); + command.arg(key).arg(lease_ms); + command +} + +#[tracing::instrument( + name = "redis.cache_lock.release", + skip_all, + fields(command = "DEL {}") +)] +pub(super) async fn release_lock( + connection: &mut C, + key: &str, +) -> Result +where + C: ConnectionLike, +{ + let released = release_lock_command(key) + .query_async::(connection) + .await?; + Ok(released == 1) +} + +fn release_lock_command(key: &str) -> redis::Cmd { + let mut command = redis::cmd("DEL"); + command.arg(key); + command +} + +#[tracing::instrument( + name = "redis.cache_lock.exists", + skip_all, + fields(command = "EXISTS {}") +)] +pub(super) async fn lock_exists( + connection: &mut C, + key: &str, +) -> Result +where + C: ConnectionLike, +{ + Ok(redis::cmd("EXISTS") + .arg(key) + .query_async::(connection) + .await? + != 0) +} diff --git a/apps/labrinth/src/database/redis/config.rs b/apps/labrinth/src/database/redis/config.rs index 7c29c491f4..780d2c1b35 100644 --- a/apps/labrinth/src/database/redis/config.rs +++ b/apps/labrinth/src/database/redis/config.rs @@ -1,8 +1,46 @@ -use std::str::FromStr; +use std::{fmt, str::FromStr}; use crate::env::ENV; use thiserror::Error; +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum CacheLockingStrategy { + #[default] + Local, + Distributed, +} + +impl CacheLockingStrategy { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Local => "local", + Self::Distributed => "distributed", + } + } +} + +impl fmt::Display for CacheLockingStrategy { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +#[derive(Debug, Error)] +#[error("invalid cache locking strategy; expected `local` or `distributed`")] +pub struct InvalidCacheLockingStrategy; + +impl FromStr for CacheLockingStrategy { + type Err = InvalidCacheLockingStrategy; + + fn from_str(value: &str) -> Result { + match value { + "local" => Ok(Self::Local), + "distributed" => Ok(Self::Distributed), + _ => Err(InvalidCacheLockingStrategy), + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RedisMode { Standalone, @@ -89,6 +127,7 @@ pub(super) struct RedisConfig { seed_urls: Vec, wait_timeout_ms: u64, blocking_pool_size: RedisPoolSize, + cache_locking_strategy: CacheLockingStrategy, } #[derive(Debug, Error)] @@ -134,6 +173,7 @@ impl RedisConfig { ENV.REDIS_BLOCKING_MAX_CONNECTIONS as usize, 0, )?, + ENV.REDIS_CACHE_LOCKING_STRATEGY, ) } @@ -145,6 +185,7 @@ impl RedisConfig { standalone_pool_size: (usize, usize), cluster_pool_size: (usize, usize), blocking_pool_size: RedisPoolSize, + cache_locking_strategy: CacheLockingStrategy, ) -> Result { let seed_urls = raw_urls .split(',') @@ -192,6 +233,7 @@ impl RedisConfig { seed_urls, wait_timeout_ms, blocking_pool_size, + cache_locking_strategy, }) } @@ -214,4 +256,37 @@ impl RedisConfig { pub(super) fn blocking_pool_size(&self) -> RedisPoolSize { self.blocking_pool_size } + + pub(super) fn cache_locking_strategy(&self) -> CacheLockingStrategy { + self.cache_locking_strategy + } +} + +#[cfg(test)] +mod tests { + use super::CacheLockingStrategy; + + #[test] + fn cache_locking_strategy_defaults_to_local() { + assert_eq!( + CacheLockingStrategy::default(), + CacheLockingStrategy::Local + ); + } + + #[test] + fn cache_locking_strategy_parsing_is_strict() { + assert_eq!( + "local".parse::().unwrap(), + CacheLockingStrategy::Local + ); + assert_eq!( + "distributed".parse::().unwrap(), + CacheLockingStrategy::Distributed + ); + + for invalid in ["", "LOCAL", "Distributed", " local", "local "] { + assert!(invalid.parse::().is_err()); + } + } } diff --git a/apps/labrinth/src/database/redis/key.rs b/apps/labrinth/src/database/redis/key.rs index 6181284cc4..ddbc9f6ec1 100644 --- a/apps/labrinth/src/database/redis/key.rs +++ b/apps/labrinth/src/database/redis/key.rs @@ -70,3 +70,25 @@ fn escape_slot_tag(value: &str) -> String { } escaped } + +#[cfg(test)] +mod tests { + use super::KeyBuilder; + use crate::database::redis::RedisMode; + + #[test] + fn cluster_slot_tags_escape_braces_and_percent_signs() { + let builder = KeyBuilder::new("labrinth", RedisMode::Cluster); + let entity = builder.entity("projects", "a{%}b"); + + assert!(entity.contains("{a%7B%25%7Db}")); + } + + #[test] + fn cluster_empty_slot_tag_is_nonempty() { + let builder = KeyBuilder::new("labrinth", RedisMode::Cluster); + let entity = builder.entity("projects", ""); + + assert!(entity.contains("{%00}")); + } +} diff --git a/apps/labrinth/src/database/redis/mod.rs b/apps/labrinth/src/database/redis/mod.rs index 0bef753ae9..9b057faf70 100644 --- a/apps/labrinth/src/database/redis/mod.rs +++ b/apps/labrinth/src/database/redis/mod.rs @@ -5,8 +5,8 @@ use std::sync::Arc; use dashmap::DashMap; use prometheus::Registry; -use redis::ToRedisArgs; use redis::aio::ConnectionLike; +use redis::{FromRedisValue, ToRedisArgs}; use serde::Serialize; use serde::de::DeserializeOwned; @@ -26,7 +26,7 @@ mod util; use cache::{CacheManager, CacheSettings, ConnectionProvider}; pub use cache::{Codec, EncodingFormat, RedisValue}; use config::RedisConfig; -pub use config::{RedisConnectionType, RedisMode}; +pub use config::{CacheLockingStrategy, RedisConnectionType, RedisMode}; use connection::RedisBackend; pub use key::KeyBuilder; @@ -49,6 +49,10 @@ impl RedisPool { pub async fn new(meta_namespace: impl Into>) -> Self { let config = RedisConfig::from_env().expect("invalid Redis configuration"); + tracing::info!( + strategy = %config.cache_locking_strategy(), + "configured Redis cache locking" + ); let backend = RedisBackend::new(&config) .await .expect("failed to initialize Redis connections"); @@ -67,6 +71,8 @@ impl RedisPool { compression_min_savings_ratio: ENV .REDIS_COMPRESSION_MIN_SAVINGS_RATIO, }, + config.cache_locking_strategy(), + backend.clone(), ); Self { @@ -276,6 +282,16 @@ impl RedisConnection { commands::get_many(&mut self.inner, keys).await } + pub async fn get_many_typed( + &mut self, + keys: &[String], + ) -> Result>, DatabaseError> + where + R: FromRedisValue, + { + commands::get_many_as(&mut self.inner, keys).await + } + pub async fn get_deserialized( &mut self, key: &str, diff --git a/apps/labrinth/src/database/redis/pubsub.rs b/apps/labrinth/src/database/redis/pubsub.rs index aed90d871c..169e1bf2f0 100644 --- a/apps/labrinth/src/database/redis/pubsub.rs +++ b/apps/labrinth/src/database/redis/pubsub.rs @@ -29,6 +29,17 @@ impl RedisPool { receiver } + #[doc(hidden)] + #[cfg(feature = "test")] + pub fn subscribe_with_seed_urls( + seed_urls: Vec, + channel: &'static str, + ) -> mpsc::Receiver> { + let (sender, receiver) = mpsc::channel(PUBSUB_BUFFER_SIZE); + tokio::spawn(run_subscription(seed_urls, channel, sender)); + receiver + } + pub async fn publish( &self, channel: &str, diff --git a/apps/labrinth/src/database/redis/util.rs b/apps/labrinth/src/database/redis/util.rs index badf2f32e1..166843fdac 100644 --- a/apps/labrinth/src/database/redis/util.rs +++ b/apps/labrinth/src/database/redis/util.rs @@ -1,32 +1,21 @@ -use std::fmt::Debug; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; - use redis::{FromRedisValue, RedisResult, ToRedisArgs}; -use tokio::sync::Notify; -use tokio::time::{Duration, timeout}; use tracing::{Instrument, info_span}; -use crate::database::models::DatabaseError; - pub fn cmd(name: &str) -> InstrumentedCmd { InstrumentedCmd { inner: redis::cmd(name), name: name.to_string(), - args: Vec::new(), } } pub struct InstrumentedCmd { inner: redis::Cmd, name: String, - args: Vec, } impl InstrumentedCmd { #[inline] - pub fn arg(&mut self, arg: T) -> &mut Self { - self.args.push(format!("{arg:?}")); + pub fn arg(&mut self, arg: T) -> &mut Self { self.inner.arg(arg); self } @@ -41,88 +30,8 @@ impl InstrumentedCmd { // db.system.name = "redis", db.operation.name = self.name, - db.query.text = format!("{} {}", self.name, self.args.join(" ")), + db.query.text = self.name, ); self.inner.query_async(con).instrument(span).await } } - -pub fn cache() -> (CacheWriter, CacheSubscriber) { - let shared = Arc::new(Shared::new()); - ( - CacheWriter { - shared: shared.clone(), - }, - CacheSubscriber { shared }, - ) -} - -pub struct CacheWriter { - shared: Arc, -} - -impl CacheWriter { - pub fn write(&self) { - self.shared.make_ready(); - } -} - -#[derive(Clone)] -pub struct CacheSubscriber { - shared: Arc, -} - -impl CacheSubscriber { - pub async fn wait_timeout( - self, - duration: Duration, - ) -> Result<(), DatabaseError> { - timeout(duration, self.shared.wait()).await.map_err(|_| { - DatabaseError::LocalCacheTimeout { - released: 0, - total: 1, - } - }) - } -} - -struct Shared { - ready: AtomicBool, - // With this implementation's intrusive linked lists, the waiters are stored inline in the future - // so there's no heap allocation per waiter. - wakers: Notify, -} - -impl Shared { - fn new() -> Self { - Self { - ready: AtomicBool::new(false), - wakers: Notify::new(), - } - } - - fn make_ready(&self) { - self.ready.store(true, Ordering::Release); - self.wakers.notify_waiters(); - } - - async fn wait(&self) { - let ready = self.ready.load(Ordering::Acquire); - - if ready { - return; - } - - let notification = self.wakers.notified(); - // Don't need to call `enable` as we use notify_waiters - - // Prevent race where the writer set the ready bit and notified waiters between the load and registering the waiter - let ready = self.ready.load(Ordering::SeqCst); - - if ready { - return; - } - - notification.await; - } -} diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index 53e097c343..f6b975fab0 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -131,6 +131,7 @@ vars! { DATABASE_URL: String = "postgresql://labrinth:labrinth@localhost/labrinth"; REDIS_MODE: crate::database::redis::RedisMode = crate::database::redis::RedisMode::Standalone; REDIS_CONNECTION_TYPE: crate::database::redis::RedisConnectionType = crate::database::redis::RedisConnectionType::Pooled; + REDIS_CACHE_LOCKING_STRATEGY: crate::database::redis::CacheLockingStrategy = crate::database::redis::CacheLockingStrategy::Local; REDIS_URL: String = "redis://localhost"; KAFKA_BOOTSTRAP_SERVERS: StringCsv = StringCsv(vec!["localhost:19092".into()]); KAFKA_CLIENT_ID: String = "labrinth"; diff --git a/apps/labrinth/src/queue/analytics/mod.rs b/apps/labrinth/src/queue/analytics/mod.rs index 4249bcc9e9..e7b867145e 100644 --- a/apps/labrinth/src/queue/analytics/mod.rs +++ b/apps/labrinth/src/queue/analytics/mod.rs @@ -1,5 +1,4 @@ use crate::database::PgPool; -use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::models::analytics::{ AffiliateCodeClick, Download, MinecraftServerPlay, PageView, Playtime, @@ -7,7 +6,6 @@ use crate::models::analytics::{ use crate::routes::ApiError; use crate::routes::analytics::MINECRAFT_SERVER_PLAYS; use dashmap::{DashMap, DashSet}; -use redis::cmd; use std::collections::HashMap; use tracing::trace; @@ -146,11 +144,8 @@ impl AnalyticsQueue { .collect::>(); let mut redis_connection = redis.connect().await?; - let results = cmd("MGET") - .arg(&redis_keys) - .query_async::>>(&mut redis_connection) - .await - .map_err(DatabaseError::CacheError)?; + let results = + redis_connection.get_many_typed::(&redis_keys).await?; for (idx, count) in results.into_iter().enumerate() { let new_count = if let Some(count) = count { if count >= MINECRAFT_SERVER_PLAYS_LIMIT { @@ -205,11 +200,8 @@ impl AnalyticsQueue { .collect::>(); let mut redis_connection = redis.connect().await?; - let results = cmd("MGET") - .arg(&redis_keys) - .query_async::>>(&mut redis_connection) - .await - .map_err(DatabaseError::CacheError)?; + let results = + redis_connection.get_many_typed::(&redis_keys).await?; for (idx, count) in results.into_iter().enumerate() { let new_count = if let Some((views, monetized)) = raw_views.get_mut(idx) { @@ -277,11 +269,8 @@ impl AnalyticsQueue { .collect::>(); let mut redis_connection = redis.connect().await?; - let results = cmd("MGET") - .arg(&redis_keys) - .query_async::>>(&mut redis_connection) - .await - .map_err(DatabaseError::CacheError)?; + let results = + redis_connection.get_many_typed::(&redis_keys).await?; for (idx, count) in results.into_iter().enumerate() { let new_count = if let Some(count) = count { if count > 5 { diff --git a/apps/labrinth/tests/redis.rs b/apps/labrinth/tests/redis.rs new file mode 100644 index 0000000000..9f2af478a0 --- /dev/null +++ b/apps/labrinth/tests/redis.rs @@ -0,0 +1,854 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use actix_http::StatusCode; +use actix_web::test; +use ariadne::ids::base62_impl::parse_base62; +use common::api_common::models::{CommonProject, CommonVersion}; +use common::api_common::{ApiProject, ApiVersion}; +use common::database::{ENEMY_USER_PAT, USER_USER_PAT}; +use common::environment::{TestEnvironment, with_test_environment}; +use dashmap::DashMap; +use labrinth::database::models::DatabaseError; +use labrinth::database::models::project_item::{ + PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, +}; +use labrinth::database::models::version_item::VERSIONS_NAMESPACE; +use labrinth::database::redis::{KeyBuilder, RedisMode, RedisPool}; +use redis::cluster_routing::Slot; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use tokio::sync::{Barrier, Notify}; +use tokio::time::timeout; +use uuid::Uuid; + +pub mod common; + +async fn isolated_redis_pool(label: &str) -> RedisPool { + labrinth::env::init().expect("failed to initialize test environment"); + RedisPool::new(format!("redis_test_{label}_{}", Uuid::new_v4())).await +} + +fn clustered_key_builder(label: &str) -> KeyBuilder { + KeyBuilder::new( + format!("redis_test_{label}_{}", Uuid::new_v4()), + RedisMode::Cluster, + ) +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +struct CachedNumber { + value: usize, +} + +/// Protects redis-rs's cross-slot result recombination and Labrinth's 32-key +/// chunk boundary, including missing and duplicate keys whose positions must +/// not shift. +#[actix_rt::test] +async fn cross_slot_mget_preserves_input_order_across_chunks() { + let pool = isolated_redis_pool("cross_slot_mget").await; + let keys = clustered_key_builder("cross_slot_mget"); + let mut connection = pool.connect().await.unwrap(); + + let stored_keys = (0..65) + .map(|index| keys.entity("raw", format!("key-{index}"))) + .collect::>(); + let slots = stored_keys + .iter() + .map(Slot::for_key) + .collect::>(); + assert!(slots.len() > 1, "test keys must span Redis hash slots"); + + let mut expected_values = HashMap::new(); + for (index, key) in stored_keys.iter().enumerate() { + let value = format!("value-{index}").into_bytes(); + connection.set(key, &value, None).await.unwrap(); + expected_values.insert(key.clone(), value); + } + + let mut query_keys = stored_keys.iter().rev().cloned().collect::>(); + query_keys.insert(1, keys.entity("raw", "missing-first-chunk")); + query_keys.insert(33, keys.entity("raw", "missing-second-chunk")); + query_keys.push(stored_keys[7].clone()); + + let actual = connection.get_many(&query_keys).await.unwrap(); + let expected = query_keys + .iter() + .map(|key| expected_values.get(key).cloned()) + .collect::>(); + assert_eq!(actual, expected); + + // The typed path is used by analytics counters, so it must receive the + // same cross-slot splitting and ordered recombination as cache reads. + let counter_keys = (0..3) + .map(|index| keys.entity("counter", format!("key-{index}"))) + .collect::>(); + assert!( + counter_keys + .iter() + .map(Slot::for_key) + .collect::>() + .len() + > 1 + ); + for (index, key) in counter_keys.iter().enumerate() { + connection.set(key, index as u32, None).await.unwrap(); + } + assert_eq!( + connection + .get_many_typed::(&counter_keys) + .await + .unwrap(), + vec![Some(0), Some(1), Some(2)] + ); +} + +/// Protects the serialized cache path from losing ordering or cardinality when +/// a many-get crosses both hash slots and Labrinth's command chunk boundary. +#[actix_rt::test] +async fn cross_slot_deserialized_mget_preserves_input_order() { + let pool = isolated_redis_pool("cross_slot_deserialized").await; + let keys = clustered_key_builder("cross_slot_deserialized"); + let mut connection = pool.connect().await.unwrap(); + + let stored_keys = (0..65) + .map(|index| keys.entity("serialized", format!("key-{index}"))) + .collect::>(); + assert!( + stored_keys + .iter() + .map(Slot::for_key) + .collect::>() + .len() + > 1 + ); + for (index, key) in stored_keys.iter().enumerate() { + connection + .set_serialized(key, CachedNumber { value: index }, None) + .await + .unwrap(); + } + + let mut query_keys = stored_keys.iter().rev().cloned().collect::>(); + query_keys.insert(32, keys.entity("serialized", "missing")); + query_keys.push(stored_keys[12].clone()); + + let actual = connection + .get_many_deserialized::(&query_keys) + .await + .unwrap(); + let expected = query_keys + .iter() + .map(|key| { + stored_keys + .iter() + .position(|stored| stored == key) + .map(|value| CachedNumber { value }) + }) + .collect::>(); + assert_eq!(actual, expected); +} + +/// Protects cross-slot invalidation, same-slot hash-tag behavior, and empty +/// batches; these are the three shapes used by model cache clear operations. +#[actix_rt::test] +async fn delete_many_and_same_slot_batches_work_in_cluster_mode() { + let pool = isolated_redis_pool("delete_many").await; + let keys = clustered_key_builder("delete_many"); + let mut connection = pool.connect().await.unwrap(); + + let cross_slot_keys = (0..5) + .map(|index| keys.entity("delete", format!("key-{index}"))) + .collect::>(); + assert!( + cross_slot_keys + .iter() + .map(Slot::for_key) + .collect::>() + .len() + > 1 + ); + for key in &cross_slot_keys { + connection.set(key, "present", None).await.unwrap(); + } + connection.delete_many(&cross_slot_keys).await.unwrap(); + assert_eq!( + connection.get_many(&cross_slot_keys).await.unwrap(), + vec![None; cross_slot_keys.len()] + ); + + let same_slot_keys = vec![ + keys.with_slot("same_slot", "left", "shared-entity"), + keys.with_slot("same_slot", "right", "shared-entity"), + ]; + assert_eq!( + Slot::for_key(&same_slot_keys[0]), + Slot::for_key(&same_slot_keys[1]) + ); + connection + .set(&same_slot_keys[0], "left", None) + .await + .unwrap(); + connection + .set(&same_slot_keys[1], "right", None) + .await + .unwrap(); + assert_eq!( + connection.get_many(&same_slot_keys).await.unwrap(), + vec![Some(b"left".to_vec()), Some(b"right".to_vec())] + ); + + connection.delete_many(&[]).await.unwrap(); + assert!(connection.get_many(&[]).await.unwrap().is_empty()); +} + +/// Protects hash tags from user-controlled braces, percent signs, and empty +/// values changing the substring Redis uses to select a cluster slot. +#[actix_rt::test] +async fn cluster_key_builder_escapes_slot_tags() { + let keys = clustered_key_builder("slot_escaping"); + + for (slot_tag, escaped) in [ + ("", "%00"), + ("contains%percent", "contains%25percent"), + ("contains{open", "contains%7Bopen"), + ("contains}close", "contains%7Dclose"), + ] { + let first = keys.with_slot("escape", "first", slot_tag); + let second = keys.with_slot("escape", "second", slot_tag); + assert!(first.contains(&format!("{{{escaped}}}"))); + assert_eq!(Slot::for_key(first), Slot::for_key(second)); + } + + // Metadata keys intentionally share one slot so metadata pipelines remain + // legal even when their logical entity IDs differ. + assert_eq!( + Slot::for_key(keys.metadata("metadata", "first")), + Slot::for_key(keys.metadata("metadata", "second")) + ); +} + +/// Protects the process-local single-flight contract: concurrent cache misses +/// for one key perform one backing fetch and all waiters receive its value. +#[actix_rt::test] +async fn cache_lock_coalesces_concurrent_misses_for_one_key() { + let pool = isolated_redis_pool("single_flight").await; + let barrier = Arc::new(Barrier::new(16)); + let fetch_count = Arc::new(AtomicUsize::new(0)); + let mut tasks = Vec::new(); + + for _ in 0..16 { + let pool = pool.clone(); + let barrier = barrier.clone(); + let fetch_count = fetch_count.clone(); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + pool.get_cached_keys_raw( + "single_flight:v1", + &["shared".to_string()], + move |keys| async move { + fetch_count.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(75)).await; + let values = DashMap::new(); + for key in keys { + values.insert(key.clone(), format!("value-{key}")); + } + Ok::<_, DatabaseError>(values) + }, + ) + .await + })); + } + + for task in tasks { + let values = task.await.unwrap().unwrap(); + assert_eq!(values.get("shared"), Some(&"value-shared".to_string())); + } + assert_eq!(fetch_count.load(Ordering::SeqCst), 1); +} + +/// Protects per-key locking for overlapping many-gets: `[A, B]` and `[B, C]` +/// may fetch independently, but the shared `B` must only be loaded once. +#[actix_rt::test] +async fn cache_lock_coalesces_only_overlapping_keys() { + let pool = isolated_redis_pool("overlapping_locks").await; + let barrier = Arc::new(Barrier::new(2)); + let fetch_counts = Arc::new(DashMap::::new()); + let mut tasks = Vec::new(); + + for requested in [ + vec!["A".to_string(), "B".to_string()], + vec!["B".to_string(), "C".to_string()], + ] { + let pool = pool.clone(); + let barrier = barrier.clone(); + let fetch_counts = fetch_counts.clone(); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + pool.get_cached_keys_raw( + "overlapping_locks:v1", + &requested, + move |keys| async move { + tokio::time::sleep(Duration::from_millis(75)).await; + let values = DashMap::new(); + for key in keys { + fetch_counts + .entry(key.clone()) + .and_modify(|count| *count += 1) + .or_insert(1); + values.insert(key.clone(), format!("value-{key}")); + } + Ok::<_, DatabaseError>(values) + }, + ) + .await + })); + } + + for task in tasks { + assert_eq!(task.await.unwrap().unwrap().len(), 2); + } + for key in ["A", "B", "C"] { + assert_eq!(fetch_counts.get(key).map(|count| *count), Some(1)); + } +} + +/// Protects the lock map from becoming a global mutex: a slow miss for one +/// key must not delay an unrelated cache key. +#[actix_rt::test] +async fn cache_lock_does_not_block_independent_keys() { + let pool = isolated_redis_pool("independent_locks").await; + let started = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + + let slow_pool = pool.clone(); + let slow_started = started.clone(); + let slow_release = release.clone(); + let slow = tokio::spawn(async move { + slow_pool + .get_cached_keys_raw( + "independent_locks:v1", + &["slow".to_string()], + move |keys| async move { + slow_started.notify_one(); + slow_release.notified().await; + let values = DashMap::new(); + values.insert(keys[0].clone(), "slow-value".to_string()); + Ok::<_, DatabaseError>(values) + }, + ) + .await + }); + started.notified().await; + + let fast = timeout( + Duration::from_secs(1), + pool.get_cached_keys_raw( + "independent_locks:v1", + &["fast".to_string()], + |keys| async move { + let values = DashMap::new(); + values.insert(keys[0].clone(), "fast-value".to_string()); + Ok::<_, DatabaseError>(values) + }, + ), + ) + .await + .expect("an unrelated key was blocked by the slow cache fill") + .unwrap(); + assert_eq!(fast.get("fast"), Some(&"fast-value".to_string())); + + release.notify_one(); + assert_eq!( + slow.await.unwrap().unwrap().get("slow"), + Some(&"slow-value".to_string()) + ); +} + +/// Protects lock cleanup on both error and cancellation so a failed cache fill +/// cannot strand future requests behind a stale local lock. +#[actix_rt::test] +async fn cache_lock_is_released_after_error_and_cancellation() { + let pool = isolated_redis_pool("lock_recovery").await; + + let failed = pool + .get_cached_keys_raw( + "error_recovery:v1", + &["key".to_string()], + |_| async { + Err::, _>(DatabaseError::Internal( + eyre::eyre!("intentional cache fill failure"), + )) + }, + ) + .await; + assert!(failed.is_err()); + + let recovered = timeout( + Duration::from_secs(1), + pool.get_cached_keys_raw( + "error_recovery:v1", + &["key".to_string()], + |keys| async move { + let values = DashMap::new(); + values.insert(keys[0].clone(), "recovered".to_string()); + Ok::<_, DatabaseError>(values) + }, + ), + ) + .await + .expect("error left the cache key locked") + .unwrap(); + assert_eq!(recovered.get("key"), Some(&"recovered".to_string())); + + let started = Arc::new(Notify::new()); + let cancelled_pool = pool.clone(); + let cancelled_started = started.clone(); + let cancelled = tokio::spawn(async move { + cancelled_pool + .get_cached_keys_raw( + "cancellation_recovery:v1", + &["key".to_string()], + move |_| async move { + cancelled_started.notify_one(); + std::future::pending::< + Result, DatabaseError>, + >() + .await + }, + ) + .await + }); + started.notified().await; + cancelled.abort(); + assert!(cancelled.await.unwrap_err().is_cancelled()); + + let recovered = timeout( + Duration::from_secs(1), + pool.get_cached_keys_raw( + "cancellation_recovery:v1", + &["key".to_string()], + |keys| async move { + let values = DashMap::new(); + values.insert(keys[0].clone(), "recovered".to_string()); + Ok::<_, DatabaseError>(values) + }, + ), + ) + .await + .expect("cancellation left the cache key locked") + .unwrap(); + assert_eq!(recovered.get("key"), Some(&"recovered".to_string())); +} + +/// Protects stale-while-revalidate behavior: an expired cached value may serve +/// a waiter immediately while exactly one writer refreshes it in the background. +#[actix_rt::test] +async fn expired_cache_value_serves_waiter_while_writer_refreshes() { + let pool = isolated_redis_pool("stale_while_revalidate").await; + let namespace = "stale_while_revalidate:v1"; + let logical_key = "key".to_string(); + let mut connection = pool.connect().await.unwrap(); + let redis_key = connection.key().entity(namespace, &logical_key); + connection + .set_serialized( + &redis_key, + json!({ + "key": logical_key, + "alias": null, + "iat": 0, + "val": "stale", + }), + None, + ) + .await + .unwrap(); + + let started = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let writer_pool = pool.clone(); + let writer_started = started.clone(); + let writer_release = release.clone(); + let writer = tokio::spawn(async move { + writer_pool + .get_cached_keys_raw( + namespace, + &["key".to_string()], + move |keys| async move { + writer_started.notify_one(); + writer_release.notified().await; + let values = DashMap::new(); + values.insert(keys[0].clone(), "fresh".to_string()); + Ok::<_, DatabaseError>(values) + }, + ) + .await + }); + started.notified().await; + + let stale = timeout( + Duration::from_secs(1), + pool.get_cached_keys_raw(namespace, &["key".to_string()], |_| async { + Err::, _>(DatabaseError::Internal( + eyre::eyre!("stale waiter unexpectedly became writer"), + )) + }), + ) + .await + .expect("stale value waited for the refresh") + .unwrap(); + assert_eq!(stale.get("key"), Some(&"stale".to_string())); + + release.notify_one(); + assert_eq!( + writer.await.unwrap().unwrap().get("key"), + Some(&"fresh".to_string()) + ); + let fresh = pool + .get_cached_keys_raw(namespace, &["key".to_string()], |_| async { + Err::, _>(DatabaseError::Internal( + eyre::eyre!("fresh value unexpectedly missed cache"), + )) + }) + .await + .unwrap(); + assert_eq!(fresh.get("key"), Some(&"fresh".to_string())); +} + +/// Protects case-insensitive slug locking: differently-cased aliases must map +/// to one local lock and one backing fetch, matching their lowercase Redis key. +#[actix_rt::test] +async fn case_insensitive_slug_requests_share_one_cache_lock() { + let pool = isolated_redis_pool("slug_lock_casing").await; + let canonical_id = "A1b2C3d4"; + let barrier = Arc::new(Barrier::new(2)); + let fetch_count = Arc::new(AtomicUsize::new(0)); + let mut tasks = Vec::new(); + + for requested in ["MiXeD-Slug".to_string(), "mixed-slug".to_string()] { + let pool = pool.clone(); + let barrier = barrier.clone(); + let fetch_count = fetch_count.clone(); + tasks.push(tokio::spawn(async move { + let requested = vec![requested]; + barrier.wait().await; + pool.get_cached_keys_raw_with_slug( + "slug_values:v1", + Some("slug_aliases:v1"), + false, + &requested, + move |_| async move { + fetch_count.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(75)).await; + let values = DashMap::new(); + values.insert( + canonical_id.to_string(), + (Some("MiXeD-Slug".to_string()), "value".to_string()), + ); + Ok::<_, DatabaseError>(values) + }, + ) + .await + })); + } + + for task in tasks { + let values = task.await.unwrap().unwrap(); + assert_eq!(values.get(canonical_id), Some(&"value".to_string())); + } + assert_eq!(fetch_count.load(Ordering::SeqCst), 1); +} + +/// Protects the cluster-aware blocking pool and Pub/Sub seed rotation, which +/// use separate connection paths from ordinary cache commands. +#[actix_rt::test] +async fn blocking_queue_and_pubsub_work_with_cluster_connections() { + let pool = isolated_redis_pool("blocking_and_pubsub").await; + let queue_key = pool.key().entity("blocking", "queue"); + let mut connection = pool.connect().await.unwrap(); + connection + .lpush(&queue_key, b"queued-value".as_slice()) + .await + .unwrap(); + let popped = pool + .brpop(&queue_key, Duration::from_secs(1)) + .await + .unwrap() + .expect("BRPOP did not receive the queued value"); + assert_eq!(popped[0], queue_key.as_bytes()); + assert_eq!(popped[1], b"queued-value"); + + // An unreachable first seed verifies that the reconnect loop advances to + // another cluster seed instead of abandoning the subscription. + let valid_seed = labrinth::env::ENV + .REDIS_URL + .split(',') + .next() + .unwrap() + .to_string(); + let channel: &'static str = Box::leak( + format!("redis-test-pubsub-{}", Uuid::new_v4()).into_boxed_str(), + ); + let mut receiver = RedisPool::subscribe_with_seed_urls( + vec!["redis://127.0.0.1:1".to_string(), valid_seed], + channel, + ); + + let mut delivered = false; + for _ in 0..20 { + pool.publish(channel, b"cluster-message".as_slice()) + .await + .unwrap(); + if let Ok(Some(message)) = + timeout(Duration::from_millis(100), receiver.recv()).await + { + assert_eq!(message, b"cluster-message"); + delivered = true; + break; + } + } + assert!(delivered, "Pub/Sub did not recover from the invalid seed"); +} + +/// Protects the real many-get routes as one end-to-end cluster scenario: +/// partial cache hits, IDs and case-insensitive slugs, duplicates, misses, +/// invalidation, file hashes, dependency caches, and visibility filtering. +#[actix_rt::test] +async fn many_get_routes_handle_cross_slot_cache_lifecycle() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let alpha = &test_env.dummy.project_alpha; + let beta = &test_env.dummy.project_beta; + let alpha_project_id = parse_base62(&alpha.project_id).unwrap(); + let beta_project_id = parse_base62(&beta.project_id).unwrap(); + let alpha_version_id = parse_base62(&alpha.version_id).unwrap(); + let beta_version_id = parse_base62(&beta.version_id).unwrap(); + let missing_project_id = "R3d1sPj8"; + let missing_version_id = "R3d1sVn8"; + + // Clearing known dummy-data entries makes the first single read a + // deliberate warm hit and the other entities deliberate cold hits. + let mut redis = test_env.db.redis_pool.connect().await.unwrap(); + let cache_keys = vec![ + redis.key().entity(PROJECTS_NAMESPACE, alpha_project_id), + redis.key().entity(PROJECTS_NAMESPACE, beta_project_id), + redis.key().entity( + PROJECTS_SLUGS_NAMESPACE, + alpha.project_slug.to_lowercase(), + ), + redis.key().entity( + PROJECTS_SLUGS_NAMESPACE, + beta.project_slug.to_lowercase(), + ), + redis.key().entity(VERSIONS_NAMESPACE, alpha_version_id), + redis.key().entity(VERSIONS_NAMESPACE, beta_version_id), + redis.key().entity( + "versions_files:v1", + format!("sha1_{}", alpha.file_hash), + ), + redis.key().entity( + "versions_files:v1", + format!("sha1_{}", beta.file_hash), + ), + ]; + assert!( + cache_keys + .iter() + .map(Slot::for_key) + .collect::>() + .len() + > 1, + "route cache keys must exercise multiple slots" + ); + redis.delete_many(&cache_keys).await.unwrap(); + + api.get_project_deserialized_common( + &alpha.project_id, + USER_USER_PAT, + ) + .await; + let project_cache_keys = vec![ + redis.key().entity(PROJECTS_NAMESPACE, alpha_project_id), + redis.key().entity(PROJECTS_NAMESPACE, beta_project_id), + ]; + let cached_projects = + redis.get_many(&project_cache_keys).await.unwrap(); + assert!(cached_projects[0].is_some()); + assert!(cached_projects[1].is_none()); + + let uppercase_beta_slug = beta.project_slug.to_uppercase(); + let project_request = [ + alpha.project_id.as_str(), + uppercase_beta_slug.as_str(), + missing_project_id, + alpha.project_slug.as_str(), + alpha.project_id.as_str(), + ]; + for _ in 0..2 { + let response = + api.get_projects(&project_request, USER_USER_PAT).await; + assert_status!(&response, StatusCode::OK); + let projects: Vec = + test::read_body_json(response).await; + let project_ids = projects + .iter() + .map(|project| project.id.to_string()) + .collect::>(); + assert_eq!(projects.len(), 2); + assert_eq!( + project_ids, + HashSet::from([ + alpha.project_id.clone(), + beta.project_id.clone(), + ]) + ); + } + + api.get_version_deserialized_common( + &alpha.version_id, + USER_USER_PAT, + ) + .await; + let versions = api + .get_versions_deserialized_common( + vec![ + alpha.version_id.clone(), + beta.version_id.clone(), + missing_version_id.to_string(), + alpha.version_id.clone(), + ], + USER_USER_PAT, + ) + .await; + assert_eq!( + versions + .iter() + .map(|version| version.id.to_string()) + .collect::>(), + HashSet::from([ + alpha.version_id.clone(), + beta.version_id.clone(), + ]) + ); + + let hashes = [ + alpha.file_hash.as_str(), + beta.file_hash.as_str(), + "missing-hash", + alpha.file_hash.as_str(), + ]; + let versions_by_hash = api + .get_versions_from_hashes_deserialized_common( + &hashes, + "sha1", + USER_USER_PAT, + ) + .await; + assert_eq!(versions_by_hash.len(), 2); + + // Warming dependencies before the edit ensures the assertion below + // depends on invalidating the project's separate dependency slot. + let dependencies = api + .get_project_dependencies(&alpha.project_id, USER_USER_PAT) + .await; + assert_status!(&dependencies, StatusCode::OK); + let dependencies: serde_json::Value = + test::read_body_json(dependencies).await; + assert!(dependencies["projects"].as_array().unwrap().is_empty()); + + let new_slug = format!("cluster-cache-{}", Uuid::new_v4().simple()); + let response = api + .edit_project( + &alpha.project_id, + json!({ "slug": new_slug }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + assert_status!( + &api.get_project(&alpha.project_slug, USER_USER_PAT).await, + StatusCode::NOT_FOUND + ); + assert_status!( + &api.get_project(&new_slug, USER_USER_PAT).await, + StatusCode::OK + ); + + let updated_version_name = "cluster cache invalidation"; + let response = api + .edit_version( + &alpha.version_id, + json!({ + "name": updated_version_name, + "dependencies": [{ + "project_id": beta.project_id, + "dependency_type": "required", + "file_name": "cluster-test.jar", + }], + }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let updated_versions: Vec = api + .get_versions_deserialized_common( + vec![alpha.version_id.clone(), beta.version_id.clone()], + USER_USER_PAT, + ) + .await; + assert_eq!( + updated_versions + .iter() + .find(|version| version.id.to_string() == alpha.version_id) + .unwrap() + .name, + updated_version_name + ); + let updated_by_hash = api + .get_versions_from_hashes_deserialized_common( + &[alpha.file_hash.as_str(), beta.file_hash.as_str()], + "sha1", + USER_USER_PAT, + ) + .await; + assert_eq!( + updated_by_hash[&alpha.file_hash].name, + updated_version_name + ); + + let dependencies = api + .get_project_dependencies(&alpha.project_id, USER_USER_PAT) + .await; + assert_status!(&dependencies, StatusCode::OK); + let dependencies: serde_json::Value = + test::read_body_json(dependencies).await; + assert!( + dependencies["projects"] + .as_array() + .unwrap() + .iter() + .any(|project| project["id"] == beta.project_id) + ); + + // Visibility is evaluated after cache resolution, so a warm private + // project must not leak through a many-get to another user. + let response = api + .get_projects( + &[alpha.project_id.as_str(), beta.project_id.as_str()], + ENEMY_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::OK); + let visible_projects: Vec = + test::read_body_json(response).await; + assert_eq!(visible_projects.len(), 1); + assert_eq!(visible_projects[0].id.to_string(), alpha.project_id); + }, + ) + .await; +} From d2c50e86c53b4dc71944691841646a342adeb57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 11:47:42 -0400 Subject: [PATCH 05/12] fix(labrinth): clippy --- apps/labrinth/src/database/redis/cache/locking/distributed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/labrinth/src/database/redis/cache/locking/distributed.rs b/apps/labrinth/src/database/redis/cache/locking/distributed.rs index b5ccfea686..2cc9f3368a 100644 --- a/apps/labrinth/src/database/redis/cache/locking/distributed.rs +++ b/apps/labrinth/src/database/redis/cache/locking/distributed.rs @@ -159,7 +159,7 @@ impl DistributedLockManager { cancellation_token: CancellationToken, ) { let manager = self.clone(); - let _ = tokio::spawn(async move { + tokio::spawn(async move { loop { let result = tokio::select! { biased; From 2f9be03b067e92eefad3c517b17e95487b76bcef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 12:09:06 -0400 Subject: [PATCH 06/12] chore(labrinth): cleanup env, remove postcard support --- apps/labrinth/src/database/redis/blocking.rs | 6 +-- apps/labrinth/src/database/redis/cache.rs | 4 -- apps/labrinth/src/database/redis/config.rs | 22 ++++---- apps/labrinth/src/database/redis/key.rs | 19 ++++--- apps/labrinth/src/database/redis/mod.rs | 2 +- apps/labrinth/src/env.rs | 54 ++++++++++++++------ apps/labrinth/tests/redis.rs | 4 +- 7 files changed, 67 insertions(+), 44 deletions(-) diff --git a/apps/labrinth/src/database/redis/blocking.rs b/apps/labrinth/src/database/redis/blocking.rs index 768c106fba..c8097d0fa2 100644 --- a/apps/labrinth/src/database/redis/blocking.rs +++ b/apps/labrinth/src/database/redis/blocking.rs @@ -5,7 +5,7 @@ use prometheus::Registry; use crate::database::models::DatabaseError; use super::RedisPool; -use super::config::{RedisConfig, RedisMode}; +use super::config::{RedisConfig, RedisTopology}; use super::connection::RedisBackendBuildError; use super::metrics::{ LogicalPoolStatus, LogicalPoolStatusProvider, @@ -34,7 +34,7 @@ impl RedisBlockingPool { ) -> Result { let pool_size = config.blocking_pool_size(); let inner = match config.mode() { - RedisMode::Standalone => { + RedisTopology::Standalone => { let connection_config = redis::AsyncConnectionConfig::new() .set_connection_timeout(None) .set_response_timeout(None); @@ -52,7 +52,7 @@ impl RedisBlockingPool { retain_standalone_pool(pool.clone()); RedisBlockingPoolInner::Standalone(pool) } - RedisMode::Cluster => { + RedisTopology::Cluster => { let manager = deadpool_redis::cluster::Manager::new( config.seed_urls().to_vec(), false, diff --git a/apps/labrinth/src/database/redis/cache.rs b/apps/labrinth/src/database/redis/cache.rs index 9b75704629..76f0f0b871 100644 --- a/apps/labrinth/src/database/redis/cache.rs +++ b/apps/labrinth/src/database/redis/cache.rs @@ -53,7 +53,6 @@ pub enum Codec { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EncodingFormat { Json, - Postcard, } #[derive(Debug, Error)] @@ -93,7 +92,6 @@ impl FromStr for EncodingFormat { fn from_str(value: &str) -> Result { match value { "json" => Ok(Self::Json), - "postcard" => Ok(Self::Postcard), _ => Err(InvalidEncodingFormat), } } @@ -115,7 +113,6 @@ impl CacheSettings { ) -> Result, DatabaseError> { let mut value = match self.encoding_format { EncodingFormat::Json => serde_json::to_vec(value)?, - EncodingFormat::Postcard => postcard::to_allocvec(value)?, }; if self.compression_level > 0 @@ -156,7 +153,6 @@ impl CacheSettings { match self.encoding_format { EncodingFormat::Json => serde_json::from_slice(&value).ok(), - EncodingFormat::Postcard => postcard::from_bytes(&value).ok(), } } } diff --git a/apps/labrinth/src/database/redis/config.rs b/apps/labrinth/src/database/redis/config.rs index 780d2c1b35..5a4ccb25f5 100644 --- a/apps/labrinth/src/database/redis/config.rs +++ b/apps/labrinth/src/database/redis/config.rs @@ -42,16 +42,16 @@ impl FromStr for CacheLockingStrategy { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RedisMode { +pub enum RedisTopology { Standalone, Cluster, } #[derive(Debug, Error)] -#[error("invalid Redis mode; expected `standalone` or `cluster`")] +#[error("invalid Redis topology; expected `standalone` or `cluster`")] pub struct InvalidRedisMode; -impl FromStr for RedisMode { +impl FromStr for RedisTopology { type Err = InvalidRedisMode; fn from_str(value: &str) -> Result { @@ -122,7 +122,7 @@ pub(super) enum RedisBackendConfig { #[derive(Debug, Clone)] pub(super) struct RedisConfig { - mode: RedisMode, + mode: RedisTopology, backend: RedisBackendConfig, seed_urls: Vec, wait_timeout_ms: u64, @@ -140,7 +140,7 @@ pub(super) enum RedisConfigError { "unsupported Redis configuration: `{mode:?}` mode with `{connection_type:?}` connections" )] UnsupportedConnectionType { - mode: RedisMode, + mode: RedisTopology, connection_type: RedisConnectionType, }, #[error( @@ -156,7 +156,7 @@ pub(super) enum RedisConfigError { impl RedisConfig { pub(super) fn from_env() -> Result { Self::new( - ENV.REDIS_MODE, + ENV.REDIS_TOPOLOGY, ENV.REDIS_CONNECTION_TYPE, &ENV.REDIS_URL, ENV.REDIS_WAIT_TIMEOUT_MS, @@ -178,7 +178,7 @@ impl RedisConfig { } fn new( - mode: RedisMode, + mode: RedisTopology, connection_type: RedisConnectionType, raw_urls: &str, wait_timeout_ms: u64, @@ -199,7 +199,7 @@ impl RedisConfig { } let backend = match (mode, connection_type) { - (RedisMode::Standalone, RedisConnectionType::Pooled) => { + (RedisTopology::Standalone, RedisConnectionType::Pooled) => { if seed_urls.len() != 1 { return Err(RedisConfigError::MultipleStandaloneUrls); } @@ -209,14 +209,14 @@ impl RedisConfig { standalone_pool_size.1, )?) } - (RedisMode::Cluster, RedisConnectionType::Pooled) => { + (RedisTopology::Cluster, RedisConnectionType::Pooled) => { RedisBackendConfig::ClusterPooled(RedisPoolSize::new( "cluster", cluster_pool_size.0, cluster_pool_size.1, )?) } - (RedisMode::Cluster, RedisConnectionType::Multiplexed) => { + (RedisTopology::Cluster, RedisConnectionType::Multiplexed) => { RedisBackendConfig::ClusterMultiplexed } (mode, connection_type) => { @@ -237,7 +237,7 @@ impl RedisConfig { }) } - pub(super) fn mode(&self) -> RedisMode { + pub(super) fn mode(&self) -> RedisTopology { self.mode } diff --git a/apps/labrinth/src/database/redis/key.rs b/apps/labrinth/src/database/redis/key.rs index ddbc9f6ec1..936ba3febd 100644 --- a/apps/labrinth/src/database/redis/key.rs +++ b/apps/labrinth/src/database/redis/key.rs @@ -1,16 +1,19 @@ use std::fmt::Display; use std::sync::Arc; -use super::RedisMode; +use super::RedisTopology; #[derive(Debug, Clone)] pub struct KeyBuilder { meta_namespace: Arc, - mode: RedisMode, + mode: RedisTopology, } impl KeyBuilder { - pub fn new(meta_namespace: impl Into>, mode: RedisMode) -> Self { + pub fn new( + meta_namespace: impl Into>, + mode: RedisTopology, + ) -> Self { Self { meta_namespace: meta_namespace.into(), mode, @@ -40,10 +43,10 @@ impl KeyBuilder { slot_tag: impl Display, ) -> String { match self.mode { - RedisMode::Standalone => { + RedisTopology::Standalone => { format!("{}_{}:{}", self.meta_namespace, namespace, logical_key) } - RedisMode::Cluster => format!( + RedisTopology::Cluster => format!( "{}_{}:{{{}}}:{}", self.meta_namespace, namespace, @@ -74,11 +77,11 @@ fn escape_slot_tag(value: &str) -> String { #[cfg(test)] mod tests { use super::KeyBuilder; - use crate::database::redis::RedisMode; + use crate::database::redis::RedisTopology; #[test] fn cluster_slot_tags_escape_braces_and_percent_signs() { - let builder = KeyBuilder::new("labrinth", RedisMode::Cluster); + let builder = KeyBuilder::new("labrinth", RedisTopology::Cluster); let entity = builder.entity("projects", "a{%}b"); assert!(entity.contains("{a%7B%25%7Db}")); @@ -86,7 +89,7 @@ mod tests { #[test] fn cluster_empty_slot_tag_is_nonempty() { - let builder = KeyBuilder::new("labrinth", RedisMode::Cluster); + let builder = KeyBuilder::new("labrinth", RedisTopology::Cluster); let entity = builder.entity("projects", ""); assert!(entity.contains("{%00}")); diff --git a/apps/labrinth/src/database/redis/mod.rs b/apps/labrinth/src/database/redis/mod.rs index 9b057faf70..2179fba109 100644 --- a/apps/labrinth/src/database/redis/mod.rs +++ b/apps/labrinth/src/database/redis/mod.rs @@ -26,7 +26,7 @@ mod util; use cache::{CacheManager, CacheSettings, ConnectionProvider}; pub use cache::{Codec, EncodingFormat, RedisValue}; use config::RedisConfig; -pub use config::{CacheLockingStrategy, RedisConnectionType, RedisMode}; +pub use config::{CacheLockingStrategy, RedisConnectionType, RedisTopology}; use connection::RedisBackend; pub use key::KeyBuilder; diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index f6b975fab0..87f80a58ce 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -5,6 +5,10 @@ use eyre::{Context, eyre}; use rust_decimal::Decimal; use serde::de::DeserializeOwned; +use crate::database::redis::{ + CacheLockingStrategy, RedisConnectionType, RedisTopology, +}; + macro_rules! vars { ( $( @@ -129,10 +133,42 @@ vars! { LABRINTH_EXTERNAL_NOTIFICATION_KEY: String = ""; RATE_LIMIT_IGNORE_KEY: String = ""; DATABASE_URL: String = "postgresql://labrinth:labrinth@localhost/labrinth"; - REDIS_MODE: crate::database::redis::RedisMode = crate::database::redis::RedisMode::Standalone; - REDIS_CONNECTION_TYPE: crate::database::redis::RedisConnectionType = crate::database::redis::RedisConnectionType::Pooled; - REDIS_CACHE_LOCKING_STRATEGY: crate::database::redis::CacheLockingStrategy = crate::database::redis::CacheLockingStrategy::Local; + + // Redis + REDIS_TOPOLOGY: RedisTopology = RedisTopology::Standalone; + REDIS_CONNECTION_TYPE: RedisConnectionType = RedisConnectionType::Pooled; + REDIS_CACHE_LOCKING_STRATEGY: CacheLockingStrategy = CacheLockingStrategy::Local; + // URL(s) for Redis. Use comma-separated values for multiple URLs in Cluster topology. REDIS_URL: String = "redis://localhost"; + + // Configures the waiting timeout for Redis connection *pools*. + // This doesn't affect the bulk of Redis work in Multiplexed connection type. + REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64; + + // Minimum and maximum number of connections when Redis is in Standalone topology. + REDIS_MAX_CONNECTIONS: u32 = 2048u32; + REDIS_MIN_CONNECTIONS: usize = 0usize; + + // Minimum and maximum number of connections when Redis is in Cluster topology, Pooled connection type. + REDIS_CLUSTER_MAX_CONNECTIONS: u32 = 16u32; + REDIS_CLUSTER_MIN_CONNECTIONS: usize = 0usize; + + // The maximum number of connections of the Redis blocking pool. There's a blocking pool regardless of topology + // and main connection type. + REDIS_BLOCKING_MAX_CONNECTIONS: u32 = 256u32; + + // The encoding format used for Redis cache values. + REDIS_ENCODING_FORMAT: crate::database::redis::EncodingFormat = crate::database::redis::EncodingFormat::Json; + // The level of LZ4 compression used for Redis cache values. A value of 0 disables compression (supports 1-12) + REDIS_COMPRESSION_LEVEL: i32 = 0i32; + // The compression algorithm used for Redis cache values. Currently only LZ4 is supported. + REDIS_COMPRESSION_ALGORITHM: crate::database::redis::Codec = crate::database::redis::Codec::Lz4; + // The minimum number of bytes required to trigger compression for Redis cache values. + REDIS_COMPRESSION_THRESHOLD_BYTES: usize = 1024usize; + // The minimum savings ratio required to trigger compression for Redis cache values. If the savings ratio is lower than this, + // the compressed payload is discarded and the original payload is stored as-is. + REDIS_COMPRESSION_MIN_SAVINGS_RATIO: f64 = 12.5f64; + KAFKA_BOOTSTRAP_SERVERS: StringCsv = StringCsv(vec!["localhost:19092".into()]); KAFKA_CLIENT_ID: String = "labrinth"; BIND_ADDR: String = ""; @@ -289,18 +325,6 @@ vars! { READONLY_DATABASE_MIN_CONNECTIONS: u32 = 0u32; READONLY_DATABASE_MAX_CONNECTIONS: u32 = 1u32; - REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64; - REDIS_MAX_CONNECTIONS: u32 = 10000u32; - REDIS_MIN_CONNECTIONS: usize = 0usize; - REDIS_CLUSTER_MAX_CONNECTIONS: u32 = 16u32; - REDIS_CLUSTER_MIN_CONNECTIONS: usize = 0usize; - REDIS_BLOCKING_MAX_CONNECTIONS: u32 = 8u32; - REDIS_ENCODING_FORMAT: crate::database::redis::EncodingFormat = crate::database::redis::EncodingFormat::Json; - REDIS_COMPRESSION_LEVEL: i32 = 0i32; - REDIS_COMPRESSION_ALGORITHM: crate::database::redis::Codec = crate::database::redis::Codec::Lz4; - REDIS_COMPRESSION_THRESHOLD_BYTES: usize = 1024usize; - REDIS_COMPRESSION_MIN_SAVINGS_RATIO: f64 = 12.5f64; - SEARCH_OPERATION_TIMEOUT: u64 = 300000u64; SMTP_REPLY_TO_NAME: String = ""; diff --git a/apps/labrinth/tests/redis.rs b/apps/labrinth/tests/redis.rs index 9f2af478a0..21a92fa212 100644 --- a/apps/labrinth/tests/redis.rs +++ b/apps/labrinth/tests/redis.rs @@ -16,7 +16,7 @@ use labrinth::database::models::project_item::{ PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, }; use labrinth::database::models::version_item::VERSIONS_NAMESPACE; -use labrinth::database::redis::{KeyBuilder, RedisMode, RedisPool}; +use labrinth::database::redis::{KeyBuilder, RedisPool, RedisTopology}; use redis::cluster_routing::Slot; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -34,7 +34,7 @@ async fn isolated_redis_pool(label: &str) -> RedisPool { fn clustered_key_builder(label: &str) -> KeyBuilder { KeyBuilder::new( format!("redis_test_{label}_{}", Uuid::new_v4()), - RedisMode::Cluster, + RedisTopology::Cluster, ) } From 18ba121c9c85a6cd1e24dc74f8b36b8477571687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 12:22:37 -0400 Subject: [PATCH 07/12] chore(ci): fix test env for labrinth --- .github/workflows/turbo-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/turbo-ci.yml b/.github/workflows/turbo-ci.yml index e5baaa800e..664ad4c0c4 100644 --- a/.github/workflows/turbo-ci.yml +++ b/.github/workflows/turbo-ci.yml @@ -83,7 +83,7 @@ jobs: SCCACHE_CACHE_SIZE: ${{ needs.skip-if-clean.outputs.internal == 'true' && '10G' || '' }} RUSTC_WRAPPER: ${{ needs.skip-if-clean.outputs.internal == 'true' && 'sccache' || '' }} # Redis config for tests - REDIS_MODE: cluster + REDIS_TOPOLOGY: cluster REDIS_CONNECTION_TYPE: multiplexed REDIS_URL: redis://127.0.0.1:7000,redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003,redis://127.0.0.1:7004,redis://127.0.0.1:7005 # Avoid stack overflows in tests @@ -214,7 +214,7 @@ jobs: - name: Test pooled standalone Redis backend if: steps.check-labrinth.outputs.needs_services == 'true' env: - REDIS_MODE: standalone + REDIS_TOPOLOGY: standalone REDIS_CONNECTION_TYPE: pooled REDIS_URL: redis://127.0.0.1:6379 run: cargo nextest run --package labrinth --test redis --no-fail-fast From 52d3114dbe14a01e39a94cd93e8f248c34241f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 17:58:41 -0400 Subject: [PATCH 08/12] chore(labrinth): bump all key versions --- .../database/models/analytics_event_item.rs | 2 +- .../src/database/models/categories.rs | 2 +- .../src/database/models/collection_item.rs | 2 +- .../labrinth/src/database/models/flow_item.rs | 2 +- .../src/database/models/image_item.rs | 2 +- .../src/database/models/loader_fields.rs | 14 +++++----- .../database/models/moderation_note_item.rs | 4 +-- .../src/database/models/notification_item.rs | 2 +- .../models/notifications_template_item.rs | 6 ++--- .../models/notifications_type_item.rs | 2 +- .../src/database/models/organization_item.rs | 4 +-- apps/labrinth/src/database/models/pat_item.rs | 6 ++--- .../src/database/models/product_item.rs | 2 +- .../src/database/models/project_item.rs | 6 ++--- .../src/database/models/session_item.rs | 6 ++--- .../labrinth/src/database/models/team_item.rs | 2 +- .../labrinth/src/database/models/user_item.rs | 6 ++--- .../src/database/models/version_item.rs | 4 +-- apps/labrinth/src/queue/analytics/cache.rs | 2 +- apps/labrinth/src/queue/analytics/mod.rs | 6 ++--- apps/labrinth/src/queue/server_ping.rs | 4 +-- apps/labrinth/src/routes/internal/campaign.rs | 2 +- apps/labrinth/src/routes/internal/flows.rs | 2 +- apps/labrinth/src/routes/v3/content/mod.rs | 6 ++--- apps/labrinth/src/sync/friends.rs | 2 +- apps/labrinth/src/sync/status.rs | 2 +- apps/labrinth/src/util/gotenberg.rs | 2 +- apps/labrinth/src/util/ratelimit.rs | 2 +- apps/labrinth/tests/redis.rs | 26 +++++++++---------- 29 files changed, 65 insertions(+), 65 deletions(-) diff --git a/apps/labrinth/src/database/models/analytics_event_item.rs b/apps/labrinth/src/database/models/analytics_event_item.rs index ab8c4b9e2e..e07854f159 100644 --- a/apps/labrinth/src/database/models/analytics_event_item.rs +++ b/apps/labrinth/src/database/models/analytics_event_item.rs @@ -11,7 +11,7 @@ use crate::{ }; use serde::{Deserialize, Serialize}; -const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v1"; +const ANALYTICS_EVENTS_NAMESPACE: &str = "analytics_events:v3"; const ANALYTICS_EVENTS_ALL_KEY: &str = "all"; #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/apps/labrinth/src/database/models/categories.rs b/apps/labrinth/src/database/models/categories.rs index 9e3012e99a..35962b9856 100644 --- a/apps/labrinth/src/database/models/categories.rs +++ b/apps/labrinth/src/database/models/categories.rs @@ -7,7 +7,7 @@ use super::ids::*; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; -const TAGS_NAMESPACE: &str = "tags:v1"; +const TAGS_NAMESPACE: &str = "tags:v3"; pub struct ProjectType { pub id: ProjectTypeId, diff --git a/apps/labrinth/src/database/models/collection_item.rs b/apps/labrinth/src/database/models/collection_item.rs index ae325b0a8a..38c6005364 100644 --- a/apps/labrinth/src/database/models/collection_item.rs +++ b/apps/labrinth/src/database/models/collection_item.rs @@ -8,7 +8,7 @@ use dashmap::DashMap; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; -const COLLECTIONS_NAMESPACE: &str = "collections:v1"; +const COLLECTIONS_NAMESPACE: &str = "collections:v3"; #[derive(Clone)] pub struct CollectionBuilder { diff --git a/apps/labrinth/src/database/models/flow_item.rs b/apps/labrinth/src/database/models/flow_item.rs index 17d7b894e4..bea2db3167 100644 --- a/apps/labrinth/src/database/models/flow_item.rs +++ b/apps/labrinth/src/database/models/flow_item.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use webauthn_rs::prelude::{DiscoverableAuthentication, PasskeyRegistration}; -const FLOWS_NAMESPACE: &str = "flows:v1"; +const FLOWS_NAMESPACE: &str = "flows:v3"; #[derive(Deserialize, Serialize)] pub enum DBFlow { diff --git a/apps/labrinth/src/database/models/image_item.rs b/apps/labrinth/src/database/models/image_item.rs index bca6399f3b..967347fb4a 100644 --- a/apps/labrinth/src/database/models/image_item.rs +++ b/apps/labrinth/src/database/models/image_item.rs @@ -6,7 +6,7 @@ use chrono::{DateTime, Utc}; use dashmap::DashMap; use serde::{Deserialize, Serialize}; -const IMAGES_NAMESPACE: &str = "images:v1"; +const IMAGES_NAMESPACE: &str = "images:v3"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DBImage { diff --git a/apps/labrinth/src/database/models/loader_fields.rs b/apps/labrinth/src/database/models/loader_fields.rs index 1f011d8872..ec5841f85e 100644 --- a/apps/labrinth/src/database/models/loader_fields.rs +++ b/apps/labrinth/src/database/models/loader_fields.rs @@ -12,14 +12,14 @@ use futures::TryStreamExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; -const GAMES_LIST_NAMESPACE: &str = "games:v1"; -const LOADER_ID: &str = "loader_id:v1"; -const LOADERS_LIST_NAMESPACE: &str = "loaders:v1"; -const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v1"; -const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v1"; -const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v1"; +const GAMES_LIST_NAMESPACE: &str = "games:v3"; +const LOADER_ID: &str = "loader_id:v3"; +const LOADERS_LIST_NAMESPACE: &str = "loaders:v3"; +const LOADER_FIELDS_NAMESPACE: &str = "loader_fields:v3"; +const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all:v3"; +const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums:v3"; pub const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = - "loader_field_enum_values:v1"; + "loader_field_enum_values:v3"; #[derive(Clone, Serialize, Deserialize, Debug)] pub struct Game { diff --git a/apps/labrinth/src/database/models/moderation_note_item.rs b/apps/labrinth/src/database/models/moderation_note_item.rs index 656d8cffc8..8a66bdf553 100644 --- a/apps/labrinth/src/database/models/moderation_note_item.rs +++ b/apps/labrinth/src/database/models/moderation_note_item.rs @@ -7,9 +7,9 @@ use crate::database::redis::RedisPool; use super::{DBOrganizationId, DBUserId, DatabaseError}; -const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v1"; +const MODERATION_NOTES_USERS_NAMESPACE: &str = "moderation_notes_users:v3"; const MODERATION_NOTES_ORGANIZATIONS_NAMESPACE: &str = - "moderation_notes_organizations:v1"; + "moderation_notes_organizations:v3"; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DBModerationNote { diff --git a/apps/labrinth/src/database/models/notification_item.rs b/apps/labrinth/src/database/models/notification_item.rs index 7a6e921ebc..b729f2decb 100644 --- a/apps/labrinth/src/database/models/notification_item.rs +++ b/apps/labrinth/src/database/models/notification_item.rs @@ -9,7 +9,7 @@ use chrono::{DateTime, Utc}; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; -const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v1"; +const USER_NOTIFICATIONS_NAMESPACE: &str = "user_notifications:v3"; pub struct NotificationBuilder { pub body: NotificationBody, diff --git a/apps/labrinth/src/database/models/notifications_template_item.rs b/apps/labrinth/src/database/models/notifications_template_item.rs index 2c8f160cf0..0e07730a39 100644 --- a/apps/labrinth/src/database/models/notifications_template_item.rs +++ b/apps/labrinth/src/database/models/notifications_template_item.rs @@ -4,11 +4,11 @@ use crate::models::v3::notifications::{NotificationChannel, NotificationType}; use crate::routes::ApiError; use serde::{Deserialize, Serialize}; -const TEMPLATES_NAMESPACE: &str = "notifications_templates:v1"; +const TEMPLATES_NAMESPACE: &str = "notifications_templates:v3"; const TEMPLATES_HTML_DATA_NAMESPACE: &str = - "notifications_templates_html_data:v1"; + "notifications_templates_html_data:v3"; const TEMPLATES_DYNAMIC_HTML_NAMESPACE: &str = - "notifications_templates_dynamic_html:v1"; + "notifications_templates_dynamic_html:v3"; const HTML_DATA_CACHE_EXPIRY: i64 = 60 * 15; // 15 minutes const TEMPLATES_CACHE_EXPIRY: i64 = 60 * 30; // 30 minutes diff --git a/apps/labrinth/src/database/models/notifications_type_item.rs b/apps/labrinth/src/database/models/notifications_type_item.rs index 77e40db0a7..943d6b7a11 100644 --- a/apps/labrinth/src/database/models/notifications_type_item.rs +++ b/apps/labrinth/src/database/models/notifications_type_item.rs @@ -3,7 +3,7 @@ use crate::database::redis::RedisPool; use crate::models::v3::notifications::NotificationType; use serde::{Deserialize, Serialize}; -const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v1"; +const NOTIFICATION_TYPES_NAMESPACE: &str = "notification_types:v3"; #[derive(Serialize, Deserialize)] pub struct NotificationTypeItem { diff --git a/apps/labrinth/src/database/models/organization_item.rs b/apps/labrinth/src/database/models/organization_item.rs index 15a00998d0..b7bd77b0f9 100644 --- a/apps/labrinth/src/database/models/organization_item.rs +++ b/apps/labrinth/src/database/models/organization_item.rs @@ -9,8 +9,8 @@ use std::hash::Hash; use super::{DBTeamMember, ids::*}; use serde::{Deserialize, Serialize}; -const ORGANIZATIONS_NAMESPACE: &str = "organizations:v1"; -const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v1"; +const ORGANIZATIONS_NAMESPACE: &str = "organizations:v3"; +const ORGANIZATIONS_TITLES_NAMESPACE: &str = "organizations_titles:v3"; #[derive(Deserialize, Serialize, Clone, Debug)] /// An organization of users who together control one or more projects and organizations. diff --git a/apps/labrinth/src/database/models/pat_item.rs b/apps/labrinth/src/database/models/pat_item.rs index 417fa82ff1..755d4c4af0 100644 --- a/apps/labrinth/src/database/models/pat_item.rs +++ b/apps/labrinth/src/database/models/pat_item.rs @@ -11,9 +11,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -const PATS_NAMESPACE: &str = "pats:v1"; -const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v1"; -const PATS_USERS_NAMESPACE: &str = "pats_users:v1"; +const PATS_NAMESPACE: &str = "pats:v3"; +const PATS_TOKENS_NAMESPACE: &str = "pats_tokens:v3"; +const PATS_USERS_NAMESPACE: &str = "pats_users:v3"; #[derive(Deserialize, Serialize, Clone, Debug)] pub struct DBPersonalAccessToken { diff --git a/apps/labrinth/src/database/models/product_item.rs b/apps/labrinth/src/database/models/product_item.rs index a8bafc41ff..5e2aa9952d 100644 --- a/apps/labrinth/src/database/models/product_item.rs +++ b/apps/labrinth/src/database/models/product_item.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use std::convert::TryFrom; use std::convert::TryInto; -const PRODUCTS_NAMESPACE: &str = "products:v1"; +const PRODUCTS_NAMESPACE: &str = "products:v3"; pub struct DBProduct { pub id: DBProductId, diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs index 9c4c3f1b14..3c82c86b64 100644 --- a/apps/labrinth/src/database/models/project_item.rs +++ b/apps/labrinth/src/database/models/project_item.rs @@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -pub const PROJECTS_NAMESPACE: &str = "projects:v1"; -pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v1"; -const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v1"; +pub const PROJECTS_NAMESPACE: &str = "projects:v3"; +pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs:v3"; +const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies:v3"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct LinkUrl { diff --git a/apps/labrinth/src/database/models/session_item.rs b/apps/labrinth/src/database/models/session_item.rs index 5769ff4986..e07335e09f 100644 --- a/apps/labrinth/src/database/models/session_item.rs +++ b/apps/labrinth/src/database/models/session_item.rs @@ -10,9 +10,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -const SESSIONS_NAMESPACE: &str = "sessions:v1"; -const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v1"; -const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v1"; +const SESSIONS_NAMESPACE: &str = "sessions:v3"; +const SESSIONS_IDS_NAMESPACE: &str = "sessions_ids:v3"; +const SESSIONS_USERS_NAMESPACE: &str = "sessions_users:v3"; pub struct SessionBuilder { pub session: String, diff --git a/apps/labrinth/src/database/models/team_item.rs b/apps/labrinth/src/database/models/team_item.rs index 4220456247..0e0ce625bf 100644 --- a/apps/labrinth/src/database/models/team_item.rs +++ b/apps/labrinth/src/database/models/team_item.rs @@ -9,7 +9,7 @@ use itertools::Itertools; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; -const TEAMS_NAMESPACE: &str = "teams:v1"; +const TEAMS_NAMESPACE: &str = "teams:v3"; pub struct TeamBuilder { pub members: Vec, diff --git a/apps/labrinth/src/database/models/user_item.rs b/apps/labrinth/src/database/models/user_item.rs index 6daef10670..71af2e5592 100644 --- a/apps/labrinth/src/database/models/user_item.rs +++ b/apps/labrinth/src/database/models/user_item.rs @@ -16,9 +16,9 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::hash::Hash; -const USERS_NAMESPACE: &str = "users:v1"; -const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v1"; -const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v1"; +const USERS_NAMESPACE: &str = "users:v3"; +const USER_USERNAMES_NAMESPACE: &str = "users_usernames:v3"; +const USERS_PROJECTS_NAMESPACE: &str = "users_projects:v3"; #[derive(Deserialize, Serialize, Clone, Debug)] pub struct DBUser { diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs index 77917211b9..62fb391354 100644 --- a/apps/labrinth/src/database/models/version_item.rs +++ b/apps/labrinth/src/database/models/version_item.rs @@ -21,8 +21,8 @@ use std::cmp::Ordering; use std::collections::HashMap; use tracing::error; -pub const VERSIONS_NAMESPACE: &str = "versions:v1"; -const VERSION_FILES_NAMESPACE: &str = "versions_files:v1"; +pub const VERSIONS_NAMESPACE: &str = "versions:v3"; +const VERSION_FILES_NAMESPACE: &str = "versions_files:v3"; pub async fn cleanup_unused_attribution_files_and_groups( transaction: &mut PgTransaction<'_>, diff --git a/apps/labrinth/src/queue/analytics/cache.rs b/apps/labrinth/src/queue/analytics/cache.rs index bd0bc4fd7c..5603ee366c 100644 --- a/apps/labrinth/src/queue/analytics/cache.rs +++ b/apps/labrinth/src/queue/analytics/cache.rs @@ -13,7 +13,7 @@ use crate::{ util::error::Context, }; -pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v1"; +pub const MINECRAFT_SERVER_ANALYTICS: &str = "minecraft_server_analytics:v3"; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MinecraftServerAnalytics { diff --git a/apps/labrinth/src/queue/analytics/mod.rs b/apps/labrinth/src/queue/analytics/mod.rs index e7b867145e..8567425ea0 100644 --- a/apps/labrinth/src/queue/analytics/mod.rs +++ b/apps/labrinth/src/queue/analytics/mod.rs @@ -11,9 +11,9 @@ use tracing::trace; pub mod cache; -const DOWNLOADS_NAMESPACE: &str = "downloads:v1"; -const VIEWS_NAMESPACE: &str = "views:v1"; -const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v1"; +const DOWNLOADS_NAMESPACE: &str = "downloads:v3"; +const VIEWS_NAMESPACE: &str = "views:v3"; +const MINECRAFT_SERVER_PLAYS_NAMESPACE: &str = "minecraft_server_plays:v3"; const MINECRAFT_SERVER_PLAYS_EXPIRY: u64 = 86_400; // 24 hours const MINECRAFT_SERVER_PLAYS_LIMIT: u32 = 5; diff --git a/apps/labrinth/src/queue/server_ping.rs b/apps/labrinth/src/queue/server_ping.rs index b6bc7602e7..9a481a7ff0 100644 --- a/apps/labrinth/src/queue/server_ping.rs +++ b/apps/labrinth/src/queue/server_ping.rs @@ -26,9 +26,9 @@ pub struct ServerPingQueue { pub incremental_search_queue: IncrementalSearchQueue, } -pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v1"; +pub const REDIS_NAMESPACE: &str = "minecraft_java_server_ping:v3"; pub const REDIS_FAILURE_NAMESPACE: &str = - "minecraft_java_server_ping_failures:v1"; + "minecraft_java_server_ping_failures:v3"; pub const CLICKHOUSE_TABLE: &str = "minecraft_java_server_pings"; impl ServerPingQueue { diff --git a/apps/labrinth/src/routes/internal/campaign.rs b/apps/labrinth/src/routes/internal/campaign.rs index 848b1900a4..79735fc721 100644 --- a/apps/labrinth/src/routes/internal/campaign.rs +++ b/apps/labrinth/src/routes/internal/campaign.rs @@ -68,7 +68,7 @@ pub struct CampaignInfo { cached_at: DateTime, } -const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v1"; +const CAMPAIGN_INFO_CACHE_NAMESPACE: &str = "campaign_info:v3"; const CAMPAIGN_INFO_CACHE_STALE_SECONDS: i64 = 15 * 60; const CAMPAIGN_INFO_CACHE_TTL_SECONDS: i64 = 24 * 60 * 60; diff --git a/apps/labrinth/src/routes/internal/flows.rs b/apps/labrinth/src/routes/internal/flows.rs index 5171742c75..50034555b2 100644 --- a/apps/labrinth/src/routes/internal/flows.rs +++ b/apps/labrinth/src/routes/internal/flows.rs @@ -2126,7 +2126,7 @@ async fn validate_2fa_code( ) .map_err(|_| AuthenticationError::InvalidCredentials)?; - const TOTP_NAMESPACE: &str = "used_totp:v1"; + const TOTP_NAMESPACE: &str = "used_totp:v3"; let mut conn = redis.connect().await?; let logical_key = format!("{}-{}", input, user_id.0); let key = redis diff --git a/apps/labrinth/src/routes/v3/content/mod.rs b/apps/labrinth/src/routes/v3/content/mod.rs index 7b7807308f..80b91eeeac 100644 --- a/apps/labrinth/src/routes/v3/content/mod.rs +++ b/apps/labrinth/src/routes/v3/content/mod.rs @@ -22,9 +22,9 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; -const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v1"; -const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v1"; -const CONTENT_RESOLVE_CACHE_SCHEMA_VERSION: &str = "v1"; +const CONTENT_RESOLVE_CACHE_NAMESPACE: &str = "content_resolve:v3"; +const CONTENT_RESOLVE_CACHE_HEAT_NAMESPACE: &str = "content_resolve_heat:v3"; +const CONTENT_RESOLVE_CACHE_SCHEMA_VERSION: &str = "v3"; const CONTENT_RESOLVE_CACHE_HEAT_WINDOW_SECONDS: i64 = 60 * 60 * 24; pub fn config(cfg: &mut actix_web::web::ServiceConfig) { diff --git a/apps/labrinth/src/sync/friends.rs b/apps/labrinth/src/sync/friends.rs index e8e5fb47d4..a7327f38eb 100644 --- a/apps/labrinth/src/sync/friends.rs +++ b/apps/labrinth/src/sync/friends.rs @@ -14,7 +14,7 @@ use redis::{RedisWrite, ToRedisArgs, ToSingleRedisArg}; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; -pub const FRIENDS_CHANNEL_NAME: &str = "friends:v1"; +pub const FRIENDS_CHANNEL_NAME: &str = "friends:v3"; #[derive(Serialize, Deserialize)] pub enum RedisFriendsMessage { diff --git a/apps/labrinth/src/sync/status.rs b/apps/labrinth/src/sync/status.rs index 3e27094e0f..bc4d67e252 100644 --- a/apps/labrinth/src/sync/status.rs +++ b/apps/labrinth/src/sync/status.rs @@ -5,7 +5,7 @@ use ariadne::users::UserStatus; use redis::AsyncCommands; const EXPIRY_TIME_SECONDS: i64 = 60; -const USER_STATUS_NAMESPACE: &str = "user_status:v1"; +const USER_STATUS_NAMESPACE: &str = "user_status:v3"; pub async fn get_user_status( user: UserId, diff --git a/apps/labrinth/src/util/gotenberg.rs b/apps/labrinth/src/util/gotenberg.rs index 834bc50278..f45c525133 100644 --- a/apps/labrinth/src/util/gotenberg.rs +++ b/apps/labrinth/src/util/gotenberg.rs @@ -14,7 +14,7 @@ pub const MODRINTH_GENERATED_PDF_TYPE: HeaderName = HeaderName::from_static("modrinth-generated-pdf-type"); pub const MODRINTH_PAYMENT_ID: HeaderName = HeaderName::from_static("modrinth-payment-id"); -pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements:v1"; +pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements:v3"; const REDIS_TIMEOUT_MARGIN_MS: u64 = 250; pub(crate) fn payment_statement_key( diff --git a/apps/labrinth/src/util/ratelimit.rs b/apps/labrinth/src/util/ratelimit.rs index 5a54f8965c..bccb381c62 100644 --- a/apps/labrinth/src/util/ratelimit.rs +++ b/apps/labrinth/src/util/ratelimit.rs @@ -11,7 +11,7 @@ use chrono::Utc; use std::str::FromStr; use std::sync::Arc; -const RATE_LIMIT_NAMESPACE: &str = "rate_limit:v1"; +const RATE_LIMIT_NAMESPACE: &str = "rate_limit:v3"; const RATE_LIMIT_EXPIRY: i64 = 300; // 5 minutes const MINUTE_IN_NANOS: i64 = 60_000_000_000; diff --git a/apps/labrinth/tests/redis.rs b/apps/labrinth/tests/redis.rs index 21a92fa212..d3faf3220e 100644 --- a/apps/labrinth/tests/redis.rs +++ b/apps/labrinth/tests/redis.rs @@ -246,7 +246,7 @@ async fn cache_lock_coalesces_concurrent_misses_for_one_key() { tasks.push(tokio::spawn(async move { barrier.wait().await; pool.get_cached_keys_raw( - "single_flight:v1", + "single_flight:v3", &["shared".to_string()], move |keys| async move { fetch_count.fetch_add(1, Ordering::SeqCst); @@ -288,7 +288,7 @@ async fn cache_lock_coalesces_only_overlapping_keys() { tasks.push(tokio::spawn(async move { barrier.wait().await; pool.get_cached_keys_raw( - "overlapping_locks:v1", + "overlapping_locks:v3", &requested, move |keys| async move { tokio::time::sleep(Duration::from_millis(75)).await; @@ -329,7 +329,7 @@ async fn cache_lock_does_not_block_independent_keys() { let slow = tokio::spawn(async move { slow_pool .get_cached_keys_raw( - "independent_locks:v1", + "independent_locks:v3", &["slow".to_string()], move |keys| async move { slow_started.notify_one(); @@ -346,7 +346,7 @@ async fn cache_lock_does_not_block_independent_keys() { let fast = timeout( Duration::from_secs(1), pool.get_cached_keys_raw( - "independent_locks:v1", + "independent_locks:v3", &["fast".to_string()], |keys| async move { let values = DashMap::new(); @@ -375,7 +375,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() { let failed = pool .get_cached_keys_raw( - "error_recovery:v1", + "error_recovery:v3", &["key".to_string()], |_| async { Err::, _>(DatabaseError::Internal( @@ -389,7 +389,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() { let recovered = timeout( Duration::from_secs(1), pool.get_cached_keys_raw( - "error_recovery:v1", + "error_recovery:v3", &["key".to_string()], |keys| async move { let values = DashMap::new(); @@ -409,7 +409,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() { let cancelled = tokio::spawn(async move { cancelled_pool .get_cached_keys_raw( - "cancellation_recovery:v1", + "cancellation_recovery:v3", &["key".to_string()], move |_| async move { cancelled_started.notify_one(); @@ -428,7 +428,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() { let recovered = timeout( Duration::from_secs(1), pool.get_cached_keys_raw( - "cancellation_recovery:v1", + "cancellation_recovery:v3", &["key".to_string()], |keys| async move { let values = DashMap::new(); @@ -448,7 +448,7 @@ async fn cache_lock_is_released_after_error_and_cancellation() { #[actix_rt::test] async fn expired_cache_value_serves_waiter_while_writer_refreshes() { let pool = isolated_redis_pool("stale_while_revalidate").await; - let namespace = "stale_while_revalidate:v1"; + let namespace = "stale_while_revalidate:v3"; let logical_key = "key".to_string(); let mut connection = pool.connect().await.unwrap(); let redis_key = connection.key().entity(namespace, &logical_key); @@ -535,8 +535,8 @@ async fn case_insensitive_slug_requests_share_one_cache_lock() { let requested = vec![requested]; barrier.wait().await; pool.get_cached_keys_raw_with_slug( - "slug_values:v1", - Some("slug_aliases:v1"), + "slug_values:v3", + Some("slug_aliases:v3"), false, &requested, move |_| async move { @@ -647,11 +647,11 @@ async fn many_get_routes_handle_cross_slot_cache_lifecycle() { redis.key().entity(VERSIONS_NAMESPACE, alpha_version_id), redis.key().entity(VERSIONS_NAMESPACE, beta_version_id), redis.key().entity( - "versions_files:v1", + "versions_files:v3", format!("sha1_{}", alpha.file_hash), ), redis.key().entity( - "versions_files:v1", + "versions_files:v3", format!("sha1_{}", beta.file_hash), ), ]; From 17e156ee12636d6946237ead2011caae348c0877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 18:03:24 -0400 Subject: [PATCH 09/12] chore(labrinth): improve redis key identities handling --- apps/labrinth/src/database/redis/cache.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/apps/labrinth/src/database/redis/cache.rs b/apps/labrinth/src/database/redis/cache.rs index 76f0f0b871..6ce385c64f 100644 --- a/apps/labrinth/src/database/redis/cache.rs +++ b/apps/labrinth/src/database/redis/cache.rs @@ -728,26 +728,31 @@ where { let mut identities = Vec::with_capacity(5); let canonical_key = value.key.to_string(); + push_identity(&mut identities, canonical_key.clone()); - push_identity( - &mut identities, - normalize_key(&canonical_key, case_sensitive), - ); + if !case_sensitive { + push_identity(&mut identities, canonical_key.to_lowercase()); + } + if let Ok(decimal_id) = canonical_key.parse::() { let base62_id = to_base62(decimal_id); push_identity(&mut identities, base62_id.clone()); - push_identity( - &mut identities, - normalize_key(&base62_id, case_sensitive), - ); + + if !case_sensitive { + push_identity(&mut identities, base62_id.to_lowercase()); + } } else if let Ok(decimal_id) = parse_base62(&canonical_key) { push_identity(&mut identities, decimal_id.to_string()); } + if let Some(alias) = &value.alias { let alias = alias.to_string(); push_identity(&mut identities, alias.clone()); - push_identity(&mut identities, normalize_key(&alias, case_sensitive)); + if !case_sensitive { + push_identity(&mut identities, alias.to_lowercase()); + } } + identities } From 07b8540d57a900c6feb65bf6a717231edba72cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 18:09:19 -0400 Subject: [PATCH 10/12] chore(labrinth): simplify deadline handling --- apps/labrinth/src/database/redis/cache.rs | 396 +++--------------- .../src/database/redis/cache/locking.rs | 13 +- .../redis/cache/locking/distributed.rs | 105 ++--- 3 files changed, 101 insertions(+), 413 deletions(-) diff --git a/apps/labrinth/src/database/redis/cache.rs b/apps/labrinth/src/database/redis/cache.rs index 6ce385c64f..43861e0b39 100644 --- a/apps/labrinth/src/database/redis/cache.rs +++ b/apps/labrinth/src/database/redis/cache.rs @@ -325,99 +325,63 @@ impl CacheManager { return Ok(HashMap::new()); } - let get_cached_values = - |ids: DashMap, - deadline: Option, - locks_released: usize, - locks_waiting: usize| { - async move { - let slug_ids = if let Some(slug_namespace) = slug_namespace - { - async { - let keys = ids - .iter() - .map(|entry| { - let logical_key = normalize_key( - &entry.value().to_string(), - case_sensitive, - ); - self.key_builder - .entity(slug_namespace, logical_key) - }) - .collect::>(); - let mut connection = connect_before_deadline( - provider, - deadline, - self.locking.strategy(), - locks_released, - locks_waiting, - ) - .await?; - Ok::<_, DatabaseError>( - query_before_deadline( - deadline, - commands::get_many_strings( - &mut connection, - &keys, - ), - self.locking.strategy(), - locks_released, - locks_waiting, - ) + let get_cached_values = |ids: DashMap| { + async move { + let slug_ids = if let Some(slug_namespace) = slug_namespace { + async { + let keys = ids + .iter() + .map(|entry| { + let logical_key = normalize_key( + &entry.value().to_string(), + case_sensitive, + ); + self.key_builder + .entity(slug_namespace, logical_key) + }) + .collect::>(); + let mut connection = provider.connect().await?; + Ok::<_, DatabaseError>( + commands::get_many_strings(&mut connection, &keys) .await? .into_iter() .flatten() .collect::>(), - ) - } - .instrument(info_span!("get slug ids")) - .await? - } else { - Vec::new() - }; - - let keys = ids - .iter() - .map(|entry| entry.value().to_string()) - .chain(ids.iter().filter_map(|entry| { - parse_base62(&entry.value().to_string()) - .ok() - .map(|value| value.to_string()) - })) - .chain(slug_ids) - .map(|key| self.key_builder.entity(namespace, key)) - .collect::>(); - - let mut connection = connect_before_deadline( - provider, - deadline, - self.locking.strategy(), - locks_released, - locks_waiting, - ) - .await?; - let mut cached_values = HashMap::new(); - for value in query_before_deadline( - deadline, - commands::get_many(&mut connection, &keys), - self.locking.strategy(), - locks_released, - locks_waiting, - ) - .await? - { - if let Some(value) = value.and_then(|value| { - self.settings - .decode_value::>(&value) - }) { - cached_values.insert(value.key.clone(), value); - } + ) } + .instrument(info_span!("get slug ids")) + .await? + } else { + Vec::new() + }; + + let keys = ids + .iter() + .map(|entry| entry.value().to_string()) + .chain(ids.iter().filter_map(|entry| { + parse_base62(&entry.value().to_string()) + .ok() + .map(|value| value.to_string()) + })) + .chain(slug_ids) + .map(|key| self.key_builder.entity(namespace, key)) + .collect::>(); - Ok::<_, DatabaseError>((cached_values, ids)) + let mut connection = provider.connect().await?; + let mut cached_values = HashMap::new(); + for value in commands::get_many(&mut connection, &keys).await? { + if let Some(value) = value.and_then(|value| { + self.settings + .decode_value::>(&value) + }) { + cached_values.insert(value.key.clone(), value); + } } - .instrument(info_span!("get cached values")) - }; + + Ok::<_, DatabaseError>((cached_values, ids)) + } + .instrument(info_span!("get cached values")) + }; let (default_expiry, actual_expiry) = cache_expiries(namespace); let current_time = Utc::now(); @@ -425,8 +389,7 @@ impl CacheManager { let mut expired_identities = HashMap::new(); let deadline = Instant::now() + WAIT_TIMEOUT; - let (cached_values_raw, ids) = - get_cached_values(ids, Some(deadline), 0, 0).await?; + let (cached_values_raw, ids) = get_cached_values(ids).await?; let mut cached_values = cached_values_raw .into_iter() .filter_map(|(key, value)| { @@ -456,28 +419,19 @@ impl CacheManager { .iter() .map(|entry| entry.key().clone()) .collect::>(); - let locks_total = fetch_ids.len(); - for key in fetch_ids { if !ids.contains_key(&key) { continue; } let lock_key = self.key_builder.entity(namespace, &key); - let acquisition = - match self.locking.acquire(lock_key, deadline).await { - Ok(acquisition) => acquisition, - Err(error) => { - let error = map_lock_operation_error( - error, - self.locking.strategy(), - 0, - locks_total, - ); - release_owned_locks(owned_locks, deadline).await; - return Err(error); - } - }; + let acquisition = match self.locking.acquire(lock_key).await { + Ok(acquisition) => acquisition, + Err(error) => { + release_owned_locks(owned_locks).await; + return Err(error); + } + }; match acquisition { LockAcquisition::Owned(guard) => { @@ -544,28 +498,10 @@ impl CacheManager { encoded_values.push((key, slug, value, encoded)); } - let publication_deadline = if is_contended { - deadline - } else { - Instant::now() + WAIT_TIMEOUT - }; - let mut connection = connect_before_deadline( - provider, - Some(publication_deadline), - self.locking.strategy(), - 0, - waiters.len(), - ) - .await?; + let mut connection = provider.connect().await?; let mut ownership_valid = true; for lock in owned_locks.values() { - match lock - .validate_with_connection( - &mut connection, - publication_deadline, - ) - .await - { + match lock.validate_with_connection(&mut connection).await { Ok(true) => {} Ok(false) => { ownership_valid = false; @@ -582,14 +518,11 @@ impl CacheManager { for (key, slug, _, encoded) in &encoded_values { let redis_key = self.key_builder.entity(namespace, key.to_string()); - publish_cache_value( + commands::set( &mut connection, - self.locking.strategy(), &redis_key, encoded, - default_expiry, - publication_deadline, - waiters.len(), + Some(default_expiry), ) .await?; if let Some(slug) = slug @@ -603,14 +536,11 @@ impl CacheManager { let slug_key = self .key_builder .entity(slug_namespace, actual_slug); - publish_cache_value( + commands::set( &mut connection, - self.locking.strategy(), &slug_key, canonical_key.as_bytes(), - default_expiry, - publication_deadline, - waiters.len(), + Some(default_expiry), ) .await?; } @@ -629,19 +559,13 @@ impl CacheManager { Ok(HashMap::new()) }; - let release_deadline = if is_contended { - deadline - } else { - Instant::now() + WAIT_TIMEOUT - }; - release_owned_locks(owned_locks, release_deadline).await; + release_owned_locks(owned_locks).await; let operation_result = match fill_result { Ok(mut values) => { if waiters.is_empty() { Ok(values) } else { - let total = waiters.len(); match wait_for_locks( self.locking.strategy(), waiters, @@ -662,14 +586,7 @@ impl CacheManager { ) }) .collect::>(); - match get_cached_values( - fetch_ids, - Some(deadline), - total, - total, - ) - .await - { + match get_cached_values(fetch_ids).await { Ok((released_values, _)) => { values.extend(released_values); Ok(values) @@ -762,68 +679,6 @@ fn push_identity(identities: &mut Vec, identity: String) { } } -async fn connect_before_deadline

( - provider: &P, - deadline: Option, - strategy: CacheLockingStrategy, - locks_released: usize, - locks_waiting: usize, -) -> Result -where - P: ConnectionProvider, -{ - if let Some(deadline) = deadline { - timeout_at(deadline, provider.connect()) - .await - .map_err(|_| { - lock_timeout_error(strategy, locks_released, locks_waiting) - })? - } else { - provider.connect().await - } -} - -async fn query_before_deadline( - deadline: Option, - query: F, - strategy: CacheLockingStrategy, - locks_released: usize, - locks_waiting: usize, -) -> Result -where - F: Future>, -{ - if let Some(deadline) = deadline { - timeout_at(deadline, query).await.map_err(|_| { - lock_timeout_error(strategy, locks_released, locks_waiting) - })? - } else { - query.await - } -} - -async fn publish_cache_value( - connection: &mut C, - strategy: CacheLockingStrategy, - key: &str, - data: &[u8], - expiry: i64, - deadline: Instant, - locks_waiting: usize, -) -> Result<(), DatabaseError> -where - C: ConnectionLike, -{ - query_before_deadline( - Some(deadline), - commands::set(connection, key, data, Some(expiry)), - strategy, - 0, - locks_waiting, - ) - .await -} - async fn wait_for_locks( strategy: CacheLockingStrategy, waiters: Vec<(I, LockWaiter)>, @@ -867,19 +722,6 @@ fn is_lock_timeout(error: &DatabaseError) -> bool { ) } -fn map_lock_operation_error( - error: DatabaseError, - strategy: CacheLockingStrategy, - locks_released: usize, - locks_waiting: usize, -) -> DatabaseError { - if is_lock_timeout(&error) { - lock_timeout_error(strategy, locks_released, locks_waiting) - } else { - error - } -} - fn lock_timeout_error( strategy: CacheLockingStrategy, locks_released: usize, @@ -899,12 +741,9 @@ fn lock_timeout_error( } } -async fn release_owned_locks( - owned_locks: HashMap, - deadline: Instant, -) { +async fn release_owned_locks(owned_locks: HashMap) { for guard in owned_locks.into_values() { - if let Err(error) = guard.release(deadline).await { + if let Err(error) = guard.release().await { tracing::warn!( error = ?error, "failed to explicitly release cache lock", @@ -929,106 +768,9 @@ impl RedisValue { #[cfg(test)] mod tests { - use std::future::{Future, pending}; - use std::time::Duration; - use dashmap::DashMap; - use redis::aio::ConnectionLike; - use tokio::time::{Instant, advance}; - - use super::{ - ConnectionProvider, RedisValue, connect_before_deadline, - query_before_deadline, remove_resolved_ids, value_identities, - }; - use crate::database::models::DatabaseError; - use crate::database::redis::CacheLockingStrategy; - - struct NeverConnection; - - impl ConnectionLike for NeverConnection { - fn req_packed_command<'a>( - &'a mut self, - _: &'a redis::Cmd, - ) -> redis::RedisFuture<'a, redis::Value> { - Box::pin(pending()) - } - - fn req_packed_commands<'a>( - &'a mut self, - _: &'a redis::Pipeline, - _: usize, - _: usize, - ) -> redis::RedisFuture<'a, Vec> { - Box::pin(pending()) - } - - fn get_db(&self) -> i64 { - 0 - } - } - - struct PendingProvider; - - impl ConnectionProvider for PendingProvider { - type Connection = NeverConnection; - fn connect( - &self, - ) -> impl Future> + Send - { - pending() - } - } - - #[tokio::test(start_paused = true)] - async fn five_second_deadline_includes_hung_pool_acquisition() { - let deadline = Instant::now() + Duration::from_secs(5); - let task = tokio::spawn(async move { - connect_before_deadline( - &PendingProvider, - Some(deadline), - CacheLockingStrategy::Local, - 0, - 1, - ) - .await - }); - - advance(Duration::from_millis(4_999)).await; - assert!(!task.is_finished()); - advance(Duration::from_millis(1)).await; - assert!(matches!( - task.await.unwrap(), - Err(DatabaseError::LocalCacheTimeout { - released: 0, - total: 1, - }) - )); - } - - #[tokio::test(start_paused = true)] - async fn five_second_deadline_includes_hung_redis_query() { - let deadline = Instant::now() + Duration::from_secs(5); - let task = tokio::spawn(query_before_deadline( - Some(deadline), - pending::>(), - CacheLockingStrategy::Distributed, - 0, - 1, - )); - - advance(Duration::from_millis(4_999)).await; - assert!(!task.is_finished()); - advance(Duration::from_millis(1)).await; - assert!(matches!( - task.await.unwrap(), - Err(DatabaseError::CacheTimeout { - locks_released: 0, - locks_waiting: 1, - .. - }) - )); - } + use super::{RedisValue, remove_resolved_ids, value_identities}; #[test] fn stale_identity_index_includes_equivalent_request_forms() { diff --git a/apps/labrinth/src/database/redis/cache/locking.rs b/apps/labrinth/src/database/redis/cache/locking.rs index 0785486091..513cf90168 100644 --- a/apps/labrinth/src/database/redis/cache/locking.rs +++ b/apps/labrinth/src/database/redis/cache/locking.rs @@ -68,11 +68,10 @@ impl LockCoordinator { pub(super) async fn acquire( &self, key: String, - deadline: Instant, ) -> Result { match self { Self::Local(manager) => Ok(manager.acquire(key).into()), - Self::Distributed(manager) => manager.acquire(key, deadline).await, + Self::Distributed(manager) => manager.acquire(key).await, } } } @@ -104,7 +103,6 @@ impl OwnedLockGuard { pub(super) async fn validate_with_connection( &self, connection: &mut C, - deadline: Instant, ) -> Result where C: ConnectionLike, @@ -112,21 +110,18 @@ impl OwnedLockGuard { match self { Self::Local(_) => Ok(true), Self::Distributed(guard) => { - guard.validate_with_connection(connection, deadline).await + guard.validate_with_connection(connection).await } } } - pub(super) async fn release( - self, - deadline: Instant, - ) -> Result { + pub(super) async fn release(self) -> Result { match self { Self::Local(guard) => { guard.release(); Ok(ReleaseOutcome::Released) } - Self::Distributed(guard) => guard.release(deadline).await, + Self::Distributed(guard) => guard.release().await, } } } diff --git a/apps/labrinth/src/database/redis/cache/locking/distributed.rs b/apps/labrinth/src/database/redis/cache/locking/distributed.rs index 2cc9f3368a..7ba9931c60 100644 --- a/apps/labrinth/src/database/redis/cache/locking/distributed.rs +++ b/apps/labrinth/src/database/redis/cache/locking/distributed.rs @@ -30,10 +30,9 @@ impl DistributedLockManager { pub(super) async fn acquire( &self, key: String, - deadline: Instant, ) -> Result { let lock_key = cache_lock_key(&key); - let acquired = self.try_acquire(&lock_key, deadline).await?; + let acquired = self.try_acquire(&lock_key).await?; if !acquired { return Ok(LockAcquisition::Waiting(LockWaiter::Distributed( @@ -62,94 +61,50 @@ impl DistributedLockManager { ))) } - async fn connect( - &self, - deadline: Instant, - ) -> Result - { - timeout_at(deadline, self.backend.connect()) - .await - .map_err(|_| lock_timeout())? - .map_err(Into::into) - } - - async fn try_acquire( - &self, - lock_key: &str, - deadline: Instant, - ) -> Result { - let mut connection = self.connect(deadline).await?; - timeout_at( - deadline, - commands::acquire_lock( - &mut connection, - lock_key, - duration_millis(self.timing.lease), - ), + async fn try_acquire(&self, lock_key: &str) -> Result { + let mut connection = self.backend.connect().await?; + commands::acquire_lock( + &mut connection, + lock_key, + duration_millis(self.timing.lease), ) .await - .map_err(|_| lock_timeout())? } - async fn renew( - &self, - lock_key: &str, - deadline: Instant, - ) -> Result { - let mut connection = self.connect(deadline).await?; - timeout_at( - deadline, - commands::renew_lock( - &mut connection, - lock_key, - duration_millis(self.timing.lease), - ), + async fn renew(&self, lock_key: &str) -> Result { + let mut connection = self.backend.connect().await?; + commands::renew_lock( + &mut connection, + lock_key, + duration_millis(self.timing.lease), ) .await - .map_err(|_| lock_timeout())? } async fn renew_with_connection( &self, connection: &mut C, lock_key: &str, - deadline: Instant, ) -> Result where C: ConnectionLike, { - timeout_at( - deadline, - commands::renew_lock( - connection, - lock_key, - duration_millis(self.timing.lease), - ), + commands::renew_lock( + connection, + lock_key, + duration_millis(self.timing.lease), ) .await - .map_err(|_| lock_timeout())? } - async fn release( - &self, - lock_key: &str, - deadline: Instant, - ) -> Result { - let mut connection = self.connect(deadline).await?; - timeout_at(deadline, commands::release_lock(&mut connection, lock_key)) - .await - .map_err(|_| lock_timeout())? + async fn release(&self, lock_key: &str) -> Result { + let mut connection = self.backend.connect().await?; + commands::release_lock(&mut connection, lock_key).await } - async fn exists( - &self, - lock_key: &str, - deadline: Instant, - ) -> Result { - let mut connection = self.connect(deadline).await?; - timeout_at(deadline, commands::lock_exists(&mut connection, lock_key)) - .await - .map_err(|_| lock_timeout())? + async fn exists(&self, lock_key: &str) -> Result { + let mut connection = self.backend.connect().await?; + commands::lock_exists(&mut connection, lock_key).await } fn spawn_renewal( @@ -166,8 +121,7 @@ impl DistributedLockManager { _ = cancellation_token.cancelled() => break, result = async { sleep(manager.timing.renewal).await; - let deadline = Instant::now() + manager.timing.renewal; - manager.renew(&lock_key, deadline).await + manager.renew(&lock_key).await } => result, }; @@ -193,8 +147,7 @@ impl DistributedLockManager { }; let manager = self.clone(); handle.spawn(async move { - let deadline = Instant::now() + manager.timing.renewal; - if manager.release(&lock_key, deadline).await.is_err() { + if manager.release(&lock_key).await.is_err() { warn!("failed to clean up distributed cache lease"); } }); @@ -213,7 +166,6 @@ impl DistributedLockGuard { pub(super) async fn validate_with_connection( &self, connection: &mut C, - deadline: Instant, ) -> Result where C: ConnectionLike, @@ -224,7 +176,7 @@ impl DistributedLockGuard { let result = self .manager - .renew_with_connection(connection, &self.lock_key, deadline) + .renew_with_connection(connection, &self.lock_key) .await; self.handle_validation_result(result) } @@ -248,10 +200,9 @@ impl DistributedLockGuard { pub(super) async fn release( mut self, - deadline: Instant, ) -> Result { self.stop_renewal(); - match self.manager.release(&self.lock_key, deadline).await { + match self.manager.release(&self.lock_key).await { Ok(true) => { self.cleanup_complete = true; self.state.owned.store(false, Ordering::Release); @@ -294,7 +245,7 @@ impl DistributedLockWaiter { ) -> Result<(), DatabaseError> { let mut attempt = 0; loop { - if !self.manager.exists(&self.lock_key, deadline).await? { + if !self.manager.exists(&self.lock_key).await? { return Ok(()); } From 1306943b7527ce20298f2fc11a61d09293e39327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 18:28:11 -0400 Subject: [PATCH 11/12] chore(labrinth): remove unused lease tracking --- apps/labrinth/src/database/redis/cache.rs | 155 ++++-------------- .../src/database/redis/cache/locking.rs | 16 -- .../redis/cache/locking/distributed.rs | 83 +--------- .../src/database/redis/cache/locking/local.rs | 99 ----------- apps/labrinth/src/database/redis/config.rs | 29 ---- apps/labrinth/src/database/redis/key.rs | 22 --- 6 files changed, 31 insertions(+), 373 deletions(-) diff --git a/apps/labrinth/src/database/redis/cache.rs b/apps/labrinth/src/database/redis/cache.rs index 43861e0b39..64e0243405 100644 --- a/apps/labrinth/src/database/redis/cache.rs +++ b/apps/labrinth/src/database/redis/cache.rs @@ -412,7 +412,6 @@ impl CacheManager { let mut waiters = Vec::new(); let mut owned_locks = HashMap::new(); - let mut has_contention = false; if !ids.is_empty() { let fetch_ids = ids @@ -438,7 +437,6 @@ impl CacheManager { owned_locks.insert(key, guard); } LockAcquisition::Waiting(waiter) => { - has_contention = true; if let Some(canonical_key) = expired_identities.get(&key).cloned() && let Some(value) = @@ -457,33 +455,25 @@ impl CacheManager { } } - let is_contended = has_contention; let fill_result = if !ids.is_empty() { async { let fetch_ids = ids .iter() .map(|entry| entry.value().clone()) .collect::>(); - let fill_deadline = if is_contended { - deadline - } else { - Instant::now() + FILL_TIMEOUT - }; + + let fill_deadline = Instant::now() + FILL_TIMEOUT; + let values = timeout_at(fill_deadline, closure(fetch_ids)) .await .map_err(|_| { - if is_contended { - lock_timeout_error( - self.locking.strategy(), - 0, - waiters.len(), - ) - } else { - DatabaseError::Internal(eyre::eyre!( - "cache fill timed out after 60 seconds" - )) - } + lock_timeout_error( + self.locking.strategy(), + 0, + waiters.len(), + ) })??; + let mut return_values = HashMap::new(); let mut encoded_values = Vec::with_capacity(values.len()); @@ -499,51 +489,32 @@ impl CacheManager { } let mut connection = provider.connect().await?; - let mut ownership_valid = true; - for lock in owned_locks.values() { - match lock.validate_with_connection(&mut connection).await { - Ok(true) => {} - Ok(false) => { - ownership_valid = false; - break; - } - Err(_) => { - ownership_valid = false; - break; - } - } - } - - if ownership_valid { - for (key, slug, _, encoded) in &encoded_values { - let redis_key = - self.key_builder.entity(namespace, key.to_string()); + for (key, slug, _, encoded) in &encoded_values { + let redis_key = + self.key_builder.entity(namespace, key.to_string()); + commands::set( + &mut connection, + &redis_key, + encoded, + Some(default_expiry), + ) + .await?; + if let Some(slug) = slug + && let Some(slug_namespace) = slug_namespace + { + let canonical_key = key.to_string(); + let actual_slug = + normalize_key(&slug.to_string(), case_sensitive); + let slug_key = self + .key_builder + .entity(slug_namespace, actual_slug); commands::set( &mut connection, - &redis_key, - encoded, + &slug_key, + canonical_key.as_bytes(), Some(default_expiry), ) .await?; - if let Some(slug) = slug - && let Some(slug_namespace) = slug_namespace - { - let canonical_key = key.to_string(); - let actual_slug = normalize_key( - &slug.to_string(), - case_sensitive, - ); - let slug_key = self - .key_builder - .entity(slug_namespace, actual_slug); - commands::set( - &mut connection, - &slug_key, - canonical_key.as_bytes(), - Some(default_expiry), - ) - .await?; - } } } @@ -765,69 +736,3 @@ impl RedisValue { &self.val } } - -#[cfg(test)] -mod tests { - use dashmap::DashMap; - - use super::{RedisValue, remove_resolved_ids, value_identities}; - - #[test] - fn stale_identity_index_includes_equivalent_request_forms() { - let value = RedisValue { - key: 1234_u64, - alias: Some("ExampleSlug"), - iat: 0, - val: (), - }; - let identities = value_identities(&value, false); - - assert!(identities.contains(&"1234".to_string())); - assert!( - identities.contains(&ariadne::ids::base62_impl::to_base62(1234)) - ); - assert!(identities.contains(&"ExampleSlug".to_string())); - assert!(identities.contains(&"exampleslug".to_string())); - } - - #[test] - fn resolving_stale_value_removes_every_equivalent_request_form() { - let value = RedisValue { - key: 1234_u64, - alias: Some("ExampleSlug"), - iat: 0, - val: (), - }; - let ids = DashMap::new(); - ids.insert("1234".to_string(), ()); - ids.insert( - ariadne::ids::base62_impl::to_base62(1234).to_lowercase(), - (), - ); - ids.insert("exampleslug".to_string(), ()); - - remove_resolved_ids(&ids, &value, false); - - assert!(ids.is_empty()); - } - - #[test] - fn case_sensitive_alias_resolution_preserves_other_case() { - let value = RedisValue { - key: 1234_u64, - alias: Some("mra_PATValue"), - iat: 0, - val: (), - }; - let identities = value_identities(&value, true); - let ids = DashMap::new(); - ids.insert("mra_PATValue".to_string(), ()); - ids.insert("mra_patvalue".to_string(), ()); - - remove_resolved_ids(&ids, &value, true); - - assert!(identities.contains(&"mra_PATValue".to_string())); - assert!(!identities.contains(&"mra_patvalue".to_string())); - assert!(ids.contains_key("mra_patvalue")); - } -} diff --git a/apps/labrinth/src/database/redis/cache/locking.rs b/apps/labrinth/src/database/redis/cache/locking.rs index 513cf90168..3630105d27 100644 --- a/apps/labrinth/src/database/redis/cache/locking.rs +++ b/apps/labrinth/src/database/redis/cache/locking.rs @@ -3,7 +3,6 @@ mod local; use std::time::Duration; -use redis::aio::ConnectionLike; use tokio::time::Instant; use crate::database::models::DatabaseError; @@ -100,21 +99,6 @@ pub(super) enum OwnedLockGuard { } impl OwnedLockGuard { - pub(super) async fn validate_with_connection( - &self, - connection: &mut C, - ) -> Result - where - C: ConnectionLike, - { - match self { - Self::Local(_) => Ok(true), - Self::Distributed(guard) => { - guard.validate_with_connection(connection).await - } - } - } - pub(super) async fn release(self) -> Result { match self { Self::Local(guard) => { diff --git a/apps/labrinth/src/database/redis/cache/locking/distributed.rs b/apps/labrinth/src/database/redis/cache/locking/distributed.rs index 7ba9931c60..f43dd47816 100644 --- a/apps/labrinth/src/database/redis/cache/locking/distributed.rs +++ b/apps/labrinth/src/database/redis/cache/locking/distributed.rs @@ -1,8 +1,5 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -use redis::aio::ConnectionLike; use tokio::runtime::Handle; use tokio::time::{Instant, sleep, timeout_at}; use tokio_util::sync::CancellationToken; @@ -43,18 +40,15 @@ impl DistributedLockManager { ))); } - let state = Arc::new(LeaseState::new()); let renewal_cancellation_token = CancellationToken::new(); self.spawn_renewal( lock_key.clone(), - state.clone(), renewal_cancellation_token.clone(), ); Ok(LockAcquisition::Owned(OwnedLockGuard::Distributed( DistributedLockGuard { manager: self.clone(), lock_key, - state, renewal_cancellation_token, cleanup_complete: false, }, @@ -81,22 +75,6 @@ impl DistributedLockManager { .await } - async fn renew_with_connection( - &self, - connection: &mut C, - lock_key: &str, - ) -> Result - where - C: ConnectionLike, - { - commands::renew_lock( - connection, - lock_key, - duration_millis(self.timing.lease), - ) - .await - } - async fn release(&self, lock_key: &str) -> Result { let mut connection = self.backend.connect().await?; commands::release_lock(&mut connection, lock_key).await @@ -110,7 +88,6 @@ impl DistributedLockManager { fn spawn_renewal( &self, lock_key: String, - state: Arc, cancellation_token: CancellationToken, ) { let manager = self.clone(); @@ -127,12 +104,8 @@ impl DistributedLockManager { match result { Ok(true) => {} - Ok(false) => { - state.mark_lost(); - break; - } + Ok(false) => break, Err(_) => { - state.mark_lost(); warn!("failed to renew distributed cache lease"); break; } @@ -157,47 +130,11 @@ impl DistributedLockManager { pub struct DistributedLockGuard { manager: DistributedLockManager, lock_key: String, - state: Arc, renewal_cancellation_token: CancellationToken, cleanup_complete: bool, } impl DistributedLockGuard { - pub(super) async fn validate_with_connection( - &self, - connection: &mut C, - ) -> Result - where - C: ConnectionLike, - { - if !self.state.owned.load(Ordering::Acquire) { - return Ok(false); - } - - let result = self - .manager - .renew_with_connection(connection, &self.lock_key) - .await; - self.handle_validation_result(result) - } - - fn handle_validation_result( - &self, - result: Result, - ) -> Result { - match result { - Ok(true) => Ok(self.state.owned.load(Ordering::Acquire)), - Ok(false) => { - self.state.mark_lost(); - Ok(false) - } - Err(error) => { - self.state.mark_lost(); - Err(error) - } - } - } - pub(super) async fn release( mut self, ) -> Result { @@ -205,12 +142,10 @@ impl DistributedLockGuard { match self.manager.release(&self.lock_key).await { Ok(true) => { self.cleanup_complete = true; - self.state.owned.store(false, Ordering::Release); Ok(ReleaseOutcome::Released) } Ok(false) => { self.cleanup_complete = true; - self.state.mark_lost(); Ok(ReleaseOutcome::NotOwner) } Err(error) => Err(error), @@ -259,22 +194,6 @@ impl DistributedLockWaiter { } } -struct LeaseState { - owned: AtomicBool, -} - -impl LeaseState { - fn new() -> Self { - Self { - owned: AtomicBool::new(true), - } - } - - fn mark_lost(&self) { - self.owned.store(false, Ordering::Release); - } -} - fn cache_lock_key(key: &str) -> String { format!("{key}/lock") } diff --git a/apps/labrinth/src/database/redis/cache/locking/local.rs b/apps/labrinth/src/database/redis/cache/locking/local.rs index 7bdada8713..7bab7c41b7 100644 --- a/apps/labrinth/src/database/redis/cache/locking/local.rs +++ b/apps/labrinth/src/database/redis/cache/locking/local.rs @@ -125,102 +125,3 @@ fn lock_timeout() -> DatabaseError { total: 1, } } - -#[cfg(test)] -mod tests { - use std::future::pending; - use std::time::Duration; - - use tokio::time::Instant; - - use super::{LocalLockAcquisition, LocalLockManager}; - - #[tokio::test] - async fn waiter_is_released_when_owner_releases() { - let manager = LocalLockManager::new(); - let owner = match manager.acquire("project".to_string()) { - LocalLockAcquisition::Owned(owner) => owner, - LocalLockAcquisition::Waiting(_) => { - panic!("first acquisition waited") - } - }; - let waiter = match manager.acquire("project".to_string()) { - LocalLockAcquisition::Waiting(waiter) => waiter, - LocalLockAcquisition::Owned(_) => { - panic!("second acquisition owned") - } - }; - - owner.release(); - waiter - .wait(Instant::now() + Duration::from_secs(1)) - .await - .unwrap(); - } - - #[tokio::test] - async fn dropping_owner_releases_waiters() { - let manager = LocalLockManager::new(); - let owner = match manager.acquire("project".to_string()) { - LocalLockAcquisition::Owned(owner) => owner, - LocalLockAcquisition::Waiting(_) => { - panic!("first acquisition waited") - } - }; - let waiter = match manager.acquire("project".to_string()) { - LocalLockAcquisition::Waiting(waiter) => waiter, - LocalLockAcquisition::Owned(_) => { - panic!("second acquisition owned") - } - }; - - drop(owner); - waiter - .wait(Instant::now() + Duration::from_secs(1)) - .await - .unwrap(); - } - - #[tokio::test] - async fn aborting_owner_task_releases_waiters() { - let manager = LocalLockManager::new(); - let owner = match manager.acquire("project".to_string()) { - LocalLockAcquisition::Owned(owner) => owner, - LocalLockAcquisition::Waiting(_) => { - panic!("first acquisition waited") - } - }; - let waiter = match manager.acquire("project".to_string()) { - LocalLockAcquisition::Waiting(waiter) => waiter, - LocalLockAcquisition::Owned(_) => { - panic!("second acquisition owned") - } - }; - let task = tokio::spawn(async move { - let _owner = owner; - pending::<()>().await; - }); - - task.abort(); - let _ = task.await; - waiter - .wait(Instant::now() + Duration::from_secs(1)) - .await - .unwrap(); - } - - #[test] - fn independent_managers_do_not_coordinate() { - let first = LocalLockManager::new(); - let second = LocalLockManager::new(); - - assert!(matches!( - first.acquire("project".to_string()), - LocalLockAcquisition::Owned(_) - )); - assert!(matches!( - second.acquire("project".to_string()), - LocalLockAcquisition::Owned(_) - )); - } -} diff --git a/apps/labrinth/src/database/redis/config.rs b/apps/labrinth/src/database/redis/config.rs index 5a4ccb25f5..6e60d2f15f 100644 --- a/apps/labrinth/src/database/redis/config.rs +++ b/apps/labrinth/src/database/redis/config.rs @@ -261,32 +261,3 @@ impl RedisConfig { self.cache_locking_strategy } } - -#[cfg(test)] -mod tests { - use super::CacheLockingStrategy; - - #[test] - fn cache_locking_strategy_defaults_to_local() { - assert_eq!( - CacheLockingStrategy::default(), - CacheLockingStrategy::Local - ); - } - - #[test] - fn cache_locking_strategy_parsing_is_strict() { - assert_eq!( - "local".parse::().unwrap(), - CacheLockingStrategy::Local - ); - assert_eq!( - "distributed".parse::().unwrap(), - CacheLockingStrategy::Distributed - ); - - for invalid in ["", "LOCAL", "Distributed", " local", "local "] { - assert!(invalid.parse::().is_err()); - } - } -} diff --git a/apps/labrinth/src/database/redis/key.rs b/apps/labrinth/src/database/redis/key.rs index 936ba3febd..7827f42fd2 100644 --- a/apps/labrinth/src/database/redis/key.rs +++ b/apps/labrinth/src/database/redis/key.rs @@ -73,25 +73,3 @@ fn escape_slot_tag(value: &str) -> String { } escaped } - -#[cfg(test)] -mod tests { - use super::KeyBuilder; - use crate::database::redis::RedisTopology; - - #[test] - fn cluster_slot_tags_escape_braces_and_percent_signs() { - let builder = KeyBuilder::new("labrinth", RedisTopology::Cluster); - let entity = builder.entity("projects", "a{%}b"); - - assert!(entity.contains("{a%7B%25%7Db}")); - } - - #[test] - fn cluster_empty_slot_tag_is_nonempty() { - let builder = KeyBuilder::new("labrinth", RedisTopology::Cluster); - let entity = builder.entity("projects", ""); - - assert!(entity.contains("{%00}")); - } -} From 207685ab9241a1fb8dc6508157a1800e34664f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Sun, 19 Jul 2026 21:54:23 -0400 Subject: [PATCH 12/12] chore(labrinth): remove distributed cache locking for now --- Cargo.lock | 2 - apps/labrinth/.env.docker-compose | 3 +- apps/labrinth/.env.local | 3 +- apps/labrinth/Cargo.toml | 1 - apps/labrinth/src/database/models/mod.rs | 9 - apps/labrinth/src/database/redis/cache.rs | 79 +------ .../src/database/redis/cache/locking.rs | 129 +--------- .../redis/cache/locking/distributed.rs | 220 ------------------ .../src/database/redis/cache/locking/local.rs | 57 +++-- apps/labrinth/src/database/redis/commands.rs | 93 -------- apps/labrinth/src/database/redis/config.rs | 8 + apps/labrinth/src/database/redis/mod.rs | 2 - 12 files changed, 50 insertions(+), 556 deletions(-) delete mode 100644 apps/labrinth/src/database/redis/cache/locking/distributed.rs diff --git a/Cargo.lock b/Cargo.lock index 9334184de9..82040aa753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5499,7 +5499,6 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-stream", - "tokio-util", "totp-rs", "tracing", "tracing-actix-web", @@ -11279,7 +11278,6 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", - "futures-util", "pin-project-lite", "slab", "tokio", diff --git a/apps/labrinth/.env.docker-compose b/apps/labrinth/.env.docker-compose index f9c726f479..6fa1309193 100644 --- a/apps/labrinth/.env.docker-compose +++ b/apps/labrinth/.env.docker-compose @@ -31,8 +31,7 @@ TYPESENSE_INDEX_PREFIX=labrinth REDIS_MODE=standalone REDIS_CONNECTION_TYPE=pooled -# Cache fill coordination: `local` per process or a tokenized `distributed` Redis lease -# Set consistently across all replicas; Redis failures never fall back to `local` +# Cache fill coordination is local to each process REDIS_CACHE_LOCKING_STRATEGY=local REDIS_URL=redis://labrinth-redis REDIS_MIN_CONNECTIONS=0 diff --git a/apps/labrinth/.env.local b/apps/labrinth/.env.local index 0ae8823928..475bbb4f06 100644 --- a/apps/labrinth/.env.local +++ b/apps/labrinth/.env.local @@ -49,8 +49,7 @@ TYPESENSE_INDEX_PREFIX=labrinth REDIS_MODE=standalone REDIS_CONNECTION_TYPE=pooled -# Cache fill coordination: `local` per process or a tokenized `distributed` Redis lease -# Set consistently across all replicas; Redis failures never fall back to `local` +# Cache fill coordination is local to each process REDIS_CACHE_LOCKING_STRATEGY=local REDIS_URL=redis://localhost REDIS_MIN_CONNECTIONS=0 diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 2a8fa1007a..fd814b5806 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -131,7 +131,6 @@ strum = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "sync"] } tokio-stream = { workspace = true } -tokio-util = { workspace = true, features = ["rt"] } totp-rs = { workspace = true, features = ["gen_secret"] } tracing = { workspace = true } tracing-actix-web = { workspace = true } diff --git a/apps/labrinth/src/database/models/mod.rs b/apps/labrinth/src/database/models/mod.rs index 2ca113b42c..5aa1d9de35 100644 --- a/apps/labrinth/src/database/models/mod.rs +++ b/apps/labrinth/src/database/models/mod.rs @@ -80,15 +80,6 @@ pub enum DatabaseError { PostcardCacheError(#[from] postcard::Error), #[error("Schema error: {0}")] SchemaError(String), - #[error( - "Timeout waiting on Redis cache lock ({locks_released}/{locks_waiting} released, spent {time_spent_pool_wait_ms}ms/{time_spent_total_ms}ms waiting on connections from pool)" - )] - CacheTimeout { - locks_released: usize, - locks_waiting: usize, - time_spent_pool_wait_ms: u64, - time_spent_total_ms: u64, - }, #[error( "Timeout waiting on local cache lock ({released}/{total} released)" )] diff --git a/apps/labrinth/src/database/redis/cache.rs b/apps/labrinth/src/database/redis/cache.rs index 64e0243405..1d5a5e83ce 100644 --- a/apps/labrinth/src/database/redis/cache.rs +++ b/apps/labrinth/src/database/redis/cache.rs @@ -19,15 +19,12 @@ use tracing::{Instrument, info_span}; use crate::database::models::DatabaseError; use super::commands; -use super::config::CacheLockingStrategy; -use super::connection::RedisBackend; use super::key::KeyBuilder; mod locking; use locking::{ - LockAcquisition, LockCoordinator, LockWaiter, OwnedLockGuard, WAIT_TIMEOUT, - normalize_key, + LockAcquisition, LockCoordinator, LockWaiter, WAIT_TIMEOUT, normalize_key, }; const ACTUAL_EXPIRY: i64 = 60 * 30; @@ -165,14 +162,9 @@ pub struct CacheManager { } impl CacheManager { - pub fn new( - key_builder: KeyBuilder, - settings: CacheSettings, - locking_strategy: CacheLockingStrategy, - backend: RedisBackend, - ) -> Self { + pub fn new(key_builder: KeyBuilder, settings: CacheSettings) -> Self { Self { - locking: LockCoordinator::new(locking_strategy, backend), + locking: LockCoordinator::new(), key_builder, settings, } @@ -424,13 +416,7 @@ impl CacheManager { } let lock_key = self.key_builder.entity(namespace, &key); - let acquisition = match self.locking.acquire(lock_key).await { - Ok(acquisition) => acquisition, - Err(error) => { - release_owned_locks(owned_locks).await; - return Err(error); - } - }; + let acquisition = self.locking.acquire(lock_key); match acquisition { LockAcquisition::Owned(guard) => { @@ -466,13 +452,7 @@ impl CacheManager { let values = timeout_at(fill_deadline, closure(fetch_ids)) .await - .map_err(|_| { - lock_timeout_error( - self.locking.strategy(), - 0, - waiters.len(), - ) - })??; + .map_err(|_| lock_timeout_error(0, waiters.len()))??; let mut return_values = HashMap::new(); let mut encoded_values = Vec::with_capacity(values.len()); @@ -530,20 +510,14 @@ impl CacheManager { Ok(HashMap::new()) }; - release_owned_locks(owned_locks).await; + drop(owned_locks); let operation_result = match fill_result { Ok(mut values) => { if waiters.is_empty() { Ok(values) } else { - match wait_for_locks( - self.locking.strategy(), - waiters, - deadline, - ) - .await - { + match wait_for_locks(waiters, deadline).await { Ok(released_ids) => { let fetch_ids = released_ids .into_iter() @@ -651,7 +625,6 @@ fn push_identity(identities: &mut Vec, identity: String) { } async fn wait_for_locks( - strategy: CacheLockingStrategy, waiters: Vec<(I, LockWaiter)>, deadline: Instant, ) -> Result, DatabaseError> { @@ -673,11 +646,7 @@ async fn wait_for_locks( Err(error) if is_lock_timeout(&error) || Instant::now() >= deadline => { - return Err(lock_timeout_error( - strategy, - released.len(), - total, - )); + return Err(lock_timeout_error(released.len(), total)); } Err(error) => return Err(error), } @@ -686,40 +655,16 @@ async fn wait_for_locks( } fn is_lock_timeout(error: &DatabaseError) -> bool { - matches!( - error, - DatabaseError::CacheTimeout { .. } - | DatabaseError::LocalCacheTimeout { .. } - ) + matches!(error, DatabaseError::LocalCacheTimeout { .. }) } fn lock_timeout_error( - strategy: CacheLockingStrategy, locks_released: usize, locks_waiting: usize, ) -> DatabaseError { - match strategy { - CacheLockingStrategy::Local => DatabaseError::LocalCacheTimeout { - released: locks_released, - total: locks_waiting, - }, - CacheLockingStrategy::Distributed => DatabaseError::CacheTimeout { - locks_released, - locks_waiting, - time_spent_pool_wait_ms: 0, - time_spent_total_ms: WAIT_TIMEOUT.as_millis() as u64, - }, - } -} - -async fn release_owned_locks(owned_locks: HashMap) { - for guard in owned_locks.into_values() { - if let Err(error) = guard.release().await { - tracing::warn!( - error = ?error, - "failed to explicitly release cache lock", - ); - } + DatabaseError::LocalCacheTimeout { + released: locks_released, + total: locks_waiting, } } diff --git a/apps/labrinth/src/database/redis/cache/locking.rs b/apps/labrinth/src/database/redis/cache/locking.rs index 3630105d27..be1f315b12 100644 --- a/apps/labrinth/src/database/redis/cache/locking.rs +++ b/apps/labrinth/src/database/redis/cache/locking.rs @@ -1,138 +1,11 @@ -mod distributed; mod local; use std::time::Duration; -use tokio::time::Instant; - -use crate::database::models::DatabaseError; - -use super::super::config::CacheLockingStrategy; -use super::super::connection::RedisBackend; - -use self::distributed::{ - DistributedLockGuard, DistributedLockManager, DistributedLockWaiter, -}; -use self::local::{ - LocalLockAcquisition, LocalLockGuard, LocalLockManager, LocalLockWaiter, -}; +pub(super) use self::local::{LockAcquisition, LockCoordinator, LockWaiter}; pub(super) const WAIT_TIMEOUT: Duration = Duration::from_secs(5); -#[derive(Debug, Clone, Copy)] -pub(super) struct LockTiming { - pub(super) lease: Duration, - pub(super) renewal: Duration, - pub(super) poll_min: Duration, - pub(super) poll_max: Duration, -} - -impl Default for LockTiming { - fn default() -> Self { - Self { - lease: Duration::from_secs(5), - renewal: Duration::from_secs(2), - poll_min: Duration::from_millis(50), - poll_max: Duration::from_millis(500), - } - } -} - -#[derive(Clone)] -pub(super) enum LockCoordinator { - Local(LocalLockManager), - Distributed(DistributedLockManager), -} - -impl LockCoordinator { - pub(super) fn new( - strategy: CacheLockingStrategy, - backend: RedisBackend, - ) -> Self { - match strategy { - CacheLockingStrategy::Local => Self::Local(LocalLockManager::new()), - CacheLockingStrategy::Distributed => Self::Distributed( - DistributedLockManager::new(backend, LockTiming::default()), - ), - } - } - - pub(super) fn strategy(&self) -> CacheLockingStrategy { - match self { - Self::Local(_) => CacheLockingStrategy::Local, - Self::Distributed(_) => CacheLockingStrategy::Distributed, - } - } - - pub(super) async fn acquire( - &self, - key: String, - ) -> Result { - match self { - Self::Local(manager) => Ok(manager.acquire(key).into()), - Self::Distributed(manager) => manager.acquire(key).await, - } - } -} - -pub(super) enum LockAcquisition { - Owned(OwnedLockGuard), - Waiting(LockWaiter), -} - -impl From for LockAcquisition { - fn from(acquisition: LocalLockAcquisition) -> Self { - match acquisition { - LocalLockAcquisition::Owned(guard) => { - Self::Owned(OwnedLockGuard::Local(guard)) - } - LocalLockAcquisition::Waiting(waiter) => { - Self::Waiting(LockWaiter::Local(waiter)) - } - } - } -} - -pub(super) enum OwnedLockGuard { - Local(LocalLockGuard), - Distributed(DistributedLockGuard), -} - -impl OwnedLockGuard { - pub(super) async fn release(self) -> Result { - match self { - Self::Local(guard) => { - guard.release(); - Ok(ReleaseOutcome::Released) - } - Self::Distributed(guard) => guard.release().await, - } - } -} - -pub(super) enum LockWaiter { - Local(LocalLockWaiter), - Distributed(DistributedLockWaiter), -} - -impl LockWaiter { - pub(super) async fn wait( - self, - deadline: Instant, - ) -> Result<(), DatabaseError> { - match self { - Self::Local(waiter) => waiter.wait(deadline).await, - Self::Distributed(waiter) => waiter.wait(deadline).await, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum ReleaseOutcome { - Released, - NotOwner, -} - /// Normalize only the requested lookup form's case. Raw IDs and aliases remain /// distinct lock identities and may therefore fill concurrently. pub(super) fn normalize_key(key: &str, case_sensitive: bool) -> String { diff --git a/apps/labrinth/src/database/redis/cache/locking/distributed.rs b/apps/labrinth/src/database/redis/cache/locking/distributed.rs deleted file mode 100644 index f43dd47816..0000000000 --- a/apps/labrinth/src/database/redis/cache/locking/distributed.rs +++ /dev/null @@ -1,220 +0,0 @@ -use std::time::Duration; - -use tokio::runtime::Handle; -use tokio::time::{Instant, sleep, timeout_at}; -use tokio_util::sync::CancellationToken; -use tracing::warn; - -use crate::database::models::DatabaseError; - -use super::super::super::commands; -use super::super::super::connection::RedisBackend; -use super::{ - LockAcquisition, LockTiming, LockWaiter, OwnedLockGuard, ReleaseOutcome, -}; - -#[derive(Clone)] -pub struct DistributedLockManager { - backend: RedisBackend, - timing: LockTiming, -} - -impl DistributedLockManager { - pub(super) fn new(backend: RedisBackend, timing: LockTiming) -> Self { - Self { backend, timing } - } - - pub(super) async fn acquire( - &self, - key: String, - ) -> Result { - let lock_key = cache_lock_key(&key); - let acquired = self.try_acquire(&lock_key).await?; - - if !acquired { - return Ok(LockAcquisition::Waiting(LockWaiter::Distributed( - DistributedLockWaiter { - manager: self.clone(), - lock_key, - }, - ))); - } - - let renewal_cancellation_token = CancellationToken::new(); - self.spawn_renewal( - lock_key.clone(), - renewal_cancellation_token.clone(), - ); - Ok(LockAcquisition::Owned(OwnedLockGuard::Distributed( - DistributedLockGuard { - manager: self.clone(), - lock_key, - renewal_cancellation_token, - cleanup_complete: false, - }, - ))) - } - - async fn try_acquire(&self, lock_key: &str) -> Result { - let mut connection = self.backend.connect().await?; - commands::acquire_lock( - &mut connection, - lock_key, - duration_millis(self.timing.lease), - ) - .await - } - - async fn renew(&self, lock_key: &str) -> Result { - let mut connection = self.backend.connect().await?; - commands::renew_lock( - &mut connection, - lock_key, - duration_millis(self.timing.lease), - ) - .await - } - - async fn release(&self, lock_key: &str) -> Result { - let mut connection = self.backend.connect().await?; - commands::release_lock(&mut connection, lock_key).await - } - - async fn exists(&self, lock_key: &str) -> Result { - let mut connection = self.backend.connect().await?; - commands::lock_exists(&mut connection, lock_key).await - } - - fn spawn_renewal( - &self, - lock_key: String, - cancellation_token: CancellationToken, - ) { - let manager = self.clone(); - tokio::spawn(async move { - loop { - let result = tokio::select! { - biased; - _ = cancellation_token.cancelled() => break, - result = async { - sleep(manager.timing.renewal).await; - manager.renew(&lock_key).await - } => result, - }; - - match result { - Ok(true) => {} - Ok(false) => break, - Err(_) => { - warn!("failed to renew distributed cache lease"); - break; - } - } - } - }); - } - - fn spawn_cleanup(&self, lock_key: String) { - let Ok(handle) = Handle::try_current() else { - return; - }; - let manager = self.clone(); - handle.spawn(async move { - if manager.release(&lock_key).await.is_err() { - warn!("failed to clean up distributed cache lease"); - } - }); - } -} - -pub struct DistributedLockGuard { - manager: DistributedLockManager, - lock_key: String, - renewal_cancellation_token: CancellationToken, - cleanup_complete: bool, -} - -impl DistributedLockGuard { - pub(super) async fn release( - mut self, - ) -> Result { - self.stop_renewal(); - match self.manager.release(&self.lock_key).await { - Ok(true) => { - self.cleanup_complete = true; - Ok(ReleaseOutcome::Released) - } - Ok(false) => { - self.cleanup_complete = true; - Ok(ReleaseOutcome::NotOwner) - } - Err(error) => Err(error), - } - } - - fn stop_renewal(&self) { - self.renewal_cancellation_token.cancel(); - } -} - -impl Drop for DistributedLockGuard { - fn drop(&mut self) { - self.stop_renewal(); - if self.cleanup_complete { - return; - } - - self.manager.spawn_cleanup(self.lock_key.clone()); - } -} - -pub struct DistributedLockWaiter { - manager: DistributedLockManager, - lock_key: String, -} - -impl DistributedLockWaiter { - pub(super) async fn wait( - self, - deadline: Instant, - ) -> Result<(), DatabaseError> { - let mut attempt = 0; - loop { - if !self.manager.exists(&self.lock_key).await? { - return Ok(()); - } - - let delay = poll_delay(self.manager.timing, attempt); - - timeout_at(deadline, sleep(delay)) - .await - .map_err(|_| lock_timeout())?; - attempt = attempt.saturating_add(1); - } - } -} - -fn cache_lock_key(key: &str) -> String { - format!("{key}/lock") -} - -fn poll_delay(timing: LockTiming, attempt: u32) -> Duration { - // With the defaults, delays are 50, 100, 200, 250 then 500 ms per poll. - timing - .poll_min - .saturating_mul(2_u32.saturating_pow(attempt)) - .min(timing.poll_max) -} - -fn duration_millis(duration: Duration) -> u64 { - duration.as_millis().min(u64::MAX as u128) as u64 -} - -fn lock_timeout() -> DatabaseError { - DatabaseError::CacheTimeout { - locks_released: 0, - locks_waiting: 1, - time_spent_pool_wait_ms: 0, - time_spent_total_ms: 0, - } -} diff --git a/apps/labrinth/src/database/redis/cache/locking/local.rs b/apps/labrinth/src/database/redis/cache/locking/local.rs index 7bab7c41b7..6d83106e88 100644 --- a/apps/labrinth/src/database/redis/cache/locking/local.rs +++ b/apps/labrinth/src/database/redis/cache/locking/local.rs @@ -9,28 +9,29 @@ use tokio::time::{Instant, timeout_at}; use crate::database::models::DatabaseError; #[derive(Clone)] -pub(in crate::database::redis::cache) struct LocalLockManager { - locks: Arc>>, +pub(in crate::database::redis::cache) struct LockCoordinator { + locks: Arc>>, } -impl LocalLockManager { - pub(super) fn new() -> Self { +impl LockCoordinator { + pub(in crate::database::redis::cache) fn new() -> Self { Self { locks: Arc::new(DashMap::with_capacity(2048)), } } - pub(super) fn acquire(&self, key: String) -> LocalLockAcquisition { + pub(in crate::database::redis::cache) fn acquire( + &self, + key: String, + ) -> LockAcquisition { match self.locks.entry(key.clone()) { - Entry::Occupied(entry) => { - LocalLockAcquisition::Waiting(LocalLockWaiter { - state: entry.get().clone(), - }) - } + Entry::Occupied(entry) => LockAcquisition::Waiting(LockWaiter { + state: entry.get().clone(), + }), Entry::Vacant(entry) => { - let state = Arc::new(LocalLockState::new()); + let state = Arc::new(LockState::new()); entry.insert(state.clone()); - LocalLockAcquisition::Owned(LocalLockGuard { + LockAcquisition::Owned(OwnedLockGuard { locks: self.locks.clone(), key, state, @@ -41,23 +42,19 @@ impl LocalLockManager { } } -pub(super) enum LocalLockAcquisition { - Owned(LocalLockGuard), - Waiting(LocalLockWaiter), +pub(in crate::database::redis::cache) enum LockAcquisition { + Owned(OwnedLockGuard), + Waiting(LockWaiter), } -pub(in crate::database::redis::cache) struct LocalLockGuard { - locks: Arc>>, +pub(in crate::database::redis::cache) struct OwnedLockGuard { + locks: Arc>>, key: String, - state: Arc, + state: Arc, released: bool, } -impl LocalLockGuard { - pub(super) fn release(mut self) { - self.release_inner(); - } - +impl OwnedLockGuard { fn release_inner(&mut self) { if self.released { return; @@ -71,18 +68,18 @@ impl LocalLockGuard { } } -impl Drop for LocalLockGuard { +impl Drop for OwnedLockGuard { fn drop(&mut self) { self.release_inner(); } } -pub(in crate::database::redis::cache) struct LocalLockWaiter { - state: Arc, +pub(in crate::database::redis::cache) struct LockWaiter { + state: Arc, } -impl LocalLockWaiter { - pub(super) async fn wait( +impl LockWaiter { + pub(in crate::database::redis::cache) async fn wait( self, deadline: Instant, ) -> Result<(), DatabaseError> { @@ -105,12 +102,12 @@ impl LocalLockWaiter { } } -struct LocalLockState { +struct LockState { released: AtomicBool, notify: Notify, } -impl LocalLockState { +impl LockState { fn new() -> Self { Self { released: AtomicBool::new(false), diff --git a/apps/labrinth/src/database/redis/commands.rs b/apps/labrinth/src/database/redis/commands.rs index c1cfbda4e7..e3c7125115 100644 --- a/apps/labrinth/src/database/redis/commands.rs +++ b/apps/labrinth/src/database/redis/commands.rs @@ -188,96 +188,3 @@ where { Ok(cmd("INCR").arg(key).query_async(connection).await?) } - -#[tracing::instrument( - name = "redis.cache_lock.acquire", - skip_all, - fields(command = "SET {} {} NX PX {}") -)] -pub(super) async fn acquire_lock( - connection: &mut C, - key: &str, - lease_ms: u64, -) -> Result -where - C: ConnectionLike, -{ - let response = acquire_lock_command(key, lease_ms) - .query_async::>(connection) - .await?; - Ok(response.is_some()) -} - -fn acquire_lock_command(key: &str, lease_ms: u64) -> redis::Cmd { - let mut command = redis::cmd("SET"); - command.arg(key).arg("").arg("NX").arg("PX").arg(lease_ms); - command -} - -#[tracing::instrument( - name = "redis.cache_lock.renew", - skip_all, - fields(command = "PEXPIRE {} {}") -)] -pub(super) async fn renew_lock( - connection: &mut C, - key: &str, - lease_ms: u64, -) -> Result -where - C: ConnectionLike, -{ - let renewed = renew_lock_command(key, lease_ms) - .query_async::(connection) - .await?; - Ok(renewed == 1) -} - -fn renew_lock_command(key: &str, lease_ms: u64) -> redis::Cmd { - let mut command = redis::cmd("PEXPIRE"); - command.arg(key).arg(lease_ms); - command -} - -#[tracing::instrument( - name = "redis.cache_lock.release", - skip_all, - fields(command = "DEL {}") -)] -pub(super) async fn release_lock( - connection: &mut C, - key: &str, -) -> Result -where - C: ConnectionLike, -{ - let released = release_lock_command(key) - .query_async::(connection) - .await?; - Ok(released == 1) -} - -fn release_lock_command(key: &str) -> redis::Cmd { - let mut command = redis::cmd("DEL"); - command.arg(key); - command -} - -#[tracing::instrument( - name = "redis.cache_lock.exists", - skip_all, - fields(command = "EXISTS {}") -)] -pub(super) async fn lock_exists( - connection: &mut C, - key: &str, -) -> Result -where - C: ConnectionLike, -{ - Ok(redis::cmd("EXISTS") - .arg(key) - .query_async::(connection) - .await? - != 0) -} diff --git a/apps/labrinth/src/database/redis/config.rs b/apps/labrinth/src/database/redis/config.rs index 6e60d2f15f..033c033843 100644 --- a/apps/labrinth/src/database/redis/config.rs +++ b/apps/labrinth/src/database/redis/config.rs @@ -151,6 +151,8 @@ pub(super) enum RedisConfigError { max: usize, min: usize, }, + #[error("unsupported Redis cache locking strategy `{strategy}`")] + UnsupportedCacheLockingStrategy { strategy: CacheLockingStrategy }, } impl RedisConfig { @@ -187,6 +189,12 @@ impl RedisConfig { blocking_pool_size: RedisPoolSize, cache_locking_strategy: CacheLockingStrategy, ) -> Result { + if cache_locking_strategy == CacheLockingStrategy::Distributed { + return Err(RedisConfigError::UnsupportedCacheLockingStrategy { + strategy: cache_locking_strategy, + }); + } + let seed_urls = raw_urls .split(',') .map(str::trim) diff --git a/apps/labrinth/src/database/redis/mod.rs b/apps/labrinth/src/database/redis/mod.rs index 2179fba109..82b133daa0 100644 --- a/apps/labrinth/src/database/redis/mod.rs +++ b/apps/labrinth/src/database/redis/mod.rs @@ -71,8 +71,6 @@ impl RedisPool { compression_min_savings_ratio: ENV .REDIS_COMPRESSION_MIN_SAVINGS_RATIO, }, - config.cache_locking_strategy(), - backend.clone(), ); Self {