From 996b3bc4ced3e190b05c186bfdf0b1baa457f893 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Mon, 28 Sep 2026 14:11:51 -0400 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 13 +-- Cargo.toml | 2 +- crates/sqlite-ffi/src/client_data.rs | 148 --------------------------- crates/sqlite-ffi/src/lib.rs | 1 - sqlite-compress/src/lib.rs | 4 + sqlite-compress/src/state.rs | 39 +++++-- 6 files changed, 40 insertions(+), 167 deletions(-) delete mode 100644 crates/sqlite-ffi/src/client_data.rs diff --git a/Cargo.lock b/Cargo.lock index 851706e..e71655d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,14 +70,15 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libsqlite3-sys" -version = "0.38.1" -source = "git+https://github.com/rusqlite/rusqlite?rev=91f876c#91f876c80114122670f455190d03c81d1f78b0af" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", "prettyplease", "quote", - "syn 3.0.5", + "syn 2.0.119", "vcpkg", ] @@ -127,12 +128,12 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "prettyplease" -version = "0.3.0" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4c2d4da..101dfd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,6 @@ resolver = "2" [workspace.dependencies] thiserror = "2.0" -libsqlite3-sys = { git = "https://github.com/rusqlite/rusqlite", rev = "91f876c",package = "libsqlite3-sys" } +libsqlite3-sys = { version = "=0.30.1", features = ["bundled"], package = "libsqlite3-sys" } sqlite-ffi = { version = "0.1.0", path = "crates/sqlite-ffi", default-features = false } sqlite-ffi-macros = { version = "0.1.0", path = "crates/sqlite-ffi-macros" } \ No newline at end of file diff --git a/crates/sqlite-ffi/src/client_data.rs b/crates/sqlite-ffi/src/client_data.rs deleted file mode 100644 index 7797e84..0000000 --- a/crates/sqlite-ffi/src/client_data.rs +++ /dev/null @@ -1,148 +0,0 @@ -use std::ffi::{CString, c_char, c_void}; - -use libsqlite3_sys::{SQLITE_OK, sqlite3, sqlite3_get_clientdata, sqlite3_set_clientdata}; - -use crate::{ - Connection, - error::SqliteError, - utils::{XDestroy, ptr_as_ref, to_sqlite_destroy}, -}; - -impl Connection { - /// # Panics - /// - /// Panics if the client data pointer cannot be cast to `&T` - pub fn get_client_data<'a, T>(&'a self, name: &str) -> Result<&'a T, SqliteError> { - let c_name = CString::new(name)?; - - let client = unsafe { get_client_data_raw(self.conn.as_ptr(), c_name.as_ptr()) }; - - if client.is_null() { - return Err(SqliteError::PointerNotValid(format!( - "client data for name: {}", - name - ))); - } - - let client = unsafe { ptr_as_ref::(client) }; - - Ok(client.expect("can't cast")) - } - - pub fn set_client_data(&self, name: &str, value: T) -> Result<(), SqliteError> { - let c_name = CString::new(name)?; - - let pointer = Box::into_raw(Box::new(value)); - - let (p, x_destroy) = unsafe { to_sqlite_destroy(pointer) }; - - let rc = unsafe { set_client_data_raw(self.conn.as_ptr(), c_name.as_ptr(), p, x_destroy) }; - - if rc != SQLITE_OK { - return Err(SqliteError::Sqlite { - operation: "set client data", - code: rc, - message: format!("cant't set client data for name: {}", name), - }); - } - - Ok(()) - } -} - -unsafe fn get_client_data_raw(db: *mut sqlite3, name: *const c_char) -> *mut c_void { - unsafe { sqlite3_get_clientdata(db, name) } -} - -unsafe fn set_client_data_raw( - db: *mut sqlite3, - name: *const c_char, - p: *mut c_void, - x_destroy: XDestroy, -) -> i32 { - unsafe { sqlite3_set_clientdata(db, name, p, x_destroy) } -} - -#[cfg(test)] -mod tests { - use std::{ - ptr, - sync::atomic::{AtomicBool, Ordering}, - }; - - use libsqlite3_sys::{sqlite3_close, sqlite3_open}; - - use super::*; - - struct Counter { - value: u32, - dropped: &'static AtomicBool, - } - - impl Drop for Counter { - fn drop(&mut self) { - self.dropped.store(true, Ordering::SeqCst); - } - } - - #[test] - fn destroy_drops_client_data_once() { - static DROPPED: AtomicBool = AtomicBool::new(false); - - let (ptr, destroy) = unsafe { - to_sqlite_destroy(Box::into_raw(Box::new(Counter { - value: 7, - dropped: &DROPPED, - }))) - }; - - assert!(!DROPPED.load(Ordering::SeqCst)); - unsafe { destroy.unwrap()(ptr) }; - assert!(DROPPED.load(Ordering::SeqCst)); - } - - #[test] - fn destroy_ignores_null_pointer() { - let (_ptr, destroy) = unsafe { to_sqlite_destroy::(ptr::null_mut()) }; - unsafe { destroy.unwrap()(ptr::null_mut()) }; - } - - #[test] - #[cfg_attr(miri, ignore)] - fn set_then_get_returns_same_value() { - static DROPPED: AtomicBool = AtomicBool::new(false); - - let mut db = ptr::null_mut(); - assert_eq!(unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, 0); - - let db = Connection::from_raw(db); - - let p = Box::into_raw(Box::new(Counter { - value: 42, - dropped: &DROPPED, - })); - db.set_client_data("demo", p).unwrap(); - - let got = db.get_client_data::("demo").unwrap(); - assert_eq!(got.value, 42); - - assert!(!DROPPED.load(Ordering::SeqCst)); - - unsafe { sqlite3_close(db.conn.as_ptr()) }; - - assert!(DROPPED.load(Ordering::SeqCst)); - } - - #[test] - #[cfg_attr(miri, ignore)] - fn get_missing_name_is_error() { - let mut db = ptr::null_mut(); - assert_eq!(unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, 0); - - let db = Connection::from_raw(db); - - assert!(db.get_client_data::("missing").is_err()); - - unsafe { sqlite3_close(db.conn.as_ptr()) }; - } -} diff --git a/crates/sqlite-ffi/src/lib.rs b/crates/sqlite-ffi/src/lib.rs index 89eef71..cd222a1 100644 --- a/crates/sqlite-ffi/src/lib.rs +++ b/crates/sqlite-ffi/src/lib.rs @@ -1,4 +1,3 @@ -mod client_data; mod connection; mod context; mod error; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index 4311087..cbe02e6 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -30,3 +30,7 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> Ok(()) } + +pub fn sqlite3_compress_destroy(connection: &Connection) { + ExtensionState::detach(connection); +} diff --git a/sqlite-compress/src/state.rs b/sqlite-compress/src/state.rs index ea55f82..3237474 100644 --- a/sqlite-compress/src/state.rs +++ b/sqlite-compress/src/state.rs @@ -1,4 +1,7 @@ -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::{Arc, LazyLock}, +}; use libsqlite3_sys::sqlite3; use parking_lot::Mutex; @@ -6,6 +9,9 @@ use sqlite_ffi::{Connection, Context, SqliteError}; use crate::cache::DictCache; +static CACHES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + /// Per-connection state for the extension: the SQLite connection and its dictionary cache. /// /// `sqlite3_compress_init` calls [`ExtensionState::attach`], which stores an empty [`Cache`] in the connection's client data. @@ -32,25 +38,36 @@ impl AsRef for ExtensionState { } impl ExtensionState { - const NAME: &str = "sqlite-compress"; - pub fn attach(connection: &Connection) -> Result<(), SqliteError> { let cache = Cache(Arc::new(Mutex::new(DictCache::new()))); + CACHES.lock().insert(connection.as_ptr() as usize, cache); - connection.set_client_data(Self::NAME, cache) + Ok(()) } - pub fn from_db(db: *mut sqlite3) -> Result { - let connection = Connection::from_raw(db); - let cache = connection.get_client_data::(Self::NAME)?.clone(); + pub fn detach(connection: &Connection) { + CACHES.lock().remove(&(connection.as_ptr() as usize)); + } + + fn lookup(db: *mut sqlite3) -> Result { + let cache = CACHES + .lock() + .get(&(db as usize)) + .cloned() + .ok_or_else(|| SqliteError::PointerNotValid("compress cache".into()))?; - Ok(Self { cache, connection }) + Ok(Self { + connection: Connection::from_raw(db), + cache, + }) + } + + pub fn from_db(db: *mut sqlite3) -> Result { + Self::lookup(db) } pub fn from_context(context: &Context) -> Result { let connection = Connection::from_context(context)?; - let cache = connection.get_client_data::(Self::NAME)?.clone(); - - Ok(Self { cache, connection }) + Self::lookup(connection.as_ptr()) } } From 3406f9a5c4f0a206f7089359f612c578266038a5 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Mon, 28 Sep 2026 14:11:51 -0400 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/sqlite-ffi/src/client_data.rs | 148 --------------------------- crates/sqlite-ffi/src/lib.rs | 1 - sqlite-compress/src/lib.rs | 4 + sqlite-compress/src/state.rs | 39 +++++-- 4 files changed, 32 insertions(+), 160 deletions(-) delete mode 100644 crates/sqlite-ffi/src/client_data.rs diff --git a/crates/sqlite-ffi/src/client_data.rs b/crates/sqlite-ffi/src/client_data.rs deleted file mode 100644 index 7797e84..0000000 --- a/crates/sqlite-ffi/src/client_data.rs +++ /dev/null @@ -1,148 +0,0 @@ -use std::ffi::{CString, c_char, c_void}; - -use libsqlite3_sys::{SQLITE_OK, sqlite3, sqlite3_get_clientdata, sqlite3_set_clientdata}; - -use crate::{ - Connection, - error::SqliteError, - utils::{XDestroy, ptr_as_ref, to_sqlite_destroy}, -}; - -impl Connection { - /// # Panics - /// - /// Panics if the client data pointer cannot be cast to `&T` - pub fn get_client_data<'a, T>(&'a self, name: &str) -> Result<&'a T, SqliteError> { - let c_name = CString::new(name)?; - - let client = unsafe { get_client_data_raw(self.conn.as_ptr(), c_name.as_ptr()) }; - - if client.is_null() { - return Err(SqliteError::PointerNotValid(format!( - "client data for name: {}", - name - ))); - } - - let client = unsafe { ptr_as_ref::(client) }; - - Ok(client.expect("can't cast")) - } - - pub fn set_client_data(&self, name: &str, value: T) -> Result<(), SqliteError> { - let c_name = CString::new(name)?; - - let pointer = Box::into_raw(Box::new(value)); - - let (p, x_destroy) = unsafe { to_sqlite_destroy(pointer) }; - - let rc = unsafe { set_client_data_raw(self.conn.as_ptr(), c_name.as_ptr(), p, x_destroy) }; - - if rc != SQLITE_OK { - return Err(SqliteError::Sqlite { - operation: "set client data", - code: rc, - message: format!("cant't set client data for name: {}", name), - }); - } - - Ok(()) - } -} - -unsafe fn get_client_data_raw(db: *mut sqlite3, name: *const c_char) -> *mut c_void { - unsafe { sqlite3_get_clientdata(db, name) } -} - -unsafe fn set_client_data_raw( - db: *mut sqlite3, - name: *const c_char, - p: *mut c_void, - x_destroy: XDestroy, -) -> i32 { - unsafe { sqlite3_set_clientdata(db, name, p, x_destroy) } -} - -#[cfg(test)] -mod tests { - use std::{ - ptr, - sync::atomic::{AtomicBool, Ordering}, - }; - - use libsqlite3_sys::{sqlite3_close, sqlite3_open}; - - use super::*; - - struct Counter { - value: u32, - dropped: &'static AtomicBool, - } - - impl Drop for Counter { - fn drop(&mut self) { - self.dropped.store(true, Ordering::SeqCst); - } - } - - #[test] - fn destroy_drops_client_data_once() { - static DROPPED: AtomicBool = AtomicBool::new(false); - - let (ptr, destroy) = unsafe { - to_sqlite_destroy(Box::into_raw(Box::new(Counter { - value: 7, - dropped: &DROPPED, - }))) - }; - - assert!(!DROPPED.load(Ordering::SeqCst)); - unsafe { destroy.unwrap()(ptr) }; - assert!(DROPPED.load(Ordering::SeqCst)); - } - - #[test] - fn destroy_ignores_null_pointer() { - let (_ptr, destroy) = unsafe { to_sqlite_destroy::(ptr::null_mut()) }; - unsafe { destroy.unwrap()(ptr::null_mut()) }; - } - - #[test] - #[cfg_attr(miri, ignore)] - fn set_then_get_returns_same_value() { - static DROPPED: AtomicBool = AtomicBool::new(false); - - let mut db = ptr::null_mut(); - assert_eq!(unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, 0); - - let db = Connection::from_raw(db); - - let p = Box::into_raw(Box::new(Counter { - value: 42, - dropped: &DROPPED, - })); - db.set_client_data("demo", p).unwrap(); - - let got = db.get_client_data::("demo").unwrap(); - assert_eq!(got.value, 42); - - assert!(!DROPPED.load(Ordering::SeqCst)); - - unsafe { sqlite3_close(db.conn.as_ptr()) }; - - assert!(DROPPED.load(Ordering::SeqCst)); - } - - #[test] - #[cfg_attr(miri, ignore)] - fn get_missing_name_is_error() { - let mut db = ptr::null_mut(); - assert_eq!(unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, 0); - - let db = Connection::from_raw(db); - - assert!(db.get_client_data::("missing").is_err()); - - unsafe { sqlite3_close(db.conn.as_ptr()) }; - } -} diff --git a/crates/sqlite-ffi/src/lib.rs b/crates/sqlite-ffi/src/lib.rs index f23b155..7a7936e 100644 --- a/crates/sqlite-ffi/src/lib.rs +++ b/crates/sqlite-ffi/src/lib.rs @@ -1,4 +1,3 @@ -mod client_data; mod connection; mod context; mod error; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index 4311087..cbe02e6 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -30,3 +30,7 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> Ok(()) } + +pub fn sqlite3_compress_destroy(connection: &Connection) { + ExtensionState::detach(connection); +} diff --git a/sqlite-compress/src/state.rs b/sqlite-compress/src/state.rs index ea55f82..3237474 100644 --- a/sqlite-compress/src/state.rs +++ b/sqlite-compress/src/state.rs @@ -1,4 +1,7 @@ -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::{Arc, LazyLock}, +}; use libsqlite3_sys::sqlite3; use parking_lot::Mutex; @@ -6,6 +9,9 @@ use sqlite_ffi::{Connection, Context, SqliteError}; use crate::cache::DictCache; +static CACHES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + /// Per-connection state for the extension: the SQLite connection and its dictionary cache. /// /// `sqlite3_compress_init` calls [`ExtensionState::attach`], which stores an empty [`Cache`] in the connection's client data. @@ -32,25 +38,36 @@ impl AsRef for ExtensionState { } impl ExtensionState { - const NAME: &str = "sqlite-compress"; - pub fn attach(connection: &Connection) -> Result<(), SqliteError> { let cache = Cache(Arc::new(Mutex::new(DictCache::new()))); + CACHES.lock().insert(connection.as_ptr() as usize, cache); - connection.set_client_data(Self::NAME, cache) + Ok(()) } - pub fn from_db(db: *mut sqlite3) -> Result { - let connection = Connection::from_raw(db); - let cache = connection.get_client_data::(Self::NAME)?.clone(); + pub fn detach(connection: &Connection) { + CACHES.lock().remove(&(connection.as_ptr() as usize)); + } + + fn lookup(db: *mut sqlite3) -> Result { + let cache = CACHES + .lock() + .get(&(db as usize)) + .cloned() + .ok_or_else(|| SqliteError::PointerNotValid("compress cache".into()))?; - Ok(Self { cache, connection }) + Ok(Self { + connection: Connection::from_raw(db), + cache, + }) + } + + pub fn from_db(db: *mut sqlite3) -> Result { + Self::lookup(db) } pub fn from_context(context: &Context) -> Result { let connection = Connection::from_context(context)?; - let cache = connection.get_client_data::(Self::NAME)?.clone(); - - Ok(Self { cache, connection }) + Self::lookup(connection.as_ptr()) } } From a4a73964b016b93bb59b06c6cd03bd20c9e33662 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Tue, 6 Oct 2026 14:13:05 +0200 Subject: [PATCH 3/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Set=20cache=20global?= =?UTF-8?q?=20for=20encoders/decoders=20remove=20LRU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .mise/tasks/sqlite-compress/test | 2 +- crates/sqlite-ffi/src/client_data.rs | 148 ++++++++++++++++++++++++ crates/sqlite-ffi/src/lib.rs | 1 + sqlite-compress/src/cache/connection.rs | 8 +- sqlite-compress/src/cache/lru.rs | 124 -------------------- sqlite-compress/src/cache/mod.rs | 2 +- sqlite-compress/src/cache/pool.rs | 92 +++++++++++++++ sqlite-compress/src/cache/prepared.rs | 51 +------- sqlite-compress/src/lib.rs | 4 - sqlite-compress/src/state.rs | 40 ++----- 10 files changed, 260 insertions(+), 212 deletions(-) create mode 100644 crates/sqlite-ffi/src/client_data.rs delete mode 100644 sqlite-compress/src/cache/lru.rs create mode 100644 sqlite-compress/src/cache/pool.rs diff --git a/.mise/tasks/sqlite-compress/test b/.mise/tasks/sqlite-compress/test index 4df439e..12f2954 100755 --- a/.mise/tasks/sqlite-compress/test +++ b/.mise/tasks/sqlite-compress/test @@ -1,3 +1,3 @@ #!/usr/bin/env bash #MISE description="Run tests" -cargo test -p sqlite-compress +cargo test -p sqlite-compress -- --test-threads=1 diff --git a/crates/sqlite-ffi/src/client_data.rs b/crates/sqlite-ffi/src/client_data.rs new file mode 100644 index 0000000..7797e84 --- /dev/null +++ b/crates/sqlite-ffi/src/client_data.rs @@ -0,0 +1,148 @@ +use std::ffi::{CString, c_char, c_void}; + +use libsqlite3_sys::{SQLITE_OK, sqlite3, sqlite3_get_clientdata, sqlite3_set_clientdata}; + +use crate::{ + Connection, + error::SqliteError, + utils::{XDestroy, ptr_as_ref, to_sqlite_destroy}, +}; + +impl Connection { + /// # Panics + /// + /// Panics if the client data pointer cannot be cast to `&T` + pub fn get_client_data<'a, T>(&'a self, name: &str) -> Result<&'a T, SqliteError> { + let c_name = CString::new(name)?; + + let client = unsafe { get_client_data_raw(self.conn.as_ptr(), c_name.as_ptr()) }; + + if client.is_null() { + return Err(SqliteError::PointerNotValid(format!( + "client data for name: {}", + name + ))); + } + + let client = unsafe { ptr_as_ref::(client) }; + + Ok(client.expect("can't cast")) + } + + pub fn set_client_data(&self, name: &str, value: T) -> Result<(), SqliteError> { + let c_name = CString::new(name)?; + + let pointer = Box::into_raw(Box::new(value)); + + let (p, x_destroy) = unsafe { to_sqlite_destroy(pointer) }; + + let rc = unsafe { set_client_data_raw(self.conn.as_ptr(), c_name.as_ptr(), p, x_destroy) }; + + if rc != SQLITE_OK { + return Err(SqliteError::Sqlite { + operation: "set client data", + code: rc, + message: format!("cant't set client data for name: {}", name), + }); + } + + Ok(()) + } +} + +unsafe fn get_client_data_raw(db: *mut sqlite3, name: *const c_char) -> *mut c_void { + unsafe { sqlite3_get_clientdata(db, name) } +} + +unsafe fn set_client_data_raw( + db: *mut sqlite3, + name: *const c_char, + p: *mut c_void, + x_destroy: XDestroy, +) -> i32 { + unsafe { sqlite3_set_clientdata(db, name, p, x_destroy) } +} + +#[cfg(test)] +mod tests { + use std::{ + ptr, + sync::atomic::{AtomicBool, Ordering}, + }; + + use libsqlite3_sys::{sqlite3_close, sqlite3_open}; + + use super::*; + + struct Counter { + value: u32, + dropped: &'static AtomicBool, + } + + impl Drop for Counter { + fn drop(&mut self) { + self.dropped.store(true, Ordering::SeqCst); + } + } + + #[test] + fn destroy_drops_client_data_once() { + static DROPPED: AtomicBool = AtomicBool::new(false); + + let (ptr, destroy) = unsafe { + to_sqlite_destroy(Box::into_raw(Box::new(Counter { + value: 7, + dropped: &DROPPED, + }))) + }; + + assert!(!DROPPED.load(Ordering::SeqCst)); + unsafe { destroy.unwrap()(ptr) }; + assert!(DROPPED.load(Ordering::SeqCst)); + } + + #[test] + fn destroy_ignores_null_pointer() { + let (_ptr, destroy) = unsafe { to_sqlite_destroy::(ptr::null_mut()) }; + unsafe { destroy.unwrap()(ptr::null_mut()) }; + } + + #[test] + #[cfg_attr(miri, ignore)] + fn set_then_get_returns_same_value() { + static DROPPED: AtomicBool = AtomicBool::new(false); + + let mut db = ptr::null_mut(); + assert_eq!(unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, 0); + + let db = Connection::from_raw(db); + + let p = Box::into_raw(Box::new(Counter { + value: 42, + dropped: &DROPPED, + })); + db.set_client_data("demo", p).unwrap(); + + let got = db.get_client_data::("demo").unwrap(); + assert_eq!(got.value, 42); + + assert!(!DROPPED.load(Ordering::SeqCst)); + + unsafe { sqlite3_close(db.conn.as_ptr()) }; + + assert!(DROPPED.load(Ordering::SeqCst)); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn get_missing_name_is_error() { + let mut db = ptr::null_mut(); + assert_eq!(unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, 0); + + let db = Connection::from_raw(db); + + assert!(db.get_client_data::("missing").is_err()); + + unsafe { sqlite3_close(db.conn.as_ptr()) }; + } +} diff --git a/crates/sqlite-ffi/src/lib.rs b/crates/sqlite-ffi/src/lib.rs index 7a7936e..f23b155 100644 --- a/crates/sqlite-ffi/src/lib.rs +++ b/crates/sqlite-ffi/src/lib.rs @@ -1,3 +1,4 @@ +mod client_data; mod connection; mod context; mod error; diff --git a/sqlite-compress/src/cache/connection.rs b/sqlite-compress/src/cache/connection.rs index 91c4cc2..510e736 100644 --- a/sqlite-compress/src/cache/connection.rs +++ b/sqlite-compress/src/cache/connection.rs @@ -1,8 +1,6 @@ use std::collections::HashMap; -use zstd::dict::{DecoderDictionary, EncoderDictionary}; - -use crate::{ColumnKey, DictId, cache::lru::DictLru}; +use crate::{ColumnKey, DictId}; /// Per-connection cache of prepared zstd dictionaries and the current dict id per column. /// /// `encoders` and `decoders` are independent LRUs keyed by [`DictKey`] (schema + dict id). @@ -13,16 +11,12 @@ use crate::{ColumnKey, DictId, cache::lru::DictLru}; /// A missing id is treated as raw zstd (`DictId` 0). pub(crate) struct DictCache { current_ids: HashMap, - pub(crate) encoders: DictLru>, - pub(crate) decoders: DictLru>, } impl DictCache { pub(crate) fn new() -> Self { Self { current_ids: HashMap::new(), - encoders: DictLru::new(), - decoders: DictLru::new(), } } diff --git a/sqlite-compress/src/cache/lru.rs b/sqlite-compress/src/cache/lru.rs deleted file mode 100644 index 560e542..0000000 --- a/sqlite-compress/src/cache/lru.rs +++ /dev/null @@ -1,124 +0,0 @@ -use std::{ - collections::{HashMap, VecDeque, hash_map::Entry}, - sync::Arc, -}; - -use crate::dict::DictKey; - -const MAX_CACHED_DICTS: usize = 5; - -/// Bounds the memory used by prepared dictionaries for one SQLite connection. -/// -/// Access through [`DictLru::get`] refreshes recency because the dictionary was -/// used for a codec operation. [`DictLru::peek`] deliberately does not, which -/// lets the registry check for an entry without changing eviction order. -pub(crate) struct DictLru { - map: HashMap>, - order: VecDeque, -} - -impl DictLru { - pub(crate) fn new() -> Self { - Self { - map: HashMap::with_capacity(MAX_CACHED_DICTS), - order: VecDeque::with_capacity(MAX_CACHED_DICTS), - } - } - - pub(crate) fn get(&mut self, key: &DictKey) -> Option> { - let value = self.map.get(key)?.clone(); - self.touch(key); - Some(value) - } - - pub(crate) fn peek(&self, key: &DictKey) -> Option> { - self.map.get(key).cloned() - } - - pub(crate) fn insert(&mut self, key: DictKey, value: Arc) { - if let Entry::Occupied(mut e) = self.map.entry(key.clone()) { - e.insert(value); - self.touch(&key); - return; - } - - while self.map.len() >= MAX_CACHED_DICTS { - if let Some(evicted) = self.order.pop_front() { - self.map.remove(&evicted); - } - } - self.map.insert(key.clone(), value); - self.order.push_back(key); - } - - fn touch(&mut self, key: &DictKey) { - self.order.retain(|x| x != key); - self.order.push_back(key.clone()); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dict::DictId; - - fn key(schema: &str, n: u32) -> DictKey { - DictKey::new(schema, DictId::new(n)) - } - - #[test] - fn evicts_oldest_after_five() { - let mut cache = DictLru::new(); - for n in 1..=5 { - cache.insert(key("main", n), Arc::new(n)); - } - cache.insert(key("main", 6), Arc::new(6)); - assert!(cache.peek(&key("main", 1)).is_none()); - assert!(cache.peek(&key("main", 2)).is_some()); - assert!(cache.peek(&key("main", 6)).is_some()); - } - - #[test] - fn get_promotes_to_most_recent() { - let mut cache = DictLru::new(); - for n in 1..=5 { - cache.insert(key("main", n), Arc::new(n)); - } - assert!(cache.get(&key("main", 1)).is_some()); - cache.insert(key("main", 6), Arc::new(6)); - assert!(cache.peek(&key("main", 1)).is_some()); - assert!(cache.peek(&key("main", 2)).is_none()); - } - - #[test] - fn peek_does_not_promote() { - let mut cache = DictLru::new(); - for n in 1..=5 { - cache.insert(key("main", n), Arc::new(n)); - } - assert!(cache.peek(&key("main", 1)).is_some()); - cache.insert(key("main", 6), Arc::new(6)); - assert!(cache.peek(&key("main", 1)).is_none()); - } - - #[test] - fn reinsert_updates_value_and_promotes() { - let mut cache = DictLru::new(); - for n in 1..=5 { - cache.insert(key("main", n), Arc::new(n)); - } - cache.insert(key("main", 1), Arc::new(100)); - cache.insert(key("main", 6), Arc::new(6)); - assert_eq!(*cache.peek(&key("main", 1)).unwrap(), 100); - assert!(cache.peek(&key("main", 2)).is_none()); - } - - #[test] - fn same_id_different_schema_are_distinct() { - let mut cache = DictLru::new(); - cache.insert(key("raw", 1), Arc::new(10)); - cache.insert(key("archive", 1), Arc::new(20)); - assert_eq!(*cache.peek(&key("raw", 1)).unwrap(), 10); - assert_eq!(*cache.peek(&key("archive", 1)).unwrap(), 20); - } -} diff --git a/sqlite-compress/src/cache/mod.rs b/sqlite-compress/src/cache/mod.rs index 220fabd..c34c645 100644 --- a/sqlite-compress/src/cache/mod.rs +++ b/sqlite-compress/src/cache/mod.rs @@ -1,5 +1,5 @@ mod connection; -mod lru; +mod pool; mod prepared; pub(crate) use connection::DictCache; diff --git a/sqlite-compress/src/cache/pool.rs b/sqlite-compress/src/cache/pool.rs new file mode 100644 index 0000000..1abd742 --- /dev/null +++ b/sqlite-compress/src/cache/pool.rs @@ -0,0 +1,92 @@ +use std::{ + collections::{HashMap, hash_map::Entry}, + sync::{Arc, LazyLock}, +}; + +use parking_lot::Mutex; +use zstd::dict::{DecoderDictionary, EncoderDictionary}; + +use crate::{dict::DictKey, functions::Level}; + +const MAX_PREPARED_DICTS: usize = 64; + +type PreparedMap = HashMap>; + +static ENCODERS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::with_capacity(MAX_PREPARED_DICTS))); + +static DECODERS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::with_capacity(MAX_PREPARED_DICTS))); + +pub(crate) fn get_encoder( + key: &DictKey, + level: Level, + load: impl FnOnce() -> Result, crate::DictError>, +) -> Result>, crate::DictError> { + if let Some(dict) = ENCODERS.lock().get(key).cloned() { + return Ok(dict); + } + + let raw = load()?; + + let mut cache = ENCODERS.lock(); + if let Some(dict) = cache.get(key).cloned() { + return Ok(dict); + } + + let encoder = Arc::new(EncoderDictionary::copy(&raw, level.get())); + cache.insert(key.clone(), encoder.clone()); + Ok(encoder) +} + +pub(crate) fn get_decoder( + key: &DictKey, + load: impl FnOnce() -> Result, crate::DictError>, +) -> Result>, crate::DictError> { + if let Some(dict) = DECODERS.lock().get(key).cloned() { + return Ok(dict); + } + + let raw = load()?; + + let mut cache = DECODERS.lock(); + if let Some(dict) = cache.get(key).cloned() { + return Ok(dict); + } + + let decoder = Arc::new(DecoderDictionary::copy(&raw)); + cache.insert(key.clone(), decoder.clone()); + Ok(decoder) +} + +fn insert_capped(map: &mut PreparedMap, key: DictKey, value: Arc) { + let at_cap = map.len() >= MAX_PREPARED_DICTS; + + match map.entry(key) { + Entry::Occupied(mut e) => { + e.insert(value); + } + Entry::Vacant(e) => { + if at_cap { + let key = e.into_key(); + map.clear(); + map.insert(key, value); + } else { + e.insert(value); + } + } + } +} + +pub(crate) fn insert_prepared(key: DictKey, dictionary: &[u8], level: Level) { + insert_capped( + &mut ENCODERS.lock(), + key.clone(), + Arc::new(EncoderDictionary::copy(dictionary, level.get())), + ); + insert_capped( + &mut DECODERS.lock(), + key, + Arc::new(DecoderDictionary::copy(dictionary)), + ); +} diff --git a/sqlite-compress/src/cache/prepared.rs b/sqlite-compress/src/cache/prepared.rs index 65b0eac..89337b6 100644 --- a/sqlite-compress/src/cache/prepared.rs +++ b/sqlite-compress/src/cache/prepared.rs @@ -4,6 +4,7 @@ use zstd::dict::{DecoderDictionary, EncoderDictionary}; use crate::{ Cache, ColumnKey, DictError, DictId, + cache::pool, dict::{DictKey, get_raw_dict, read_current_id}, functions::Level, state::ExtensionState, @@ -18,22 +19,7 @@ pub(crate) fn get_encoder_in_cache( ) -> Result>, DictError> { let key = DictKey::new(schema, dict_id); - if let Some(dict) = state.cache.lock().encoders.peek(&key) { - return Ok(dict); - } - - let raw = get_raw_dict(&key, state)?; - - let mut cache = state.cache.lock(); - - if let Some(dict) = cache.encoders.get(&key) { - return Ok(dict); - } - - let encoder = Arc::new(EncoderDictionary::copy(&raw, level.get())); - cache.encoders.insert(key, encoder.clone()); - - Ok(encoder) + pool::get_encoder(&key, level, || get_raw_dict(&key, state)) } /// Return the prepared zstd decoder for `(schema, dict_id)`, loading it from `__compress_dicts` and caching it on a miss. @@ -44,22 +30,7 @@ pub(crate) fn get_decoder_in_cache( ) -> Result>, DictError> { let key = DictKey::new(schema, dict_id); - if let Some(dict) = state.cache.lock().decoders.peek(&key) { - return Ok(dict); - } - - let raw = get_raw_dict(&key, state)?; - - let mut cache = state.cache.lock(); - - if let Some(dict) = cache.decoders.get(&key) { - return Ok(dict); - } - - let decoder = Arc::new(DecoderDictionary::copy(&raw)); - cache.decoders.insert(key, decoder.clone()); - - Ok(decoder) + pool::get_decoder(&key, || get_raw_dict(&key, state)) } /// Load this column's current dict id, encoder, and decoder. Call it as early as possible so later `compress` and `decompress` hit the cache. @@ -91,20 +62,8 @@ pub(crate) fn insert_into_caches( dictionary: &[u8], level: Level, ) { - let mut cache = cache.lock(); - - cache.set_current_id(column.clone(), dict_id); - - let key = DictKey::new(column.schema(), dict_id); - - cache.encoders.insert( - key.clone(), - Arc::new(EncoderDictionary::copy(dictionary, level.get())), - ); - - cache - .decoders - .insert(key, Arc::new(DecoderDictionary::copy(dictionary))); + cache.lock().set_current_id(column.clone(), dict_id); + pool::insert_prepared(DictKey::new(column.schema(), dict_id), dictionary, level); } #[cfg(test)] diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index cbe02e6..4311087 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -30,7 +30,3 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> Ok(()) } - -pub fn sqlite3_compress_destroy(connection: &Connection) { - ExtensionState::detach(connection); -} diff --git a/sqlite-compress/src/state.rs b/sqlite-compress/src/state.rs index 3237474..e4f3b8c 100644 --- a/sqlite-compress/src/state.rs +++ b/sqlite-compress/src/state.rs @@ -1,7 +1,4 @@ -use std::{ - collections::HashMap, - sync::{Arc, LazyLock}, -}; +use std::sync::Arc; use libsqlite3_sys::sqlite3; use parking_lot::Mutex; @@ -9,9 +6,6 @@ use sqlite_ffi::{Connection, Context, SqliteError}; use crate::cache::DictCache; -static CACHES: LazyLock>> = - LazyLock::new(|| Mutex::new(HashMap::new())); - /// Per-connection state for the extension: the SQLite connection and its dictionary cache. /// /// `sqlite3_compress_init` calls [`ExtensionState::attach`], which stores an empty [`Cache`] in the connection's client data. @@ -38,36 +32,24 @@ impl AsRef for ExtensionState { } impl ExtensionState { + const NAME: &str = "sqlite-compress"; + pub fn attach(connection: &Connection) -> Result<(), SqliteError> { let cache = Cache(Arc::new(Mutex::new(DictCache::new()))); - CACHES.lock().insert(connection.as_ptr() as usize, cache); - - Ok(()) - } - - pub fn detach(connection: &Connection) { - CACHES.lock().remove(&(connection.as_ptr() as usize)); - } - - fn lookup(db: *mut sqlite3) -> Result { - let cache = CACHES - .lock() - .get(&(db as usize)) - .cloned() - .ok_or_else(|| SqliteError::PointerNotValid("compress cache".into()))?; - - Ok(Self { - connection: Connection::from_raw(db), - cache, - }) + connection.set_client_data(Self::NAME, cache) } pub fn from_db(db: *mut sqlite3) -> Result { - Self::lookup(db) + let connection = Connection::from_raw(db); + let cache = connection.get_client_data::(Self::NAME)?.clone(); + + Ok(Self { cache, connection }) } pub fn from_context(context: &Context) -> Result { let connection = Connection::from_context(context)?; - Self::lookup(connection.as_ptr()) + let cache = connection.get_client_data::(Self::NAME)?.clone(); + + Ok(Self { cache, connection }) } } From 6efae749f10f4251632ce6db3483bdfe8b039e3d Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Tue, 6 Oct 2026 15:19:41 +0200 Subject: [PATCH 4/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Set=20correct=20insert?= =?UTF-8?q?=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sqlite-compress/src/cache/pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite-compress/src/cache/pool.rs b/sqlite-compress/src/cache/pool.rs index 1abd742..9c4dd57 100644 --- a/sqlite-compress/src/cache/pool.rs +++ b/sqlite-compress/src/cache/pool.rs @@ -35,7 +35,7 @@ pub(crate) fn get_encoder( } let encoder = Arc::new(EncoderDictionary::copy(&raw, level.get())); - cache.insert(key.clone(), encoder.clone()); + insert_capped(&mut cache, key.clone(), encoder.clone()); Ok(encoder) } @@ -55,7 +55,7 @@ pub(crate) fn get_decoder( } let decoder = Arc::new(DecoderDictionary::copy(&raw)); - cache.insert(key.clone(), decoder.clone()); + insert_capped(&mut cache, key.clone(), decoder.clone()); Ok(decoder) } From 6f278981cde77e704f5317d0603e9cc52314ef0c Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 8 Oct 2026 09:38:28 +0200 Subject: [PATCH 5/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Cache=20shared=20by=20?= =?UTF-8?q?connection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sqlite-compress/src/cache/connection.rs | 4 +- sqlite-compress/src/state.rs | 19 +++++--- sqlite-compress/tests/common.rs | 20 ++++++++- sqlite-compress/tests/dict_train.rs | 60 +++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/sqlite-compress/src/cache/connection.rs b/sqlite-compress/src/cache/connection.rs index 510e736..15dfee6 100644 --- a/sqlite-compress/src/cache/connection.rs +++ b/sqlite-compress/src/cache/connection.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; use crate::{ColumnKey, DictId}; -/// Per-connection cache of prepared zstd dictionaries and the current dict id per column. +/// Process-wide map of the current dict id per column (shared across all connections via [`crate::Cache`]). /// -/// `encoders` and `decoders` are independent LRUs keyed by [`DictKey`] (schema + dict id). +/// Prepared encoders/decoders live in the separate process-wide pool, not here. /// /// `current_ids` maps a [`ColumnKey`] to the dict id that `compress` should use. It is /// filled by `warm_cache` (latest row in `__compress_dicts`) and `insert_into_caches` diff --git a/sqlite-compress/src/state.rs b/sqlite-compress/src/state.rs index e4f3b8c..434981c 100644 --- a/sqlite-compress/src/state.rs +++ b/sqlite-compress/src/state.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use libsqlite3_sys::sqlite3; use parking_lot::Mutex; @@ -6,11 +6,17 @@ use sqlite_ffi::{Connection, Context, SqliteError}; use crate::cache::DictCache; -/// Per-connection state for the extension: the SQLite connection and its dictionary cache. +// Shared cache for all connections. +static SHARED: LazyLock = LazyLock::new(|| Cache(Arc::new(Mutex::new(DictCache::new())))); + +/// Per-connection handle: this SQLite connection plus a clone of the process-wide dictionary cache. /// -/// `sqlite3_compress_init` calls [`ExtensionState::attach`], which stores an empty [`Cache`] in the connection's client data. -/// `setup`, `train_all`, `compress`, and `decompress` read that same cache back with [`ExtensionState::from_db`] or [`ExtensionState::from_context`]. -/// The cache is an `Arc`, so every lookup shares the encoders, decoders, and current dict ids for this connection. +/// `sqlite3_compress_init` calls [`ExtensionState::attach`], which stores a clone of the shared +/// [`Cache`] in the connection's client data (not a fresh empty cache). +/// `setup`, `train_all`, `compress`, and `decompress` read it back with [`ExtensionState::from_db`] +/// or [`ExtensionState::from_context`]. +/// All connections therefore share the same `current_ids`; prepared encoders/decoders live in the +/// separate process-wide pool. pub struct ExtensionState { pub(crate) connection: Connection, pub(crate) cache: Cache, @@ -35,8 +41,7 @@ impl ExtensionState { const NAME: &str = "sqlite-compress"; pub fn attach(connection: &Connection) -> Result<(), SqliteError> { - let cache = Cache(Arc::new(Mutex::new(DictCache::new()))); - connection.set_client_data(Self::NAME, cache) + connection.set_client_data(Self::NAME, SHARED.clone()) } pub fn from_db(db: *mut sqlite3) -> Result { diff --git a/sqlite-compress/tests/common.rs b/sqlite-compress/tests/common.rs index 510a9fe..d149929 100644 --- a/sqlite-compress/tests/common.rs +++ b/sqlite-compress/tests/common.rs @@ -1,4 +1,4 @@ -use std::{ptr, sync::Once}; +use std::{ffi::CString, ptr, sync::Once}; use libsqlite3_sys::{SQLITE_OK, sqlite3, sqlite3_auto_extension, sqlite3_close, sqlite3_open}; use sqlite_compress::{ExtensionState, SetupConnection, sqlite3_compress_init}; @@ -29,6 +29,24 @@ impl TestDb { Self { db } } + #[allow(dead_code)] + pub fn open_path(path: &str) -> Self { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| unsafe { + #[allow(clippy::missing_transmute_annotations)] + sqlite3_auto_extension(Some(std::mem::transmute( + sqlite3_compress_init as *const (), + ))); + }); + + let c_path = CString::new(path).unwrap(); + let mut db = ptr::null_mut(); + let rc = unsafe { sqlite3_open(c_path.as_ptr(), &mut db) }; + + assert_eq!(rc, SQLITE_OK); + Self { db } + } + pub fn state(&self) -> ExtensionState { ExtensionState::from_db(self.db).unwrap() } diff --git a/sqlite-compress/tests/dict_train.rs b/sqlite-compress/tests/dict_train.rs index 9b64209..be61f95 100644 --- a/sqlite-compress/tests/dict_train.rs +++ b/sqlite-compress/tests/dict_train.rs @@ -7,6 +7,8 @@ mod common; use common::DEFAULT_MAX_SAMPLES; use sqlite_ffi::{SqlValue, first_value}; +use crate::common::TestDb; + pub fn compress_blob(state: &ExtensionState, data: &[u8]) -> Vec { let rows = state .as_ref() @@ -628,3 +630,61 @@ fn compress_uses_current_id_in_header() { let blob2 = compress_blob(&state, payload); assert_eq!(header_dict_id(&blob2), id2); } + +#[test] +fn second_connection_compress_uses_trained_dict_id() { + let sample_count = 64; + + let path = std::env::temp_dir().join(format!( + "sqlite-compress-shared-cache-{}.db", + std::process::id() + )); + let _ = std::fs::remove_file(&path); + let path_str = path.to_str().unwrap(); + + let db_a = TestDb::open_path(path_str); + let db_b = TestDb::open_path(path_str); + + let state_a = db_a.state(); + let state_b = db_b.state(); + + let attach = format!( + "ATTACH DATABASE '{}' AS raw; + CREATE TABLE IF NOT EXISTS raw.requests_raw ( + id INTEGER PRIMARY KEY, + data BLOB + );", + path_str.replace('\'', "''") + ); + + state_a.as_ref().batch_execute(&attach).unwrap(); + + state_b + .as_ref() + .batch_execute(&format!( + "ATTACH DATABASE '{}' AS raw;", + path_str.replace('\'', "''") + )) + .unwrap(); + + insert_samples(&state_a, sample_count); + + let config = sample_config(sample_count); + setup(&db_a, &config).unwrap(); + + let dict_ids = train_all(&db_a, &config, 1024).unwrap(); + let trained_id = dict_ids[0].get(); + assert_ne!(trained_id, 0); + + let payload = b"GET /api/users/42 HTTP/1.1\r\nHost: example.com\r\n\r\n"; + let blob = compress_blob(&state_b, payload); + + assert_eq!( + header_dict_id(&blob), + trained_id, + "conn B must use the dict id trained on conn A" + ); + assert_eq!(decompress_blob(&state_b, &blob), payload); + + let _ = std::fs::remove_file(&path); +}