From 7ae6e9ef1357dbc88bed2f091695c5ad06295d23 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 00:18:14 +0000 Subject: [PATCH] Add byte sum mode, notes on the gematria channel, light mode and an escape menu - b, the byte sum bucket: the middle letter cycles o -> r -> b, memories are indexed by byte sum, and b mode shows the input's byte sum story (written bytes like 104,105 are read as bytes) - notes on the gematria channel, mirroring lists on the search channel - light mode toggled from the escape menu, themed through zgui tokens - escape menu: memory details, saves (open/new/close/export/import/recent), settings and keys; settings persist in the config folder - the open memory can be swapped at run time; open failures no longer panic - split into a library (gematria, store, settings) plus the window - schema v2 with migration for v1 memories - tests, CI, README, CHANGELOG, CONTRIBUTING, inspect example Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q9xKZd6m8irSsnZNnbkucs --- .github/workflows/ci.yml | 30 + .gitignore | 6 + CHANGELOG.md | 43 + CONTRIBUTING.md | 32 + Cargo.lock | 4 +- Cargo.toml | 29 +- README.md | 109 ++- examples/inspect.rs | 32 + src/gematria.rs | 649 ++++++++++++ src/lib.rs | 26 + src/main.rs | 2014 +++++++++++++++----------------------- src/settings.rs | 139 +++ src/store.rs | 821 ++++++++++++++++ tests/current.rs | 45 + tests/settings.rs | 74 ++ tests/store.rs | 239 +++++ 16 files changed, 3083 insertions(+), 1209 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 examples/inspect.rs create mode 100644 src/gematria.rs create mode 100644 src/lib.rs create mode 100644 src/settings.rs create mode 100644 src/store.rs create mode 100644 tests/current.rs create mode 100644 tests/settings.rs create mode 100644 tests/store.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..979ef2e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + +env: + CARGO_TERM_COLOR: always + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - name: windowing headers + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libxkbcommon-dev libwayland-dev pkg-config + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - run: cargo build --all-targets --locked + - run: cargo test --locked + - run: cargo clippy --all-targets --locked -- -D warnings diff --git a/.gitignore b/.gitignore index c3ec7d2..d3167ed 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,9 @@ rustc-ice-*.txt # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# memories and exports made while running neum here +/memory +/*.memory +/memory.*.json +neum-settings.json diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..546bcee --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,43 @@ +# changelog + +## 0.2.0 + +### added + +- **b**, the byte sum bucket: a third sum beside ordinal and reversed. The middle letter now cycles + `o → r → b`. Every memory is indexed by its byte sum, so the bucket can be searched like the + other two. +- the byte sum story of the input, shown in **b** mode. Input written as bytes (`104,105`) is read + as those bytes. +- notes on the gematria channel, working as lists do on the search channel: Enter notes what was + typed on the alt name, Enter on an empty bar shows the alt name's notes, Delete unlinks one. +- light mode, toggled from the escape menu. +- the escape menu: the open memory's details, saves (open, new, close, export, import, recent), + settings and keys. +- settings kept between runs (`neum/settings.json` in the config folder, or `$NEUM_CONFIG`). +- `neum [MEMORY]`, `--help` and `--version`. +- a status line under the bar instead of messages printed to the terminal. +- tests: the sums and their AVX2 fast paths against scalar, byte sums, factoring, the memory end + to end (ids, indexes, lists, notes, search, backup and restore, repair, migration, saves) and + settings. CI on GitHub Actions. +- `examples/inspect.rs`. + +### changed + +- the code is split into a library (`gematria`, `store`, `settings`) and the window (`main.rs`). +- the open memory can be swapped at run time; a memory that can't be opened no longer panics. +- a number typed on the gematria channel looks up that sum (ctrl+Enter sums it as text). +- a note now goes *on* the alt name (before, the alt name went on what was typed). +- entries show in id order, and memories that aren't UTF-8 show as their bytes. +- forgetting a list lets its members go; a memory can't be a note on itself. +- dependencies have real version requirements instead of `*`. + +### schema + +- memories gain a version in `STATE` (v2) and the `MEMORY_BYTE_SUM` index. A v1 memory is brought + up to date the first time it opens. + +## 0.1.0 + +- ordinal and reversed gematria over a redb memory: saturate, search by words and sums, forget. +- lists on the search channel. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6af9543 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,32 @@ +# contributing + +## layout + +| file | what it is | +|---|---| +| `src/gematria.rs` | the sums (ordinal, reversed, bytes), tokens, factoring, the byte sum story | +| `src/store.rs` | the memory: tables, ids, indexes, lists, notes, search, backup, repair, the open memory | +| `src/settings.rs` | settings kept between runs | +| `src/main.rs` | the window | +| `tests/` | the library end to end, on temporary files | + +## before a pull request + +```console +cargo test +cargo clippy --all-targets +``` + +Both run in CI. On Linux, building needs `libxkbcommon-dev`, `libwayland-dev` and `pkg-config`. + +## changing the memory's tables + +Bump `SCHEMA` in `src/store.rs`. `Memory::open` runs `repair_in` on anything older, which rebuilds +every index from the memories themselves; a new index only has to be part of `index`, `unindex` +and `repair_in`. Add a test that opens a memory written the old way (see +`a_schema_1_memory_gets_its_byte_bucket` in `tests/store.rs`). + +## the fast paths + +The AVX2 sums and tokenizer must agree with their scalar versions byte for byte; the tests in +`src/gematria.rs` check that over many lengths, including the block edges. diff --git a/Cargo.lock b/Cargo.lock index 94a9edb..a405f93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2093,15 +2093,17 @@ dependencies = [ [[package]] name = "neum" -version = "0.1.0" +version = "0.2.0" dependencies = [ "anyhow", "rayon", "redb", "serde", "serde_json", + "tempfile", "zgui", "zgui-ui", + "zgui-ui-tokens", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e8f29f0..13bc171 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,16 +1,35 @@ [package] name = "neum" -version = "0.1.0" +version = "0.2.0" edition = "2024" +rust-version = "1.88" +description = "ordinal, reversed and byte-sum gematria over a stateful redb memory, with lists and notes" +license = "MIT" +readme = "README.md" +repository = "https://github.com/SaulDoesCode/neum" +keywords = ["gematria", "redb", "zgui", "notes"] +categories = ["gui", "text-processing"] + +[lib] +name = "neum" +path = "src/lib.rs" + +[[bin]] +name = "neum" +path = "src/main.rs" [dependencies] -anyhow = "*" +anyhow = "1" zgui = { git = "https://github.com/zortax/zgui" } zgui-ui = { git = "https://github.com/zortax/zgui" } +zgui-ui-tokens = { git = "https://github.com/zortax/zgui" } redb = { version = "4.4.0" } -rayon = { version = "*"} -serde = "*" -serde_json = "*" +rayon = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +tempfile = "3" [profile.release] opt-level = 3 diff --git a/README.md b/README.md index e09bc1d..c85b9e8 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,111 @@ +# neum + ![neum app screenshot and code in the background](./Screenshot_2026-10-22_16-58-36.png) -ordinal and reversed gematria, in a stateful redb app that uses zgui for a basic interface +ordinal, reversed and byte-sum gematria, in a stateful [redb](https://github.com/cberner/redb) app +that uses [zgui](https://github.com/zortax/zgui) for its interface. Everything typed can be +saturated (saved) into a memory, found again by any of its sums or its words, gathered into lists +and hung with notes. You can divinate with this so long #ccru + +## build and run + +```console +cargo run --release # opens the last memory, or ./memory +cargo run --release -- my.memory # opens (or makes) my.memory +``` + +Linux needs the usual windowing headers to build: `libxkbcommon-dev` and `libwayland-dev` +(and `pkg-config`). Rust 1.88 or newer. + +## the bar + +Type into the bar and its sums show underneath. The three letters between them are toggles: + +``s [o/r/b] s`` — **S**earch, which sum (**O**rdinal / **R**eversed / **B**ytes), **S**aturate. + +| letter | on | off | +|---|---|---| +| first `s` | the **search channel**: Enter looks memories up by their words | the **gematria channel**: Enter looks memories up by the sum | +| middle | click to cycle `o → r → b → o` | hidden while searching by words | +| last `s` | Enter also saves what was typed (once) | Enter only looks | + +**o** is a=1 … z=26, **r** is a=26 … z=1, and **b** is the byte sum bucket: every byte counts +`byte + 1`, so it sums anything, letters or not, text or raw bytes. With **b** on, the byte sum +story of the input shows above the sums: its length, mean, digit sum and roots, the byte sum's +prime factors, and its prime-factor reduction chain. Input written as bytes (`104,105`, the +way keys are shown) is read as those bytes, so `hi` and `104,105` tell the same story. + +On the gematria channel a number is a sum to look up (ctrl+Enter sums it as text instead). + +### lists and notes + +Middle click the bar to move its text into the **alt name** (right click the bar to see it). The +alt name is a memory too, and what Enter and Delete do with it depends on the channel: + +| | search channel (`s` on) | gematria channel (`s` off) | +|---|---|---| +| Enter with text | adds what was found to the alt name's **list** | makes it a **note** on the alt name | +| Enter on an empty bar | shows the list | shows the notes | +| Delete on a key | takes it off the list | unlinks the note | + +Without an alt name, Delete on a key forgets that memory (and puts its text back in the bar). + +### the entries + +Click an entry to put its text in the bar; double click to put its key there (keys are bytes, +like `0,1`). Ctrl + right click looks up the mean of the two sums shown. Each entry is underlined +in a colour from one of its other sums, so entries that agree on that sum share a colour. + +## the escape menu + +Esc opens it (Esc again, or a click outside, closes it). + +- **memory** — the open memory's file, size on disk, how many memories, lists and notes it holds, + the last id handed out and how many freed ids wait to be reused. *repair* rebuilds every index + and drops links that point at nothing. +- **saves** — memories are files, kept like game saves. Type a path and *open* it (a new path makes + a new memory), *close* the open one, *export* it to JSON (to a typed `.json` path, or next to the + memory with the time in its name) or *import* a JSON export into an empty memory. The memories + opened before are listed; click one to open it, × to take it off the list. +- **settings** — light or dark mode, whether saturate starts on, whether a memory is compacted when + opened, and how many of a query's words a search match must share (50 / 75 / 90 / 100%). +- **keys** — the controls above, in short. + +Settings live in `neum/settings.json` under your config folder (`$XDG_CONFIG_HOME`, `~/.config` +or `%APPDATA%`), or wherever `NEUM_CONFIG` points. + +## which memory opens + +The first argument, else `$NEUM_MEMORY`, else the last memory open, else `./memory`. A memory +that can't be opened (already open in another neum, say) leaves the window up, and the escape +menu can open another. + +Memories made by 0.1 open as they are; their byte sum bucket is built the first time they open. + +## as a library + +The window is `src/main.rs`; everything under it is the `neum` library, usable on its own: + +```rust +use neum::{Memory, Mode, gematria::sums}; + +let m = Memory::open("my.memory", false)?; +let id = m.saturate(b"ordinary cunt", None)?; +assert_eq!(sums(b"ordinary cunt"), (162, 162)); +assert!(m.gsearch(162, Mode::Ordinal)?.contains_key(&id)); +``` + +`cargo run --example inspect -- my.memory 162` prints a memory's details and what sits at a sum. -pressing enter saves things and does gsearch or search depening on the the middle three letters +## development -``(s [o/r] s)`` which stands for Search, Orinal/Reversed and Saturate (save) +```console +cargo test # sums, fast paths against scalar, the memory end to end, settings +cargo clippy --all-targets +``` -double clicking entries puts their key in the bar, pressing delete on the bar deletes that one +See [CONTRIBUTING.md](CONTRIBUTING.md) and [CHANGELOG.md](CHANGELOG.md). -lists and notes still need ui, but you can divinate with this so long #ccru +## license +MIT, see [LICENSE](LICENSE). diff --git a/examples/inspect.rs b/examples/inspect.rs new file mode 100644 index 0000000..9e76033 --- /dev/null +++ b/examples/inspect.rs @@ -0,0 +1,32 @@ +//! what a memory holds, without the window: +//! +//! ```text +//! cargo run --example inspect -- path/to/memory # its details +//! cargo run --example inspect -- path/to/memory 162 # memories at 162, by every sum +//! ``` + +use neum::{Memory, Mode, gematria::byte_list, human_bytes}; + +fn main() -> anyhow::Result<()> { + let mut args = std::env::args().skip(1); + let path = args.next().unwrap_or_else(|| "./memory".into()); + let m = Memory::open(&path, false)?; + let s = m.stats()?; + println!("{}", s.path.display()); + println!(" {} memories ({}), {} on disk", s.memories, human_bytes(s.stored_bytes), human_bytes(s.file_bytes)); + println!(" {} lists holding {}, {} notes on {} memories", s.lists, s.listed, s.notes, s.noted); + println!(" {} ids to reuse, schema v{}", s.reusable, s.schema); + if let Some(sum) = args.next() { + let sum: u64 = sum.parse()?; + for mode in Mode::ALL { + let mut found: Vec<_> = m.gsearch(sum, mode)?.into_iter().collect(); + found.sort(); + println!("{} {sum}: {}", mode.name(), found.len()); + for (k, v) in found { + let shown = String::from_utf8(v.clone()).unwrap_or_else(|_| format!("⟨{}⟩", byte_list(&v))); + println!(" [{}] {shown}", byte_list(&k)); + } + } + } + Ok(()) +} diff --git a/src/gematria.rs b/src/gematria.rs new file mode 100644 index 0000000..5233d8f --- /dev/null +++ b/src/gematria.rs @@ -0,0 +1,649 @@ +//! the sums: ordinal and reversed letter gematria, byte sums, and what can be said about a number + +use std::borrow::Cow; + +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; + +use serde::{Deserialize, Serialize}; + +/// which sum the middle letter is on: o, r, or b (the byte sum bucket). Clicking it cycles. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Mode { + #[default] + Ordinal, + Reversed, + Bytes, +} + +impl Mode { + pub const ALL: [Mode; 3] = [Mode::Ordinal, Mode::Reversed, Mode::Bytes]; + + /// o → r → b → o + pub fn next(self) -> Mode { + match self { + Mode::Ordinal => Mode::Reversed, + Mode::Reversed => Mode::Bytes, + Mode::Bytes => Mode::Ordinal, + } + } + + pub fn letter(self) -> &'static str { + match self { + Mode::Ordinal => "o", + Mode::Reversed => "r", + Mode::Bytes => "b", + } + } + + pub fn name(self) -> &'static str { + match self { + Mode::Ordinal => "ordinal", + Mode::Reversed => "reversed", + Mode::Bytes => "bytes", + } + } + + /// this mode's sum of some data + pub fn sum(self, data: &[u8]) -> u64 { + match self { + Mode::Ordinal => sums(data).0, + Mode::Reversed => sums(data).1, + Mode::Bytes => byte_sum(data), + } + } +} + +pub fn parse_u8_list(s: &str) -> Result, String> { + let trimmed = s.trim(); + if trimmed.is_empty() { + return Ok(Vec::new()); + } + + trimmed + .split(',') + .map(|part| { + let part_trimmed = part.trim(); + part_trimmed + .parse::() + .map_err(|e| format!("Invalid integer '{}': {}", part_trimmed, e)) + }) + .collect() +} + +/// whether the input is written as bytes: two or more comma separated 0..=255 (`104,105`), +/// the way keys are shown. A lone number is a sum to look up, so it doesn't count. +pub fn is_byte_list(s: &str) -> bool { + s.contains(',') && parse_u8_list(s).is_ok_and(|b| !b.is_empty()) +} + +/// the bytes an input stands for: written bytes as those bytes, anything else as its UTF-8 +pub fn input_bytes(s: &str) -> Cow<'_, [u8]> { + if is_byte_list(s) + && let Ok(b) = parse_u8_list(s) + { + return Cow::Owned(b); + } + Cow::Borrowed(s.as_bytes()) +} + +/// bytes as the comma list keys are written in (`104,105`) +pub fn byte_list(b: &[u8]) -> String { + b.iter().map(|x| x.to_string()).collect::>().join(",") +} + +pub fn hsl(h: f64, s: f64, l: f64) -> [u8; 3] { + let c = (1.0 - (2.0 * l - 1.0).abs()) * s; + let x = c * (1.0 - ((h / 60.0) % 2.0 - 1.0).abs()); + let m = l - c / 2.0; + let (r, g, b) = match (h / 60.0) as u32 { + 0 => (c, x, 0.0), + 1 => (x, c, 0.0), + 2 => (0.0, c, x), + 3 => (0.0, x, c), + 4 => (x, 0.0, c), + _ => (c, 0.0, x), + }; + let to = |v: f64| ((v + m) * 255.0).round().clamp(0.0, 255.0) as u8; + [to(r), to(g), to(b)] +} + +/// golden-angle hues, so neighbouring sums land far apart on the wheel +pub fn color_for_sum(sum: u64) -> [u8; 3] { + hsl((sum as f64 * 137.507_764) % 360.0, 0.45, 0.45) +} + +/// AVX2 fast path from weum: 32 bytes per block, 4 blocks unrolled, PSADBW accumulation. +/// +/// Unlike weum's kernel, the letter test is `(b | 0x20) - 'a' <= 25` rather than +/// `(b | 0x20) - ('a' - 1) <= 26`: with the latter, '@' and '`' pass as letters of value +/// 0 and add 27 to the reversed sum. +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn dual_sum_avx2(data: &[u8]) -> (u64, u64) { + let lower = _mm256_set1_epi8(0x20); + let a = _mm256_set1_epi8(b'a' as i8); + let max = _mm256_set1_epi8(25); + let one = _mm256_set1_epi8(1); + let twentyseven = _mm256_set1_epi8(27); + let zero = _mm256_setzero_si256(); + + let mut s = [zero; 4]; + let mut r = [zero; 4]; + + macro_rules! block { + ($i:expr, $p:expr) => {{ + let v = unsafe { _mm256_loadu_si256($p as *const __m256i) }; + // 0 for 'a' .. 25 for 'z', anything else lands above 25 + let off = _mm256_sub_epi8(_mm256_or_si256(v, lower), a); + let mask = _mm256_cmpeq_epi8(_mm256_min_epu8(off, max), off); + let gval = _mm256_and_si256(_mm256_add_epi8(off, one), mask); + let rval = _mm256_and_si256(_mm256_sub_epi8(twentyseven, gval), mask); + s[$i] = _mm256_add_epi64(s[$i], _mm256_sad_epu8(gval, zero)); + r[$i] = _mm256_add_epi64(r[$i], _mm256_sad_epu8(rval, zero)); + }}; + } + + let len = data.len(); + let ptr = data.as_ptr(); + let mut i = 0; + while i + 128 <= len { + block!(0, ptr.add(i)); + block!(1, ptr.add(i + 32)); + block!(2, ptr.add(i + 64)); + block!(3, ptr.add(i + 96)); + i += 128; + } + while i + 32 <= len { + block!(0, ptr.add(i)); + i += 32; + } + + let reduce = |acc: [__m256i; 4]| -> u64 { + let t = _mm256_add_epi64(_mm256_add_epi64(acc[0], acc[1]), _mm256_add_epi64(acc[2], acc[3])); + let lanes = _mm_add_epi64(_mm256_castsi256_si128(t), _mm256_extracti128_si256(t, 1)); + (_mm_cvtsi128_si64(lanes) as u64).wrapping_add(_mm_extract_epi64(lanes, 1) as u64) + }; + let (ordinal, reversed) = sums_scalar(&data[i..]); + (reduce(s) + ordinal, reduce(r) + reversed) +} + +pub fn sums_scalar(data: &[u8]) -> (u64, u64) { + let (mut o, mut r) = (0u64, 0u64); + for &b in data { + let lc = b | 0x20; + if lc.is_ascii_lowercase() { + let v = (lc - b'a' + 1) as u64; + o += v; + r += 27 - v; + } + } + (o, r) +} + +pub fn sums(data: &[u8]) -> (u64, u64) { + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("avx2") { + return unsafe { dual_sum_avx2(data) }; + } + sums_scalar(data) +} + +fn is_delimiter(b: u8) -> bool { + matches!(b, b' ' | b'\n' | b'\t' | b'\r') +} + +/// splits on spaces, newlines, tabs and carriage returns like weum's `tokenate`; tokens +/// are the raw bytes between delimiters, in order +pub fn tokenate(data: &[u8]) -> Vec<&[u8]> { + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("avx2") { + return unsafe { tokenize_avx2(data) }; + } + tokenize_scalar(data) +} + +pub fn tokenize_scalar(data: &[u8]) -> Vec<&[u8]> { + data.split(|&b| is_delimiter(b)).filter(|t| !t.is_empty()).collect() +} + +/// 32 bytes at a time: one delimiter bitmask per block, from which word starts and ends +/// fall out as bit transitions (carried across blocks) +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn tokenize_avx2(data: &[u8]) -> Vec<&[u8]> { + let space = _mm256_set1_epi8(b' ' as i8); + let newline = _mm256_set1_epi8(b'\n' as i8); + let tab = _mm256_set1_epi8(b'\t' as i8); + let cr = _mm256_set1_epi8(b'\r' as i8); + + let mut tokens = Vec::new(); + let mut start = 0usize; + let mut in_word = false; + let len = data.len(); + let mut i = 0; + while i + 32 <= len { + let v = unsafe { _mm256_loadu_si256(data.as_ptr().add(i) as *const __m256i) }; + let delim = _mm256_or_si256( + _mm256_or_si256(_mm256_cmpeq_epi8(v, space), _mm256_cmpeq_epi8(v, newline)), + _mm256_or_si256(_mm256_cmpeq_epi8(v, tab), _mm256_cmpeq_epi8(v, cr)), + ); + let word = !(_mm256_movemask_epi8(delim) as u32); + // a bit per byte saying whether the byte before it was part of a word + let before = (word << 1) | in_word as u32; + let mut edges = (word & !before) | (!word & before); + while edges != 0 { + let bit = edges.trailing_zeros() as usize; + if word >> bit & 1 == 1 { + start = i + bit; + } else { + tokens.push(&data[start..i + bit]); + } + edges &= edges - 1; + } + in_word = word >> 31 == 1; + i += 32; + } + for (j, &b) in data.iter().enumerate().skip(i) { + match (is_delimiter(b), in_word) { + (true, true) => { + tokens.push(&data[start..j]); + in_word = false; + } + (false, false) => { + start = j; + in_word = true; + } + _ => {} + } + } + if in_word { + tokens.push(&data[start..]); + } + tokens +} + +/// each distinct token once (sorted by bytes). FIX: `dedup` alone only removes repeats +/// that sit next to each other, so "a b a" kept both a's +pub fn unique_tokens(data: &[u8]) -> Vec<&[u8]> { + let mut tkns = tokenate(data); + tkns.sort_unstable(); + tkns.dedup(); + tkns +} + +/// a^b written out in decimal, however long it gets +pub fn permsq(a: u64, b: u64) -> String { + let mut digits = vec![1u8]; + for _ in 0..b { + let mut carry = 0u64; + for d in digits.iter_mut().rev() { + let prod = (*d as u64) * a + carry; + *d = (prod % 10) as u8; + carry = prod / 10; + } + while carry > 0 { + digits.insert(0, (carry % 10) as u8); + carry /= 10; + } + } + digits.into_iter().map(|d| (d + b'0') as char).collect() +} + +/// the byte sums of some data +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ByteSums { + /// Σ (b + 1) + pub bytes: u64, + /// Σ (256 − b) + pub reversed: u64, + /// Σ digit_sum(b + 1) + pub digits: u64, + /// how many bytes were summed + pub len: u64, +} + +impl ByteSums { + /// the digit roots of `bytes`, `reversed` and `digits` (0 for 0, else 1 + (n − 1) mod 9) + pub fn roots(&self) -> [u64; 3] { + [digit_root(self.bytes), digit_root(self.reversed), digit_root(self.digits)] + } + + /// the mean byte value, 1 ..= 256 (0 for no bytes) + pub fn mean(&self) -> f64 { + if self.len == 0 { 0.0 } else { self.bytes as f64 / self.len as f64 } + } +} + +/// the value of one byte: 1 ..= 256 +pub fn value(b: u8) -> u64 { + b as u64 + 1 +} + +/// the sum of n's decimal digits +pub fn digit_sum(mut n: u64) -> u64 { + let mut s = 0; + while n > 0 { + s += n % 10; + n /= 10; + } + s +} + +/// 0 for 0, else 1 + (n − 1) mod 9 +pub fn digit_root(n: u64) -> u64 { + if n == 0 { 0 } else { 1 + (n - 1) % 9 } +} + +const DIGITS: [u8; 256] = { + let mut t = [0u8; 256]; + let mut i = 0; + while i < 256 { + let mut n = i + 1; + let mut s = 0; + while n > 0 { + s += n % 10; + n /= 10; + } + t[i] = s as u8; + i += 1; + } + t +}; + +/// every byte sum of `data` +pub fn byte_sums(data: &[u8]) -> ByteSums { + let (mut bytes, mut digits) = (0u64, 0u64); + for &b in data { + bytes += b as u64; + digits += DIGITS[b as usize] as u64; + } + let len = data.len() as u64; + bytes += len; + ByteSums {bytes, reversed: 257 * len - bytes, digits, len} +} + +/// Σ (b + 1) alone +pub fn byte_sum(data: &[u8]) -> u64 { + data.iter().map(|&b| b as u64).sum::() + data.len() as u64 +} + +pub fn prime_factors(mut n: u64) -> Vec { + let mut found = Vec::new(); + let mut d = 2; + while n > 1 { + while n.is_multiple_of(d) { + found.push(d); + n /= d; + } + d += 1; + if d * d > n && n > 1 { + found.push(n); + break; + } + } + found +} + +fn pf_sum(n: u64) -> u64 { + prime_factors(n).iter().sum() +} + +pub fn pf_reduction(s: u64) -> (Vec, u64, u64) { + const LIMIT: usize = 32; + let pf = prime_factors(s); + let pfr: u64 = pf.iter().sum(); + let ds = digit_sum(s); + let dp = ds * pfr; + let mut chain: Vec = Vec::new(); + for _ in 0..=LIMIT { + let xn = match chain.last() { + Some(&last) => pf_sum(last) * digit_sum(last), + None => pf_sum(dp) * ds, + }; + if chain.contains(&xn) { break; } + chain.push(xn); + } + let total = chain.iter().sum::() + pfr; + (chain, total, digit_sum(total)) +} + +#[allow(dead_code)] +pub(crate) fn vowel(c: u8) -> u64 { + match c { + b'a' => 1, b'e' => 2, + b'o' => 3, b'u' => 4, + b'i' => 5, b'y' => 6, + _ => 0, + } +} + +#[allow(dead_code)] +pub(crate) fn consonant(c: u8) -> u64 { + match c { + b'b' => 1, b'c' => 2, b'd' => 3, b'f' => 4, + b'g' => 5, b'h' => 6, b'j' => 7, b'k' => 8, + b'l' => 9, b'm' => 10, b'n' => 11, b'p' => 12, + b'q' => 13, b'r' => 14, b's' => 15, b't' => 16, + b'v' => 17, b'w' => 18, b'x' => 19, b'z' => 20, + _ => 0, + } +} + +/// the byte sum story of some data: its byte sums, and the factoring of the byte sum +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Story { + pub sums: ByteSums, + /// prime factors of `sums.bytes` + pub factors: Vec, + /// `pf_reduction(sums.bytes)`: the chain, its total and that total's digit sum + pub chain: Vec, + pub total: u64, + pub total_digits: u64, +} + +pub fn byte_story(data: &[u8]) -> Story { + let sums = byte_sums(data); + let (chain, total, total_digits) = pf_reduction(sums.bytes); + Story { sums, factors: prime_factors(sums.bytes), chain, total, total_digits } +} + +impl Story { + /// `2 × 3 × 173`, `prime` for primes, `·` below 2 + pub fn factored(&self) -> String { + match self.factors.as_slice() { + [] => "·".into(), + [_] => "prime".into(), + f => f.iter().map(u64::to_string).collect::>().join(" × "), + } + } + + /// the story as short lines, for under the sums + pub fn lines(&self) -> Vec { + let s = &self.sums; + if s.len == 0 { + return vec!["no bytes yet".into()]; + } + let [rb, rr, rd] = s.roots(); + let chain = self.chain.iter().map(u64::to_string).collect::>().join(" → "); + vec![ + format!( + "{} byte{} · mean {:.2} · digits {} · roots {rb}/{rr}/{rd}", + s.len, + if s.len == 1 { "" } else { "s" }, + s.mean(), + s.digits + ), + format!("{} = {}", s.bytes, self.factored()), + format!("{chain} ⇒ {} ⇒ {}", self.total, self.total_digits), + ] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// a small deterministic generator, so the fast paths see plenty of shapes + fn noise(seed: u64, len: usize) -> Vec { + let mut x = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 24) as u8 + }) + .collect() + } + + #[test] + fn ordinal_and_reversed() { + assert_eq!(sums(b"a"), (1, 26)); + assert_eq!(sums(b"z"), (26, 1)); + assert_eq!(sums(b"ABC abc"), (12, 150)); + assert_eq!(sums(b"ordinary cunt"), (162, 162)); + // '@' and '`' sit beside the letters and aren't letters + assert_eq!(sums(b"@`[{"), (0, 0)); + assert_eq!(sums(b""), (0, 0)); + } + + #[test] + fn fast_sums_match_scalar() { + for len in [0, 1, 31, 32, 33, 127, 128, 129, 300, 1000] { + for seed in 0..8 { + let d = noise(seed + len as u64, len); + assert_eq!(sums(&d), sums_scalar(&d), "len {len} seed {seed}"); + } + } + } + + #[test] + fn tokens() { + assert_eq!(tokenize_scalar(b" a bb\tccc\r\nd "), vec![&b"a"[..], b"bb", b"ccc", b"d"]); + assert!(tokenate(b" \n\t ").is_empty()); + assert_eq!(unique_tokens(b"a b a c b"), vec![&b"a"[..], b"b", b"c"]); + } + + #[test] + fn fast_tokens_match_scalar() { + for len in [0, 5, 31, 32, 33, 64, 65, 200, 777] { + for seed in 0..8 { + // mostly letters, with delimiters sprinkled in + let d: Vec = noise(seed * 31 + len as u64, len) + .into_iter() + .map(|b| match b % 7 { 0 => b' ', 1 => b'\n', 2 if b % 3 == 0 => b'\t', _ => b'a' + b % 26 }) + .collect(); + assert_eq!(tokenate(&d), tokenize_scalar(&d), "len {len} seed {seed}"); + } + } + } + + #[test] + fn modes_cycle_through_three() { + let mut m = Mode::Ordinal; + let mut seen = vec![]; + for _ in 0..3 { + seen.push(m.letter()); + m = m.next(); + } + assert_eq!(seen, ["o", "r", "b"]); + assert_eq!(m, Mode::Ordinal); + assert_eq!(Mode::Bytes.sum(b"hi"), 105 + 106); + assert_eq!(Mode::Reversed.sum(b"a"), 26); + } + + #[test] + fn byte_sums_add_up() { + let s = byte_sums(b"hi"); + assert_eq!(s.bytes, 105 + 106); + assert_eq!(s.reversed, (256 - 104) + (256 - 105)); + assert_eq!(s.len, 2); + assert_eq!(s.digits, digit_sum(105) + digit_sum(106)); + assert_eq!(s.bytes, byte_sum(b"hi")); + assert_eq!(s.mean(), 105.5); + assert_eq!(byte_sums(&[]), ByteSums::default()); + assert_eq!(byte_sums(&[255]).bytes, 256); + assert_eq!(byte_sums(&[255]).reversed, 1); + for b in 0..=255u8 { + assert_eq!(DIGITS[b as usize] as u64, digit_sum(value(b))); + } + } + + #[test] + fn roots_and_digits() { + assert_eq!(digit_sum(0), 0); + assert_eq!(digit_sum(9875), 29); + assert_eq!(digit_root(0), 0); + assert_eq!(digit_root(9), 9); + assert_eq!(digit_root(10), 1); + assert_eq!(digit_root(9875), 2); + } + + #[test] + fn factoring() { + assert_eq!(prime_factors(0), Vec::::new()); + assert_eq!(prime_factors(1), Vec::::new()); + assert_eq!(prime_factors(2), vec![2]); + assert_eq!(prime_factors(12), vec![2, 2, 3]); + assert_eq!(prime_factors(97), vec![97]); + assert_eq!(prime_factors(1038), vec![2, 3, 173]); + for n in 2..2000u64 { + assert_eq!(prime_factors(n).iter().product::(), n); + } + } + + #[test] + fn pf_reduction_settles() { + for n in 0..3000 { + let (chain, total, td) = pf_reduction(n); + assert!(!chain.is_empty() && chain.len() <= 33, "{n}"); + assert_eq!(td, digit_sum(total)); + } + } + + #[test] + fn written_bytes() { + assert!(is_byte_list("104,105")); + assert!(is_byte_list(" 0, 1 ")); + assert!(!is_byte_list("162"), "a lone number is a sum"); + assert!(!is_byte_list("a, b")); + assert!(!is_byte_list("1,256")); + assert_eq!(&*input_bytes("104,105"), b"hi"); + assert_eq!(&*input_bytes("hi"), b"hi"); + assert_eq!(&*input_bytes("162"), b"162"); + assert_eq!(byte_list(b"hi"), "104,105"); + assert_eq!(parse_u8_list("1, 2,3").unwrap(), vec![1, 2, 3]); + assert_eq!(parse_u8_list("").unwrap(), Vec::::new()); + assert!(parse_u8_list("x").is_err()); + } + + #[test] + fn the_same_story_for_text_and_its_bytes() { + let text = byte_story(&input_bytes("hi")); + let bytes = byte_story(&input_bytes("104,105")); + assert_eq!(text, bytes); + assert_eq!(text.sums.bytes, 211); + assert_eq!(text.factored(), "prime"); + assert_eq!(text.lines().len(), 3); + assert_eq!(byte_story(b"").lines(), vec!["no bytes yet".to_string()]); + assert_eq!(byte_story(&[1, 2]).factored(), "prime"); + assert_eq!(byte_story(&[3, 3]).factored(), "2 × 2 × 2"); + } + + #[test] + fn colors_stay_in_range() { + assert_eq!(hsl(0.0, 0.0, 0.0), [0, 0, 0]); + assert_eq!(hsl(0.0, 0.0, 1.0), [255, 255, 255]); + assert_eq!(hsl(0.0, 1.0, 0.5), [255, 0, 0]); + assert_ne!(color_for_sum(1), color_for_sum(2)); + } + + #[test] + fn big_powers() { + assert_eq!(permsq(2, 10), "1024"); + assert_eq!(permsq(10, 0), "1"); + assert_eq!(permsq(256, 2), "65536"); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..685828d --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,26 @@ +//! neum: ordinal, reversed and byte-sum gematria over a stateful redb memory, with lists +//! and notes. The window lives in `main.rs`; everything it stands on is here, so it can be +//! tested without one. + +pub mod gematria; +pub mod settings; +pub mod store; + +pub use gematria::Mode; +pub use settings::Settings; +pub use store::{Found, Memory, Stats}; + +/// a byte count the way people read them: `0 B`, `12.3 KiB`, `4.0 MiB` +pub fn human_bytes(n: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + if n < 1024 { + return format!("{n} B"); + } + let mut v = n as f64; + let mut u = 0; + while v >= 1024.0 && u < UNITS.len() - 1 { + v /= 1024.0; + u += 1; + } + format!("{v:.1} {}", UNITS[u]) +} diff --git a/src/main.rs b/src/main.rs index b042d4f..0e14777 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,294 +1,703 @@ -use anyhow::anyhow; - -use zgui::{prelude::*}; +use zgui::{prelude::*, reactive::LocalStorage}; use zgui_ui::prelude::*; +use zgui_ui_tokens::prelude::{ColorScheme, ThemeProviderProps}; + +use std::{collections::HashMap, path::Path}; -use std::{ - sync::{LazyLock}, //time::{SystemTime, UNIX_EPOCH}, //simd::{cmp::SimdPartialEq, prelude::*}, - collections::{HashMap, HashSet} +use neum::{ + Found, Mode, Settings, Stats, + gematria::{ByteSums, byte_list, byte_story, byte_sums, color_for_sum, input_bytes, is_byte_list, parse_u8_list, sums}, + human_bytes, + settings::startup_memory, + store::{self, hex, id_order}, }; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; +type Local = RwSignal; + +/// every signal the window runs on, in one Copy bundle so handlers can share the actions below +#[derive(Clone, Copy)] +struct Ui { + txt: Local, + alt_name: Local, + osum: RwSignal, + rsum: RwSignal, + bsum: RwSignal, + sch: RwSignal, + mode: RwSignal, + sat: RwSignal, + display_output: RwSignal, + esc_mode: RwSignal, + term_ctx_menu: RwSignal, + /// what the last thing done had to say + status: Local, + settings: Local, + stats: Local>, + /// the path typed into the escape menu + path_in: Local, +} + +impl Ui { + fn say(self, msg: impl Into) { + let msg = msg.into(); + eprintln!("{msg}"); + self.status.set(msg); + } + + fn show(self, found: Found) { + self.display_output.set(found); + } + + /// change the settings and keep them + fn set(self, f: impl FnOnce(&mut Settings)) { + self.settings.update(f); + if let Err(e) = self.settings.get_untracked().save() { + self.say(format!("couldn't save settings: {e}")); + } + } -use rayon::prelude::*; -use redb::{ - Database, MultimapTableDefinition, ReadableDatabase, ReadableMultimapTable, ReadableTable, - ReadableTableMetadata, TableDefinition, WriteTransaction, -}; + /// the sums of what's in the bar + fn compute(self) { + let t = self.txt.get_untracked(); + let (o, r) = sums(t.as_bytes()); + if self.osum.get_untracked() != o { self.osum.set(o); } + if self.rsum.get_untracked() != r { self.rsum.set(r); } + let b = byte_sums(&input_bytes(&t)); + if self.bsum.get_untracked() != b { self.bsum.set(b); } + } -use serde_json::{Value, json}; + /// the two numbers either side of the toggles + fn shown_sums(self) -> (u64, u64) { + match self.mode.get() { + Mode::Bytes => { + let b = self.bsum.get(); + (b.bytes, b.reversed) + } + _ => (self.osum.get(), self.rsum.get()), + } + } -#[component] -fn Gematria() -> impl IntoView { - let txt = RwSignal::new_local(String::new()); - let alt_name = RwSignal::new_local(String::new()); - let osum = RwSignal::new(0_u64); - let rsum = RwSignal::new(0_u64); - let sch = RwSignal::new(false); - let o_or_r = RwSignal::new(true); - let sat = RwSignal::new(true); - let display_output: RwSignal, Vec>> = RwSignal::new(HashMap::new()); - let esc_mode = RwSignal::new(false); - let term_ctx_menu = RwSignal::new(false); + fn shown_sums_untracked(self) -> (u64, u64) { + match self.mode.get_untracked() { + Mode::Bytes => { + let b = self.bsum.get_untracked(); + (b.bytes, b.reversed) + } + _ => (self.osum.get_untracked(), self.rsum.get_untracked()), + } + } - view! { - row(class = "spread", - on:click = {move |_ev| { - esc_mode.update(|n| *n = false); - }} - ) {{move || { // 'ordinary cunt' sums to 162/162 - let mut els = vec![]; - for (k, v) in display_output.get() { - if let Ok(v) = String::from_utf8(v) { - let ov = v.to_string(); - let key = format!("{:?}", k).replace("\"", "").replace("[", "").replace("]", ""); - let (o, r) = sums(v.as_bytes()); - let culr = color_for_sum(if o_or_r.get() { r } else { o }); - let unrlnclr = format!("{:?},{:?},{:?}", culr[0], culr[1], culr[2]); + /// the sum the middle letter is on + fn active_sum(self) -> u64 { + match self.mode.get_untracked() { + Mode::Ordinal => self.osum.get_untracked(), + Mode::Reversed => self.rsum.get_untracked(), + Mode::Bytes => self.bsum.get_untracked().bytes, + } + } - els.push(view!{ - box( - class = "otxt", - style = format!("border-bottom: 3px solid rgb({});", unrlnclr), - on:double_click = move |_| { - let key = key.clone(); - txt.update(move |t| *t = key); - }, - on:pointer_down = move |ev| { - if let Some(btn) = ev.button { - if btn == PointerButton::Secondary { - if ev.modifiers.control() { - let n = (osum.get() as f64 * rsum.get() as f64).sqrt().round() as u64; - txt.update(move |t| *t = n.to_string()); - if let Ok(out) = gsearch(n, !o_or_r.get()) { - display_output.update(move |disout| *disout = out); - } - } - } - } - }, - on:click = move |_ev| { - let ov = ov.to_string(); - txt.update(move |t| *t = ov.to_string()); - let (o, r) = sums(txt.get().as_bytes()); - if osum.get() != o { osum.update(move |n| *n = o); } - if rsum.get() != r { rsum.update(move |n| *n = r); } - } - ) { text() {{v}} } - }); + fn regsearch(self) { + if !store::is_open() { return; } + match store::gsearch(self.active_sum(), self.mode.get_untracked()) { + Ok(out) => self.show(out), + Err(e) => self.say(format!("search error: {e}")), + } + } + + fn threshold(self) -> f32 { + self.settings.get_untracked().threshold_fraction() + } + + /// the memory the alt name is, if there is one (made, with `make`) + fn alt_memory(self, make: bool) -> Option> { + let alt = self.alt_name.get_untracked(); + if alt.is_empty() { return None; } + match store::search(alt.as_bytes(), make) { + Ok((_, Some(l))) => Some(l), + Ok((_, None)) => { + self.say(format!("look no find.. no memory is «{alt}»")); + None + } + Err(e) => { + self.say(format!("search error: {e}")); + None + } + } + } + + fn enter(self, ctrl: bool) { + let t = self.txt.get_untracked(); + let alt = self.alt_name.get_untracked(); + if t.is_empty() { + // an empty bar shows what's under the alt name: its list on the search channel, + // its notes on the gematria channel + if alt.is_empty() { return; } + let Some(l) = self.alt_memory(false) else { + self.show(HashMap::new()); + return; + }; + if self.sch.get_untracked() { + match store::get_list(&l) { + Ok(lst) => { + self.say(format!("list «{alt}»: {} in it", lst.len())); + self.show(lst); + } + Err(e) => self.say(format!("trouble getting the list: {e}")), + } + } else { + match store::get_notes_on(&l) { + Ok(notes) => { + self.say(format!("notes on «{alt}»: {}", notes.len())); + self.show(notes); + } + Err(e) => self.say(format!("trouble getting the notes: {e}")), } } - els - }}} - column(class = "col") { - if move || term_ctx_menu.get() { - row(class = "term-menu") { - Input( - value = alt_name, - ) + return; + } + + if self.sch.get_untracked() { + // search channel: words, and lists + let (out, key) = match store::search_with(t.as_bytes(), self.sat.get_untracked(), self.threshold()) { + Ok(r) => r, + Err(e) => return self.say(format!("search error: {e}")), + }; + self.say(format!("{} found", out.len())); + self.show(out); + if alt.is_empty() { return; } + let Some(k) = key else { + return self.say(format!("«{t}» isn't a memory, so it can't be listed (turn saturate on)")); + }; + if let Some(l) = self.alt_memory(true) { + match store::add_to_list(&l, &k) { + Ok(()) => self.say(format!("listed in «{alt}»")), + Err(e) => self.say(format!("trouble adding thing to list: {e}")), } } - row( - class = "inp", - on:pointer_down = move |ev| { - if let Some(btn) = ev.button { - if btn == PointerButton::Secondary { - term_ctx_menu.update(|n| *n = !*n); - } else if btn == PointerButton::Middle { - let n = txt.get(); - alt_name.update(move |ln| *ln = n); - txt.update(move |t| *t = String::new()); + } else { + // gematria channel: sums, and notes + let mode = self.mode.get_untracked(); + let number = t.trim().parse::().ok().filter(|_| !ctrl); + let written_bytes = mode == Mode::Bytes && is_byte_list(&t); + let sum = number.unwrap_or_else(|| self.active_sum()); + let mut out = match store::gsearch(sum, mode) { + Ok(out) => out, + Err(e) => return self.say(format!("search error: {e}")), + }; + // a number is a sum to look up and written bytes are only looked up; anything + // else is text, which can be saturated and noted + if number.is_none() && !written_bytes { + let tb = t.as_bytes().to_vec(); + let mut key = out.iter().find(|(_, v)| **v == tb).map(|(k, _)| k.clone()); + if key.is_none() && self.sat.get_untracked() { + match store::saturate(&tb, None) { + Ok(k) => { + out.insert(k.clone(), tb); + key = Some(k); } + Err(e) => self.say(format!("trouble saturating: {e}")), } - }, - ) { - Input( - value = txt, - label = "type text to sum with gematria", - on:input = move |_| { - let (o, r) = sums(txt.get().as_bytes()); - osum.update(move |n| *n = o); - rsum.update(move |n| *n = r); - }, - on:click = move |_ev| { + } + if !alt.is_empty() { + match (key, self.alt_memory(true)) { + (Some(k), Some(l)) => match store::note_on(&k, &l) { + Ok(()) => self.say(format!("noted on «{alt}»")), + Err(e) => self.say(format!("trouble noting on thing: {e}")), + }, + (None, _) => self.say(format!("«{t}» isn't a memory, so it can't be a note (turn saturate on)")), + _ => {} + } + } else { + self.status.set(format!("{} at {} {sum}", out.len(), mode.name())); + } + } else { + self.status.set(format!("{} at {} {sum}", out.len(), mode.name())); + } + self.show(out); + } + } + + fn delete(self) { + let t = self.txt.get_untracked(); + let Ok(key) = parse_u8_list(&t) else { return }; + if key.is_empty() { return; } + if !self.alt_name.get_untracked().is_empty() { + // out of the alt name's list (search channel) or off its notes (gematria channel) + let Some(l) = self.alt_memory(false) else { return }; + let r = if self.sch.get_untracked() { store::rm_from_list(&l, &key) } else { store::rm_note(&key, &l) }; + match r { + Ok(()) => { + self.display_output.update(|d| { d.remove(&key); }); + self.say(if self.sch.get_untracked() { "taken off the list" } else { "no longer a note on it" }); + } + Err(e) => self.say(format!("trouble unlinking: {e}")), + } + } else { + match store::forget(&key) { + Ok(was) => { + self.display_output.update(|d| { d.remove(&key); }); + self.say(format!("forgot [{}]", byte_list(&key))); + if let Ok(stwing) = String::from_utf8(was) { + self.txt.set(stwing); + self.compute(); + } + } + Err(e) => self.say(format!("{e}")), + } + } + } + + // ---- the escape menu ---- + + fn refresh(self) { + self.stats.set(store::stats().ok()); + } + + fn toggle_esc(self) { + let open = !self.esc_mode.get_untracked(); + if open { + self.refresh(); + if self.path_in.get_untracked().is_empty() + && let Some(p) = store::current_path() + { + self.path_in.set(p.display().to_string()); + } + } + self.esc_mode.set(open); + } + + fn open_memory(self, path: String) { + let path = path.trim().to_string(); + if path.is_empty() { + return self.say("type the path of a memory to open (a new path makes a new one)"); + } + match store::open(&path, self.settings.get_untracked().compact_on_open) { + Ok(m) => { + let p = m.path().display().to_string(); + let p = canonical(&p); + self.set(|s| s.opened(&p)); + self.path_in.set(p.clone()); + self.show(HashMap::new()); + self.refresh(); + self.say(format!("opened {p}")); + } + Err(e) => self.say(format!("{e}")), + } + } + + fn close_memory(self) { + match store::close() { + Some(p) => { + self.show(HashMap::new()); + self.refresh(); + self.say(format!("closed {}", p.display())); + } + None => self.say("no memory is open"), + } + } + + /// to the path typed if it is a .json, else next to the memory with the time in its name + fn export(self) { + let typed = self.path_in.get_untracked().trim().to_string(); + let to = if typed.ends_with(".json") { + typed + } else if let Some(p) = store::current_path() { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + format!("{}.{secs}.json", p.display()) + } else { + return self.say("no memory open to export"); + }; + match store::backup_memories_to_file(&to) { + Ok(n) => self.say(format!("exported {n} memories to {to}")), + Err(e) => self.say(format!("export failed: {e}")), + } + } + + fn import(self) { + let from = self.path_in.get_untracked().trim().to_string(); + if !from.ends_with(".json") { + return self.say("type the path of a .json export to import it"); + } + match store::restore_from_file(&from) { + Ok(n) => { + self.refresh(); + self.say(format!("imported {n} memories from {from}")); + } + Err(e) => self.say(format!("import failed: {e}")), + } + } + + fn repair(self) { + match store::repair() { + Ok(()) => { + self.refresh(); + self.say("repaired: indexes rebuilt, dangling links dropped"); + } + Err(e) => self.say(format!("repair failed: {e}")), + } + } +} + +fn canonical(p: &str) -> String { + std::fs::canonicalize(p).map(|c| c.display().to_string()).unwrap_or_else(|_| p.to_string()) +} + +/// the file name of a save, for the list +fn save_name(p: &str) -> String { + Path::new(p).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| p.to_string()) +} + +fn stat_rows(s: &Stats) -> Vec<(&'static str, String)> { + vec![ + ("file", s.path.display().to_string()), + ("on disk", human_bytes(s.file_bytes)), + ("memories", format!("{} · {}", s.memories, human_bytes(s.stored_bytes))), + ("lists", format!("{} holding {}", s.lists, s.listed)), + ("notes", format!("{} on {} memories", s.notes, s.noted)), + ("last id", s.allocated.as_deref().map(|a| format!("[{}] 0x{}", byte_list(a), hex(a))).unwrap_or_else(|| "none yet".into())), + ("ids to reuse", s.reusable.to_string()), + ("schema", format!("v{}", s.schema)), + ] +} + +const KEYS: [(&str, &str); 11] = [ + ("enter", "sum, look up, saturate; numbers look up that sum (ctrl+enter sums them as text)"), + ("enter, alt name set", "s on: add to the alt name's list · s off: note it on the alt name"), + ("enter, empty bar", "s on: show the alt name's list · s off: show its notes"), + ("delete on a key", "forget it, or unlink it from the alt name's list or notes"), + ("middle click bar", "move the text to the alt name"), + ("right click bar", "show the alt name"), + ("click / double click", "an entry's text / its key into the bar"), + ("ctrl + right click", "an entry: look up the mean of the two sums"), + ("s · o/r/b · s", "search channel · which sum (click to cycle) · saturate"), + ("b", "the byte sum bucket: Σ(byte+1); 104,105 is read as bytes"), + ("esc", "this menu"), +]; + +#[component] +fn Gematria(settings: Settings, greeting: String) -> impl IntoView { + let ui = Ui { + txt: RwSignal::new_local(String::new()), + alt_name: RwSignal::new_local(String::new()), + osum: RwSignal::new(0_u64), + rsum: RwSignal::new(0_u64), + bsum: RwSignal::new(ByteSums::default()), + sch: RwSignal::new(false), + mode: RwSignal::new(settings.mode), + sat: RwSignal::new(settings.saturate), + display_output: RwSignal::new(HashMap::new()), + esc_mode: RwSignal::new(false), + term_ctx_menu: RwSignal::new(false), + status: RwSignal::new_local(greeting), + settings: RwSignal::new_local(settings), + stats: RwSignal::new_local(None), + path_in: RwSignal::new_local(String::new()), + }; + let Ui { txt, alt_name, sch, mode, sat, display_output, esc_mode, term_ctx_menu, status, settings, stats, path_in, .. } = ui; + + let light = move || settings.get().light; + let scheme = Signal::derive_local(move || if light() { ColorScheme::Light } else { ColorScheme::Dark }); + let sat_at_start = Binding::controlled( + Signal::derive_local(move || settings.get().saturate), + move |on: bool| ui.set(|s| s.saturate = on), + ); + let compact_on_open = Binding::controlled( + Signal::derive_local(move || settings.get().compact_on_open), + move |on: bool| ui.set(|s| s.compact_on_open = on), + ); + + view! { + ThemeProvider(scheme = scheme) { + column(class = "app", class:light = light) { + row(class = "spread", + on:click = {move |_ev| { esc_mode.update(|n| *n = false); - }, - on:key_down = move |ev| match &ev.key { - Key::Named(NamedKey::Enter) => { - if sch.get() { - let t = txt.get(); - if t.len() == 0 { - let alt = alt_name.get(); - if alt.len() != 0 { - let sr = search(alt.as_bytes(), false); - if let Ok((_found, that_one)) = sr { - if let Some(l) = that_one { - match get_list(&l) { - Ok(lst) => display_output.update(move |disout| *disout = lst), - Err(e) => println!("trouble adding thing to list: {e}") - } - } else { - display_output.update(move |disout| disout.clear()); - println!("look no find.. that one"); - } - } else if let Err(e) = sr { - println!("search error: {e}"); - } - } - } else { - let r = search(t.as_bytes(), sat.get()); - let mut k = vec![]; - if let Ok(out) = r { - display_output.update(move |disout| *disout = out.0); - if let Some(key) = out.1 { k.extend_from_slice(&key); } - } - let an = alt_name.get(); - if an.len() != 0 { - let sr = search(an.as_bytes(), true); - if let Ok((_found, that_one)) = sr { - if let Some(l) = that_one { - if let Err(e) = add_to_list(&l, &k) { - println!("trouble adding thing to list: {e}"); - } - } else { - println!("look no find.. that one"); - } - } else if let Err(e) = sr { - display_output.update(move |disout| disout.clear()); - println!("search error: {e}"); + }} + ) {{move || { // 'ordinary cunt' sums to 162/162 + let mut found: Vec<(Vec, Vec)> = display_output.get().into_iter().collect(); + found.sort_by(|a, b| id_order(&a.0).cmp(&id_order(&b.0))); + let mode_now = mode.get(); + let mut els = vec![]; + for (k, v) in found { + let key = byte_list(&k); + let (o, r) = sums(&v); + // everything shown shares the searched sum, so colour by another one + let culr = color_for_sum(if mode_now == Mode::Ordinal { r } else { o }); + let unrlnclr = format!("{:?},{:?},{:?}", culr[0], culr[1], culr[2]); + let (shown, is_text) = match String::from_utf8(v) { + Ok(s) => (s, true), + Err(e) => (format!("⟨{}⟩", byte_list(e.as_bytes())), false), + }; + let ov = if is_text { shown.clone() } else { key.clone() }; + + els.push(view!{ + box( + class = "otxt", + class:bytes = !is_text, + style = format!("border-bottom: 3px solid rgb({});", unrlnclr), + on:double_click = move |_| { + let key = key.clone(); + txt.update(move |t| *t = key); + ui.compute(); + }, + on:pointer_down = move |ev| { + if let Some(btn) = ev.button + && btn == PointerButton::Secondary + && ev.modifiers.control() + { + let (a, b) = ui.shown_sums_untracked(); + let n = (a as f64 * b as f64).sqrt().round() as u64; + txt.update(move |t| *t = n.to_string()); + ui.compute(); + match store::gsearch(n, mode.get_untracked()) { + Ok(out) => display_output.update(move |disout| *disout = out), + Err(e) => ui.say(format!("search error: {e}")), } } + }, + on:click = move |_ev| { + let ov = ov.to_string(); + txt.update(move |t| *t = ov.to_string()); + ui.compute(); } + ) { text() {{shown}} } + }); + } + els + }}} + column(class = "col") { + if move || term_ctx_menu.get() { + row(class = "term-menu") { + Input( + value = alt_name, + label = "alt name: a list (s on) or what notes are on (s off)", + ) + } + } + row( + class = "inp", + on:pointer_down = move |ev| { + if let Some(btn) = ev.button { + if btn == PointerButton::Secondary { + term_ctx_menu.update(|n| *n = !*n); + } else if btn == PointerButton::Middle { + let n = txt.get(); + alt_name.update(move |ln| *ln = n); + txt.update(move |t| *t = String::new()); + ui.compute(); + } + } + }, + ) { + Input( + value = txt, + label = "type text to sum with gematria", + on:input = move |_| { + ui.compute(); + if !status.get_untracked().is_empty() { status.set(String::new()); } + }, + on:click = move |_ev| { + esc_mode.update(|n| *n = false); + }, + on:key_down = move |ev| match &ev.key { + Key::Named(NamedKey::Enter) => ui.enter(ev.modifiers.control()), + Key::Named(NamedKey::Delete) => ui.delete(), + Key::Named(NamedKey::Escape) => ui.toggle_esc(), + _k => {} + } + ) + } + row(class = "hint") { + text() {{move || { + let s = status.get(); + if !s.is_empty() { return s; } + let alt = alt_name.get(); + if alt.is_empty() { + String::new() + } else if sch.get() { + format!("→ list «{alt}»") } else { + format!("→ notes on «{alt}»") + } + }}} + } + if move || mode.get() == Mode::Bytes { + column(class = "story") { + {move || { let t = txt.get(); - let an = alt_name.get(); - if an.len() != 0 && sat.get() && t.parse::().is_err() { - match gsearch(osum.get(), false) { - Ok(mut fnd) => { - let tb = t.as_bytes().to_vec(); - if fnd.par_iter().find_any(|(_k, v)| tb.eq(*v)).is_none() { - if let Ok(k) = saturate(&tb, None) { - let sr = search(an.as_bytes(), true); - if let Ok((_found, that_one)) = sr { - if let Some(l) = that_one { - if let Err(e) = note_on(&l, &k) { - println!("trouble noting on thing: {e}"); - } - } else { - println!("look no find.. that one"); + byte_story(&input_bytes(&t)) + .lines() + .into_iter() + .map(|l| view! { text(class = "story-line") {{l}} }) + .collect::>() + }} + } + } + row( + class = "sums", + on:click = {move |_ev| { + esc_mode.update(|n| *n = false); + }}, + ) { + column() { + column() { + text(class = "number") {{move || format!("{}", ui.shown_sums().0)}} + text(class = "value") {{move || if mode.get() == Mode::Bytes { "bytes " } else { "ordinal " }}} + } + } + row(class = "toggles") { + text(class="sch", class:ing = move || sch.get(), on:click = move |_| { + if sch.get() { + ui.regsearch(); + } + sch.update(move |b| *b = !*b) + }) { "s" } + text(class = "o-or-r", class:hidden = move || sch.get(), on:click = move |_| { + let next = mode.get_untracked().next(); + mode.set(next); + ui.set(|s| s.mode = next); + ui.regsearch(); + }) { + {move || mode.get().letter()} + } + text(class = "saturate", class:sat = move || sat.get(), on:click = move |_| sat.update(move |b| *b = !*b)) { + "s" + } + } + column() { + column() { + text(class = "number") {{move || format!("{}", ui.shown_sums().1)}} + text(class = "value") {{move || if mode.get() == Mode::Bytes { "reversed bytes " } else { "reversed " }}} + } + } + } + + if move || esc_mode.get() { + box( + class = "esc", + on:key_down = move |ev| match &ev.key { + Key::Named(NamedKey::Escape) => ui.toggle_esc(), + _k => {} + } + ) { + scroll(class = "esc-scroll") { + column(class = "esc-inner") { + row(class = "esc-head") { + column() { + text(class = "esc-title") { "neum" } + text(class = "esc-sub") {{format!("v{} · esc to go back", env!("CARGO_PKG_VERSION"))}} + } + spacer() + Button( + variant = ButtonVariant::Outline, + on:click = move |_| ui.set(|s| s.light = !s.light) + ) { + {move || if light() { "☾ dark mode" } else { "☀ light mode" }} + } + } + row(class = "esc-cards") { + column(class = "esc-card") { + text(class = "esc-h") { "memory" } + {move || match stats.get() { + Some(s) => stat_rows(&s) + .into_iter() + .map(|(k, v)| view! { + row(class = "kv") { + text(class = "k") {{k}} + text(class = "v") {{v}} } - fnd.insert(k, tb); - } else if let Err(e) = sr { - display_output.update(move |disout| disout.clear()); - println!("search error: {e}"); + }) + .collect::>(), + None => vec![view! { + row(class = "kv") { + text(class = "k") {"closed"} + text(class = "v") {"no memory open: open one under saves"} } - } + }], + }} + row(class = "esc-buttons") { + Button(variant = ButtonVariant::Secondary, size = ButtonSize::Sm, on:click = move |_| ui.refresh()) { "refresh" } + Button(variant = ButtonVariant::Secondary, size = ButtonSize::Sm, on:click = move |_| ui.repair()) { "repair" } } - }, - Err(e) => println!("search error: {e}") - } - } else if let Ok(mut out) = gsearch(if let Ok(sum) = t.parse::() && !ev.modifiers == Modifiers::CONTROL { sum } else if o_or_r.get() { osum.get() } else { rsum.get() }, !o_or_r.get()) { - if sat.get() { - if t.parse::().is_err() { - let tb = txt.get().as_bytes().to_vec(); - if out.par_iter().find_any(|(_k, v)| tb.eq(*v)).is_none() { - if let Ok(key) = saturate(&tb, None) { out.insert(key, tb); } + } + column(class = "esc-card") { + text(class = "esc-h") { "saves" } + Input(value = path_in, label = "path: a memory to open or make, or a .json") + row(class = "esc-buttons") { + Button(size = ButtonSize::Sm, on:click = move |_| ui.open_memory(path_in.get_untracked())) { "open" } + Button(variant = ButtonVariant::Outline, size = ButtonSize::Sm, on:click = move |_| ui.close_memory()) { "close" } + Button(variant = ButtonVariant::Secondary, size = ButtonSize::Sm, on:click = move |_| ui.export()) { "export" } + Button(variant = ButtonVariant::Secondary, size = ButtonSize::Sm, on:click = move |_| ui.import()) { "import" } } + {move || { + let open_now = stats.get().map(|s| canonical(&s.path.display().to_string())); + settings.get().recent.into_iter().map(|p| { + let on = open_now.as_deref() == Some(p.as_str()); + let (to_open, to_drop) = (p.clone(), p.clone()); + view! { + row(class = "save", class:on = on) { + column( + class = "save-name", + on:click = move |_| ui.open_memory(to_open.clone()) + ) { + text(class = "save-title") {{save_name(&p)}} + text(class = "save-path") {{p.clone()}} + } + spacer() + text( + class = "save-drop", + on:click = move |_| { + let p = to_drop.clone(); + ui.set(move |s| s.forget_save(&p)); + } + ) {"×"} + } + } + }).collect::>() + }} } - } - display_output.update(move |disout| *disout = out); - } - } - } - Key::Named(NamedKey::Delete) => { - if let Ok(key) = parse_u8_list(&txt.get()) { - let alt = alt_name.get(); - if alt.len() != 0 { - let sr = search(alt.as_bytes(), false); - if let Ok((_found, that_one)) = sr { - if let Some(l) = that_one { - match rm_from_list(&l, &key) { - Ok(()) => display_output.update(move |disout| { - disout.remove_entry(&key); - }), - Err(e) => println!("trouble removing thing from list: {e}") + column(class = "esc-card") { + text(class = "esc-h") { "settings" } + row(class = "setting") { + text() {"saturate on at start"} + spacer() + Switch(checked = sat_at_start) + } + row(class = "setting") { + text() {"compact a memory when opening it"} + spacer() + Switch(checked = compact_on_open) + } + row(class = "setting") { + text() {"words a search match must share"} + spacer() + Button( + variant = ButtonVariant::Outline, + size = ButtonSize::Sm, + on:click = move |_| ui.set(|s| s.threshold = s.next_threshold()) + ) { + {move || format!("{}%", settings.get().threshold)} } - } else { - println!("rm from list, illegal 'nothing' removal attempt"); } - } else if let Err(e) = sr { - println!("rm from list, search error: {e}"); + text(class = "esc-note") {{format!("kept in {}", Settings::path().display())}} + } + column(class = "esc-card") { + text(class = "esc-h") { "keys" } + {KEYS.iter().map(|(k, v)| view! { + row(class = "kv") { + text(class = "k") {{*k}} + text(class = "v") {{*v}} + } + }).collect::>()} } - } else if let Ok(was) = forget(&key) { - display_output.update(move |disout| { - disout.remove_entry(&key); - }); - if let Ok(stwing) = String::from_utf8(was) { - txt.update(move |t| *t = stwing); } + text(class = "esc-status") {{move || status.get()}} } } } - Key::Named(NamedKey::Escape) => { - esc_mode.update(|n| *n = !*n); - } - _k => {} - } - ) - } - row( - class = "sums", - on:click = {move |_ev| { - esc_mode.update(|n| *n = false); - }}, - ) { - column() { - column() { - text(class = "number") {{move || format!("{}", osum.get())}} - text(class = "value") {{move || format!("ordinal ")}} - } - } - row(class = "toggles") { - text(class="sch", class:ing = move || sch.get(), on:click = move |_| { - if sch.get() { - if let Ok(out) = gsearch(if o_or_r.get() { osum.get() } else { rsum.get() }, !o_or_r.get()) { - display_output.update(move |disout| *disout = out); - } - } - sch.update(move |b| *b = !*b) - }) { "s" } - text(class = "o-or-r", class:hidden = move || sch.get(), on:click = move |_| o_or_r.update(move |b| *b = !*b)) { - {move || { - if let Ok(out) = gsearch(if o_or_r.get() { osum.get() } else { rsum.get() }, !o_or_r.get()) { - display_output.update(move |disout| *disout = out); - } - if o_or_r.get() { "o" } else { "r" } - }} - } - text(class = "saturate", class:sat = move || sat.get(), on:click = move |_| sat.update(move |b| *b = !*b)) { - "s" - } - } - column() { - column() { - text(class = "number") {{move || format!("{}", rsum.get())}} - text(class = "value") {{move || format!("reversed ")}} - } - } - } - - if move || esc_mode.get() { - box( - class = "esc", - on:key_down = move |ev| match &ev.key { - Key::Named(NamedKey::Escape) => { - esc_mode.update(|n| *n = !*n); - } - _k => {} } - ) { - text() { "escape" } } } } @@ -296,42 +705,89 @@ fn Gematria() -> impl IntoView { } const STYLE: &str = css!(" - .col { + :root { background-color: #000; } + .app { + --bg: #000; + --fg: #fff; + --muted: #888; + --card: #f9f3e2; + --card-fg: #111; + --line: #fff; + --panel: #16121a; + --panel-fg: #f4eef8; + --panel-muted: #a89cb2; + --accent: #ff278f; + --shadow: rgba(0,0,0,0.5); + width: 100vw; + height: 100vh; + background: var(--bg); + color: var(--fg); + } + .app.light { + --bg: #f4efe3; + --fg: #1d1a16; + --muted: #9a907f; + --card: #fffdf8; + --card-fg: #1d1a16; + --line: #1d1a16; + --panel: #fffaf2; + --panel-fg: #1d1a16; + --panel-muted: #7d7365; + --accent: #d4136f; + --shadow: rgba(60,40,20,0.18); + } + .col { position: fixed; bottom: 0; width: calc(99% - 2mm); - gap: 12px; + gap: 12px; align-items: center; justify-content: center; flex-flow: row wrap; - color: #fff; + color: var(--fg); } .sums { margin: 1mm; padding: 1mm; width: 80%; justify-content: space-evenly; } - .value { text-align: center; font-size: 13px; margin-top: 1mm; margin-left: 1.5mm; } - .number { text-align: center; font-size: 15px; margin: 0 1mm; } - control { padding: 8px 16px; border-radius: 8px; background: #d9b176; } - control:hover { background: #e30e55; } + .value { text-align: center; font-size: 13px; margin-top: 1mm; margin-left: 1.5mm; color: var(--fg); } + .number { text-align: center; font-size: 15px; margin: 0 1mm; color: var(--fg); } .inp { width: 95%; max-width: 16cm; padding: 1mm; - border-bottom: 1px solid #fff; + border-bottom: 1px solid var(--line); font-size: 20px; } + .hint { + width: 95%; + max-width: 16cm; + min-height: 14px; + justify-content: center; + font-size: 12px; + color: var(--muted); + } + .story { + width: 95%; + max-width: 16cm; + align-items: center; + gap: 2px; + font-size: 12px; + color: var(--muted); + } + .story-line { text-align: center; } .toggles > * { cursor: pointer; transition: all 97ms ease; } + .o-or-r { color: var(--fg); } .o-or-r:hover { font-weight: 500; margin: 0 .25em; } .saturate, .sch { - color: #888; + color: var(--muted); margin: 0 .25em; } .sat, .ing { - color: #fff; + color: var(--fg); } .hidden { opacity: 0; @@ -352,970 +808,132 @@ const STYLE: &str = css!(" text-align: left; margin: 2px 4px; padding: 2px 3px; - background: #f9f3e2; - box-shadow: 0 1px 3px rgba(0,0,0,0.5); + background: var(--card); + color: var(--card-fg); + box-shadow: 0 1px 3px var(--shadow); + cursor: pointer; } + .otxt.bytes { font-family: monospace; font-size: 12px; } .esc { position: absolute; - background: #ff278f; - color: #fff; + background: var(--panel); + color: var(--panel-fg); width: 90%; height: 90vh; bottom: 5vh; left: 5%; - justify-content: center; - text-align: center; + border-top: 3px solid var(--accent); + border-radius: 6px; + box-shadow: 0 4px 18px var(--shadow); + } + .esc-scroll { width: 100%; height: 100%; overflow-y: auto; } + .esc-inner { width: 100%; padding: 10px 14px; gap: 10px; } + .esc-head { align-items: center; gap: 8px; } + .esc-title { font-size: 20px; font-weight: 700; color: var(--accent); } + .esc-sub { font-size: 11px; color: var(--panel-muted); } + .esc-cards { flex-flow: row wrap; gap: 10px; align-items: flex-start; } + .esc-card { + flex: 1 1 240px; + min-width: 220px; + gap: 6px; + padding: 8px 10px; + border: 1px solid var(--panel-muted); + border-radius: 6px; + } + .esc-h { + font-size: 12px; + font-weight: 700; + letter-spacing: 1px; + text-transform: uppercase; + color: var(--accent); + } + .kv { gap: 8px; font-size: 12px; align-items: flex-start; } + .k { flex: 0 0 auto; min-width: 84px; color: var(--panel-muted); } + .v { flex: 1 1 auto; color: var(--panel-fg); overflow-wrap: anywhere; } + .esc-buttons { flex-flow: row wrap; gap: 6px; margin-top: 4px; } + .setting { align-items: center; gap: 8px; font-size: 13px; } + .esc-note { font-size: 11px; color: var(--panel-muted); overflow-wrap: anywhere; } + .esc-status { font-size: 12px; color: var(--accent); min-height: 14px; } + .save { align-items: center; - align-content: center; - box-shadow: 0 1px 3px rgba(0,0,0,0.5); + gap: 6px; + padding: 3px 6px; + border-left: 3px solid transparent; + cursor: pointer; } + .save:hover { background: rgba(127,127,127,0.12); } + .save.on { border-left-color: var(--accent); } + .save-name { gap: 1px; } + .save-title { font-size: 13px; } + .save-path { font-size: 10px; color: var(--panel-muted); overflow-wrap: anywhere; } + .save-drop { color: var(--panel-muted); padding: 0 4px; } + .save-drop:hover { color: var(--accent); } .term-menu { display: inline-flex; - background: #ff278f; + background: var(--accent); color: #fff; width: 100%; - max-width: 256px; + max-width: 320px; height: 2cm; justify-content: center; text-align: center; align-items: center; align-content: center; - box-shadow: 0 1px 3px rgba(0,0,0,0.5); + box-shadow: 0 1px 3px var(--shadow); } "); -fn main() -> anyhow::Result<()> { - //println!("permsq(256) = {}", permsq(256, 256)); - - let neum = app() - .with_title("Neum") // .with_decorations(Decorations::NoTitleBar) - .with_stylesheet(STYLE) - .with_size(666.0, 444.0); - - neum.run(move || { view! { Gematria() } })?; - Ok(()) -} - -fn hsl(h: f64, s: f64, l: f64) -> [u8; 3] { - let c = (1.0 - (2.0 * l - 1.0).abs()) * s; - let x = c * (1.0 - ((h / 60.0) % 2.0 - 1.0).abs()); - let m = l - c / 2.0; - let (r, g, b) = match (h / 60.0) as u32 { - 0 => (c, x, 0.0), - 1 => (x, c, 0.0), - 2 => (0.0, c, x), - 3 => (0.0, x, c), - 4 => (x, 0.0, c), - _ => (c, 0.0, x), - }; - let to = |v: f64| ((v + m) * 255.0).round().clamp(0.0, 255.0) as u8; - [to(r), to(g), to(b)] -} - -fn color_for_sum(sum: u64) -> [u8; 3] { - hsl((sum as f64 * 137.507_764) % 360.0, 0.45, 0.45) -} - -const ALLOCATED: &[u8] = b"allocated".as_slice(); - -macro_rules! multimap { - ($i:ident, $k:ty, $v:ty) => { - const $i: MultimapTableDefinition<$k, $v> = MultimapTableDefinition::new(stringify!($i)); - }; -} - -macro_rules! map { - ($i:ident, $k:ty, $v:ty) => { - const $i: TableDefinition<$k, $v> = TableDefinition::new(stringify!($i)); - }; -} - -map!(STATE, &[u8], &[u8]); -map!(MEMORIES, &[u8], &[u8]); -multimap!(MEMORY_INDEX, &[u8], &[u8]); - -multimap!(TO_REUSE, &[u8], &[u8]); - - -multimap!(LISTS, &[u8], &[u8]); -multimap!(LISTED_IN, &[u8], &[u8]); - -multimap!(NOTES, &[u8], &[u8]); -multimap!(NOTED_ON, &[u8], &[u8]); - -multimap!(MEMORY_GEMATRIA, u64, &[u8]); -multimap!(MEMORY_GEMATRIA_REVERSED, u64, &[u8]); - -/// the database: `./memory` as before, or the path in NEUM_MEMORY (tests use that) -static MEMORY: LazyLock = LazyLock::new(|| { - let path = std::env::var("NEUM_MEMORY").unwrap_or_else(|_| "./memory".into()); - let mut db = Database::create(path).expect("opening/creating db had issues"); - init_state(&mut db).expect("state init failed"); - db -}); - -fn init_state(db: &mut Database) -> anyhow::Result<()> { - let wx = db.begin_write()?; - ensure_tables(&wx)?; - wx.commit()?; - if db.compact()? { - println!("memory db compacted"); - } - Ok(()) -} - -fn ensure_tables(wx: &WriteTransaction) -> anyhow::Result<()> { - wx.open_table(STATE)?; - wx.open_table(MEMORIES)?; - for t in [MEMORY_INDEX, TO_REUSE, LISTS, LISTED_IN, NOTES, NOTED_ON] { - wx.open_multimap_table(t)?; - } - wx.open_multimap_table(MEMORY_GEMATRIA)?; - wx.open_multimap_table(MEMORY_GEMATRIA_REVERSED)?; - Ok(()) -} - -fn parse_u8_list(s: &str) -> Result, String> { - let trimmed = s.trim(); - if trimmed.is_empty() { - return Ok(Vec::new()); - } - - trimmed - .split(',') - .map(|part| { - let part_trimmed = part.trim(); - part_trimmed - .parse::() - .map_err(|e| format!("Invalid integer '{}': {}", part_trimmed, e)) - }) - .collect() -} - -// ---- ids ---- - -/// ids are big-endian counters that grow a byte when they run out: [0]..[255], [0,0].., -/// up to 4 bytes (the same sequence your `saturate` made) -fn next_id(prev: &[u8]) -> anyhow::Result> { - let mut b = prev.to_vec(); - for i in (0..b.len()).rev() { - if b[i] == u8::MAX { - b[i] = 0; - } else { - b[i] += 1; - return Ok(b); - } - } - if b.len() >= 4 { - return Err(anyhow!("ran out of 4-byte ids")); - } - Ok(vec![0; b.len() + 1]) -} - -/// which of two ids came later in that sequence -fn id_order(id: &[u8]) -> (usize, &[u8]) { - (id.len(), id) -} - -fn exists(wx: &WriteTransaction, id: &[u8]) -> anyhow::Result { - Ok(wx.open_table(MEMORIES)?.get(id)?.is_some()) -} - -/// a fresh id: the smallest one to reuse, else one past ALLOCATED, else [0] -fn allocate(wx: &WriteTransaction) -> anyhow::Result> { - let mut reusables = wx.open_multimap_table(TO_REUSE)?; - let reuse = match reusables.get(ALLOCATED)?.next() { - Some(r) => Some(r?.value().to_vec()), - None => None, - }; - if let Some(id) = reuse { - reusables.remove(ALLOCATED, id.as_slice())?; - return Ok(id); - } - let mut state = wx.open_table(STATE)?; - let last = state.get(ALLOCATED)?.map(|g| g.value().to_vec()); - let id = match last { - Some(last) => next_id(&last)?, - None => vec![0], - }; - state.insert(ALLOCATED, id.as_slice())?; - Ok(id) -} - -// ---- indexing ---- +const HELP: &str = "neum: ordinal, reversed and byte-sum gematria over a redb memory -fn index(wx: &WriteTransaction, id: &[u8], value: &[u8]) -> anyhow::Result<()> { - let (f, r) = sums(value); - wx.open_multimap_table(MEMORY_GEMATRIA)?.insert(f, id)?; - wx.open_multimap_table(MEMORY_GEMATRIA_REVERSED)?.insert(r, id)?; - let mut memi = wx.open_multimap_table(MEMORY_INDEX)?; - for tk in unique_tokens(value) { - memi.insert(tk, id)?; - } - Ok(()) -} +usage: neum [MEMORY] -fn unindex(wx: &WriteTransaction, id: &[u8], value: &[u8]) -> anyhow::Result<()> { - let (f, r) = sums(value); - wx.open_multimap_table(MEMORY_GEMATRIA)?.remove(f, id)?; - wx.open_multimap_table(MEMORY_GEMATRIA_REVERSED)?.remove(r, id)?; - let mut memi = wx.open_multimap_table(MEMORY_INDEX)?; - for tk in unique_tokens(value) { - memi.remove(tk, id)?; - } - Ok(()) -} + MEMORY the memory file to open (made if it isn't there). Without one: $NEUM_MEMORY, + else the last memory open, else ./memory -/// every value of a multimap key, removed, collected (so the table can be used again) -fn take_all(wx: &WriteTransaction, t: MultimapTableDefinition<&[u8], &[u8]>, k: &[u8]) -> anyhow::Result>> { - let mut table = wx.open_multimap_table(t)?; - let mut out = vec![]; - for r in table.remove_all(k)? { - out.push(r?.value().to_vec()); - } - Ok(out) -} +env: + NEUM_MEMORY the memory to open when none is given + NEUM_CONFIG where settings are kept (default: /neum/settings.json) -// ---- memories ---- +in the window, esc opens the menu: the memory's details, saves, settings and keys."; -/// with `None`, store a new memory under a fresh id; with `Some(id)`, put this -/// value at that id, moving its index entries from the old value (if there was one) to -/// the new. Returns the id. -pub fn saturate(quidity: &[u8], t: Option>) -> anyhow::Result> { - let wx = MEMORY.begin_write()?; - let id = match t { - None => allocate(&wx)?, - Some(id) => { - if id.is_empty() || id.len() > 4 { - return Err(anyhow!("ids are 1 to 4 bytes")); - } - let old = wx.open_table(MEMORIES)?.get(id.as_slice())?.map(|g| g.value().to_vec()); - if let Some(old) = old { - unindex(&wx, &id, &old)?; - } // an id written by hand is in use now: not to be handed out again - wx.open_multimap_table(TO_REUSE)?.remove(ALLOCATED, id.as_slice())?; - let mut state = wx.open_table(STATE)?; - let last = state.get(ALLOCATED)?.map(|g| g.value().to_vec()); - if last.as_deref().is_none_or(|l| id_order(&id) > id_order(l)) { - state.insert(ALLOCATED, id.as_slice())?; - } - id +fn main() -> anyhow::Result<()> { + let arg = std::env::args().nth(1); + match arg.as_deref() { + Some("-h" | "--help") => { + println!("{HELP}"); + return Ok(()); } - }; - wx.open_table(MEMORIES)?.insert(id.as_slice(), quidity)?; - index(&wx, &id, quidity)?; - wx.commit()?; - Ok(id) -} - -pub fn remember(this: &[u8]) -> anyhow::Result> { - match MEMORY.begin_read()?.open_table(MEMORIES)?.get(this)? { - Some(ag) => Ok(ag.value().to_vec()), - None => Err(anyhow!("didn't find that")), - } -} - -/// FIX 2: forget a memory and every link to it. Notes are memories of their own, so the -/// notes on it stay; they are only no longer linked to it. -pub fn forget(this: &[u8]) -> anyhow::Result> { - let wx = MEMORY.begin_write()?; - let old = wx.open_table(MEMORIES)?.remove(this)?.map(|g| g.value().to_vec()); - let Some(old) = old else { - wx.abort()?; - return Err(anyhow!("there was nothing with that name anyways")); - }; - unindex(&wx, this, &old)?; - wx.open_multimap_table(TO_REUSE)?.insert(ALLOCATED, this)?; - // the notes on it - for note in take_all(&wx, NOTES, this)? { - wx.open_multimap_table(NOTED_ON)?.remove(note.as_slice(), this)?; - } - // what it was a note on - for on in take_all(&wx, NOTED_ON, this)? { - wx.open_multimap_table(NOTES)?.remove(on.as_slice(), this)?; - } - // the lists it was in - for list in take_all(&wx, LISTED_IN, this)? { - wx.open_multimap_table(LISTS)?.remove(list.as_slice(), this)?; - } - wx.commit()?; - Ok(old) -} - -// ---- lists ---- - -/// put a memory into a list (it must exist, so lists never point at nothing) -pub fn add_to_list(ln: &[u8], v: &[u8]) -> anyhow::Result<()> { - let wx = MEMORY.begin_write()?; - if !exists(&wx, v)? { - wx.abort()?; - return Err(anyhow!("no memory with that id to list")); - } - wx.open_multimap_table(LISTS)?.insert(ln, v)?; - wx.open_multimap_table(LISTED_IN)?.insert(v, ln)?; - wx.commit()?; - Ok(()) -} - -pub fn rm_from_list(ln: &[u8], v: &[u8]) -> anyhow::Result<()> { - let wx = MEMORY.begin_write()?; - wx.open_multimap_table(LISTS)?.remove(ln, v)?; - wx.open_multimap_table(LISTED_IN)?.remove(v, ln)?; - wx.commit()?; - Ok(()) -} - -/// a list's memories by id (empty if the list is empty or unknown) -pub fn get_list(ln: &[u8]) -> anyhow::Result, Vec>> { - let rx = MEMORY.begin_read()?; - let lists = rx.open_multimap_table(LISTS)?; - let memories = rx.open_table(MEMORIES)?; - let mut hm = HashMap::new(); - for r in lists.get(ln)? { - let k = r?; - if let Some(v) = memories.get(k.value())? { - hm.insert(k.value().to_vec(), v.value().to_vec()); + Some("-V" | "--version") => { + println!("neum {}", env!("CARGO_PKG_VERSION")); + return Ok(()); } + _ => {} } - Ok(hm) -} - -/// the lists a memory is in -pub fn lists_of(v: &[u8]) -> anyhow::Result>> { - let rx = MEMORY.begin_read()?; - let listed = rx.open_multimap_table(LISTED_IN)?; - let mut out = vec![]; - for r in listed.get(v)? { - out.push(r?.value().to_vec()); - } - Ok(out) -} - -// ---- notes ---- -fn link_note(wx: &WriteTransaction, note: &[u8], on: &[u8]) -> anyhow::Result<()> { - if !exists(wx, note)? || !exists(wx, on)? { - return Err(anyhow!("a note and what it is on must both exist")); - } - wx.open_multimap_table(NOTES)?.insert(on, note)?; - wx.open_multimap_table(NOTED_ON)?.insert(note, on)?; - Ok(()) -} - -pub fn note_on(note: &[u8], on: &[u8]) -> anyhow::Result<()> { - let wx = MEMORY.begin_write()?; - link_note(&wx, note, on)?; - wx.commit()?; - Ok(()) -} - -/// many (on, note) pairs in one transaction (one commit, so one fsync: use this for imports) -pub fn notes_on<'a>(pairs: impl IntoIterator) -> anyhow::Result<()> { - let wx = MEMORY.begin_write()?; - for (on, note) in pairs { - link_note(&wx, note, on)?; - } - wx.commit()?; - Ok(()) -} - -pub fn rm_note(note: &[u8], on: &[u8]) -> anyhow::Result<()> { - let wx = MEMORY.begin_write()?; - wx.open_multimap_table(NOTES)?.remove(on, note)?; - wx.open_multimap_table(NOTED_ON)?.remove(note, on)?; - wx.commit()?; - Ok(()) -} - -/// get notes on a memory, by key (empty if none) -pub fn get_notes_on(this: &[u8]) -> anyhow::Result, Vec>> { - let rx = MEMORY.begin_read()?; - let notes = rx.open_multimap_table(NOTES)?; - let memories = rx.open_table(MEMORIES)?; - let mut found = HashMap::new(); - for r in notes.get(this)? { - let k = r?; - if let Some(v) = memories.get(k.value())? { - found.insert(k.value().to_vec(), v.value().to_vec()); - } - } - Ok(found) -} - -/// what a note is on -pub fn noted_on(note: &[u8]) -> anyhow::Result>> { - let rx = MEMORY.begin_read()?; - let noted = rx.open_multimap_table(NOTED_ON)?; - let mut out = vec![]; - for r in noted.get(note)? { - out.push(r?.value().to_vec()); - } - Ok(out) -} - -// ---- search ---- - -/// memories holding at least 3/4 of the query's distinct tokens. With -/// `also_saturate`, the query is stored as a memory too unless one already equals it; -/// its new id comes back second (and it is in the results). -pub fn search(query: &[u8], also_saturate: bool) -> anyhow::Result<(HashMap, Vec>, Option>)> { - if query.len() == 0 { return Err(anyhow!("can't look for nothing")); } - let tkns = unique_tokens(query); - let mut out = HashMap::new(); - let mut that_one: Option> = None; - if !tkns.is_empty() { - let threshold = ((tkns.len() as f32 * 0.75).ceil() as usize).max(1); - let rx = MEMORY.begin_read()?; - let mi = rx.open_multimap_table(MEMORY_INDEX)?; - let mem = rx.open_table(MEMORIES)?; - let mut scores: HashMap, usize> = HashMap::new(); - for tkn in &tkns { - for r in mi.get(*tkn)? { - *scores.entry(r?.value().to_vec()).or_insert(0) += 1; - } - } - for (k, hits) in scores { - if hits >= threshold { - if let Some(v) = mem.get(k.as_slice())? { - let val = v.value(); - if val == query { that_one = Some(k.clone()); } - out.insert(k, val.to_vec()); - } + let mut settings = Settings::load(); + let path = startup_memory(arg, &settings); + // a memory that won't open still leaves the window, where esc can open another + let greeting = match store::open(&path, settings.compact_on_open) { + Ok(_) => { + settings.opened(&canonical(&path)); + if let Err(e) = settings.save() { + eprintln!("couldn't save settings: {e}"); } + String::new() } - } - if that_one.is_none() { - that_one = if also_saturate && out.par_iter().find_any(|v| v.1 == query).is_none() { - let id = saturate(query, None)?; - out.insert(id.clone(), query.to_vec()); - Some(id) - } else { None }; - } - Ok((out, that_one)) -} - -/// memories whose ordinal (or, with `rev`, reversed) sum is `sum` -pub fn gsearch(sum: u64, rev: bool) -> anyhow::Result, Vec>> { - let rx = MEMORY.begin_read()?; - let mem = rx.open_table(MEMORIES)?; - let memg = rx.open_multimap_table(if rev { MEMORY_GEMATRIA_REVERSED } else { MEMORY_GEMATRIA })?; - let mut out = HashMap::new(); - for r in memg.get(sum)? { - let k = r?; - if let Some(v) = mem.get(k.value())? { - out.insert(k.value().to_vec(), v.value().to_vec()); - } - } - Ok(out) -} - -// ---- repair ---- - -/// rebuild what can be rebuilt and drop what points at nothing (see the module notes) -pub fn repair() -> anyhow::Result<()> { - let wx = MEMORY.begin_write()?; - repair_in(&wx)?; - wx.commit()?; - Ok(()) -} - -fn repair_in(wx: &WriteTransaction) -> anyhow::Result<()> { - let mut all: Vec<(Vec, Vec)> = vec![]; - for r in wx.open_table(MEMORIES)?.iter()? { - let (k, v) = r?; - all.push((k.value().to_vec(), v.value().to_vec())); - } - let live: HashSet> = all.iter().map(|(k, _)| k.clone()).collect(); - - // indexes from scratch - wx.delete_multimap_table(MEMORY_INDEX)?; - wx.delete_multimap_table(MEMORY_GEMATRIA)?; - wx.delete_multimap_table(MEMORY_GEMATRIA_REVERSED)?; - for (k, v) in &all { - index(wx, k, v)?; - } - - // lists: keep live members, rebuild the reverse - let pairs = multimap_pairs(wx, LISTS)?; - wx.delete_multimap_table(LISTS)?; - wx.delete_multimap_table(LISTED_IN)?; - { - let mut lists = wx.open_multimap_table(LISTS)?; - let mut listed = wx.open_multimap_table(LISTED_IN)?; - for (ln, v) in pairs.into_iter().filter(|(_, v)| live.contains(v)) { - lists.insert(ln.as_slice(), v.as_slice())?; - listed.insert(v.as_slice(), ln.as_slice())?; - } - } - - // notes: every link known from either side, if both ends are live - let mut links: HashSet<(Vec, Vec)> = multimap_pairs(wx, NOTES)?.into_iter().collect(); - links.extend(multimap_pairs(wx, NOTED_ON)?.into_iter().map(|(note, on)| (on, note))); - wx.delete_multimap_table(NOTES)?; - wx.delete_multimap_table(NOTED_ON)?; - { - let mut notes = wx.open_multimap_table(NOTES)?; - let mut noted = wx.open_multimap_table(NOTED_ON)?; - for (on, note) in links.into_iter().filter(|(on, note)| live.contains(on) && live.contains(note)) { - notes.insert(on.as_slice(), note.as_slice())?; - noted.insert(note.as_slice(), on.as_slice())?; - } - } - - // ids to reuse: only the ones not in use - let reusable: Vec> = multimap_pairs(wx, TO_REUSE)?.into_iter().map(|(_, id)| id).collect(); - wx.delete_multimap_table(TO_REUSE)?; - { - let mut reuse = wx.open_multimap_table(TO_REUSE)?; - for id in reusable.into_iter().filter(|id| !live.contains(id)) { - reuse.insert(ALLOCATED, id.as_slice())?; - } - } - - // the counter at least as far as the largest id in use - let mut state = wx.open_table(STATE)?; - let last = state.get(ALLOCATED)?.map(|g| g.value().to_vec()); - let top = live.iter().max_by(|a, b| id_order(a).cmp(&id_order(b))).cloned(); - if let Some(top) = top { - if last.as_deref().is_none_or(|l| id_order(&top) > id_order(l)) { - state.insert(ALLOCATED, top.as_slice())?; - } - } - drop(state); - // tables emptied above come back even when nothing was put in them - ensure_tables(wx) -} - -fn multimap_pairs(wx: &WriteTransaction, t: MultimapTableDefinition<&[u8], &[u8]>) -> anyhow::Result, Vec)>> { - let table = wx.open_multimap_table(t)?; - let mut out = vec![]; - for r in table.iter()? { - let (k, vs) = r?; - for v in vs { - out.push((k.value().to_vec(), v?.value().to_vec())); - } - } - Ok(out) -} - -// ---- backup ---- - -/// bytes as JSON: {"text": ..} when they are UTF-8, else {"hex": ..} -fn enc(b: &[u8]) -> Value { - match std::str::from_utf8(b) { - Ok(s) => json!({ "text": s }), - Err(_) => json!({ "hex": b.iter().map(|x| format!("{x:02x}")).collect::() }), - } -} - -fn dec(v: &Value) -> anyhow::Result> { - if let Some(s) = v.get("text").and_then(Value::as_str) { - return Ok(s.as_bytes().to_vec()); - } - let h = v.get("hex").and_then(Value::as_str).ok_or_else(|| anyhow!("neither text nor hex: {v}"))?; - if h.len() % 2 != 0 { - return Err(anyhow!("odd hex length")); - } - (0..h.len()).step_by(2).map(|i| Ok(u8::from_str_radix(&h[i..i + 2], 16)?)).collect() -} - -fn hex(b: &[u8]) -> String { - b.iter().map(|x| format!("{x:02x}")).collect() -} - -fn unhex(s: &str) -> anyhow::Result> { - dec(&json!({ "hex": s })) -} - -/// everything needed to restore, as JSON at `path` (your old one was -/// "./it-all.json"). Returns how many memories were written. -pub fn backup_memories_to_file(path: &str) -> anyhow::Result { - let rx = MEMORY.begin_read()?; - let mem = rx.open_table(MEMORIES)?; - let mut memories = vec![]; - for r in mem.iter()? { - let (k, v) = r?; - memories.push(json!({ "id": hex(k.value()), "value": enc(v.value()) })); - } - let pairs = |t: MultimapTableDefinition<&[u8], &[u8]>| -> anyhow::Result, Vec)>> { - let table = rx.open_multimap_table(t)?; - let mut out = vec![]; - for r in table.iter()? { - let (k, vs) = r?; - for v in vs { - out.push((k.value().to_vec(), v?.value().to_vec())); - } - } - Ok(out) + Err(e) => format!("{e} · esc to open another"), }; - let lists: Vec = pairs(LISTS)?.iter().map(|(ln, id)| json!({ "list": enc(ln), "id": hex(id) })).collect(); - let notes: Vec = pairs(NOTES)?.iter().map(|(on, note)| json!({ "on": hex(on), "note": hex(note) })).collect(); - let reuse: Vec = pairs(TO_REUSE)?.iter().map(|(_, id)| hex(id)).collect(); - let allocated = rx.open_table(STATE)?.get(ALLOCATED)?.map(|g| hex(g.value())); - let all = json!({ - "allocated": allocated, - "reuse": reuse, - "memories": memories, - "lists": lists, - "notes": notes, - }); - std::fs::write(path, serde_json::to_string(&all)?)?; - Ok(mem.len()?) -} - -/// read a backup into an empty database (refuses to mix one into memories already there) -pub fn restore_from_file(path: &str) -> anyhow::Result { - let all: Value = serde_json::from_str(&std::fs::read_to_string(path)?)?; - let wx = MEMORY.begin_write()?; - if !wx.open_table(MEMORIES)?.is_empty()? { - wx.abort()?; - return Err(anyhow!("restore wants an empty database")); - } - let arr = |k: &str| all.get(k).and_then(Value::as_array).cloned().unwrap_or_default(); - let field = |v: &Value, k: &str| -> anyhow::Result { - Ok(v.get(k).and_then(Value::as_str).ok_or_else(|| anyhow!("missing {k}"))?.to_string()) - }; - let mut n = 0; - for m in arr("memories") { - let id = unhex(&field(&m, "id")?)?; - let v = dec(m.get("value").ok_or_else(|| anyhow!("missing value"))?)?; - wx.open_table(MEMORIES)?.insert(id.as_slice(), v.as_slice())?; - n += 1; - } - for l in arr("lists") { - let ln = dec(l.get("list").ok_or_else(|| anyhow!("missing list"))?)?; - wx.open_multimap_table(LISTS)?.insert(ln.as_slice(), unhex(&field(&l, "id")?)?.as_slice())?; - } - for x in arr("notes") { - let (on, note) = (unhex(&field(&x, "on")?)?, unhex(&field(&x, "note")?)?); - wx.open_multimap_table(NOTES)?.insert(on.as_slice(), note.as_slice())?; - } - for id in arr("reuse") { - let id = unhex(id.as_str().ok_or_else(|| anyhow!("reuse ids are hex strings"))?)?; - wx.open_multimap_table(TO_REUSE)?.insert(ALLOCATED, id.as_slice())?; - } - if let Some(a) = all.get("allocated").and_then(Value::as_str) { - wx.open_table(STATE)?.insert(ALLOCATED, unhex(a)?.as_slice())?; - } - // indexes, reverse tables and the counter follow from what was read - repair_in(&wx)?; - wx.commit()?; - Ok(n) -} - -/// AVX2 fast path from weum: 32 bytes per block, 4 blocks unrolled, PSADBW accumulation. -/// -/// Unlike weum's kernel, the letter test is `(b | 0x20) - 'a' <= 25` rather than -/// `(b | 0x20) - ('a' - 1) <= 26`: with the latter, '@' and '`' pass as letters of value -/// 0 and add 27 to the reversed sum. -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn dual_sum_avx2(data: &[u8]) -> (u64, u64) { - let lower = _mm256_set1_epi8(0x20); - let a = _mm256_set1_epi8(b'a' as i8); - let max = _mm256_set1_epi8(25); - let one = _mm256_set1_epi8(1); - let twentyseven = _mm256_set1_epi8(27); - let zero = _mm256_setzero_si256(); - - let mut s = [zero; 4]; - let mut r = [zero; 4]; - - macro_rules! block { - ($i:expr, $p:expr) => {{ - let v = unsafe { _mm256_loadu_si256($p as *const __m256i) }; - // 0 for 'a' .. 25 for 'z', anything else lands above 25 - let off = _mm256_sub_epi8(_mm256_or_si256(v, lower), a); - let mask = _mm256_cmpeq_epi8(_mm256_min_epu8(off, max), off); - let gval = _mm256_and_si256(_mm256_add_epi8(off, one), mask); - let rval = _mm256_and_si256(_mm256_sub_epi8(twentyseven, gval), mask); - s[$i] = _mm256_add_epi64(s[$i], _mm256_sad_epu8(gval, zero)); - r[$i] = _mm256_add_epi64(r[$i], _mm256_sad_epu8(rval, zero)); - }}; - } - - let len = data.len(); - let ptr = data.as_ptr(); - let mut i = 0; - while i + 128 <= len { - block!(0, ptr.add(i)); - block!(1, ptr.add(i + 32)); - block!(2, ptr.add(i + 64)); - block!(3, ptr.add(i + 96)); - i += 128; - } - while i + 32 <= len { - block!(0, ptr.add(i)); - i += 32; - } - - let reduce = |acc: [__m256i; 4]| -> u64 { - let t = _mm256_add_epi64(_mm256_add_epi64(acc[0], acc[1]), _mm256_add_epi64(acc[2], acc[3])); - let lanes = _mm_add_epi64(_mm256_castsi256_si128(t), _mm256_extracti128_si256(t, 1)); - (_mm_cvtsi128_si64(lanes) as u64).wrapping_add(_mm_extract_epi64(lanes, 1) as u64) - }; - let (ordinal, reversed) = sums_scalar(&data[i..]); - (reduce(s) + ordinal, reduce(r) + reversed) -} - -pub fn sums_scalar(data: &[u8]) -> (u64, u64) { - let (mut o, mut r) = (0u64, 0u64); - for &b in data { - let lc = b | 0x20; - if lc.is_ascii_lowercase() { - let v = (lc - b'a' + 1) as u64; - o += v; - r += 27 - v; - } - } - (o, r) -} - -pub fn sums(data: &[u8]) -> (u64, u64) { - #[cfg(target_arch = "x86_64")] - if std::is_x86_feature_detected!("avx2") { - return unsafe { dual_sum_avx2(data) }; - } - sums_scalar(data) -} - -fn is_delimiter(b: u8) -> bool { - matches!(b, b' ' | b'\n' | b'\t' | b'\r') -} - -/// splits on spaces, newlines, tabs and carriage returns like weum's `tokenate`; tokens -/// are the raw bytes between delimiters, in order -pub fn tokenate(data: &[u8]) -> Vec<&[u8]> { - #[cfg(target_arch = "x86_64")] - if std::is_x86_feature_detected!("avx2") { - return unsafe { tokenize_avx2(data) }; - } - tokenize_scalar(data) -} - -pub fn tokenize_scalar(data: &[u8]) -> Vec<&[u8]> { - data.split(|&b| is_delimiter(b)).filter(|t| !t.is_empty()).collect() -} - -/// 32 bytes at a time: one delimiter bitmask per block, from which word starts and ends -/// fall out as bit transitions (carried across blocks) -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn tokenize_avx2(data: &[u8]) -> Vec<&[u8]> { - let space = _mm256_set1_epi8(b' ' as i8); - let newline = _mm256_set1_epi8(b'\n' as i8); - let tab = _mm256_set1_epi8(b'\t' as i8); - let cr = _mm256_set1_epi8(b'\r' as i8); - - let mut tokens = Vec::new(); - let mut start = 0usize; - let mut in_word = false; - let len = data.len(); - let mut i = 0; - while i + 32 <= len { - let v = unsafe { _mm256_loadu_si256(data.as_ptr().add(i) as *const __m256i) }; - let delim = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(v, space), _mm256_cmpeq_epi8(v, newline)), - _mm256_or_si256(_mm256_cmpeq_epi8(v, tab), _mm256_cmpeq_epi8(v, cr)), - ); - let word = !(_mm256_movemask_epi8(delim) as u32); - // a bit per byte saying whether the byte before it was part of a word - let before = (word << 1) | in_word as u32; - let mut edges = (word & !before) | (!word & before); - while edges != 0 { - let bit = edges.trailing_zeros() as usize; - if word >> bit & 1 == 1 { - start = i + bit; - } else { - tokens.push(&data[start..i + bit]); - } - edges &= edges - 1; - } - in_word = word >> 31 == 1; - i += 32; - } - for (j, &b) in data.iter().enumerate().skip(i) { - match (is_delimiter(b), in_word) { - (true, true) => { - tokens.push(&data[start..j]); - in_word = false; - } - (false, false) => { - start = j; - in_word = true; - } - _ => {} - } - } - if in_word { - tokens.push(&data[start..]); - } - tokens -} - -/// each distinct token once (sorted by bytes). FIX: `dedup` alone only removes repeats -/// that sit next to each other, so "a b a" kept both a's -pub fn unique_tokens(data: &[u8]) -> Vec<&[u8]> { - let mut tkns = tokenate(data); - tkns.sort_unstable(); - tkns.dedup(); - tkns -} - -#[allow(dead_code)] -fn permsq(a: u64, b: u64) -> String { - let mut digits = vec![1u8]; - for _ in 0..b { - let mut carry = 0u64; - for d in digits.iter_mut().rev() { - let prod = (*d as u64) * a + carry; - *d = (prod % 10) as u8; - carry = prod / 10; - } - while carry > 0 { - digits.insert(0, (carry % 10) as u8); - carry /= 10; - } - } - digits.into_iter().map(|d| (d + b'0') as char).collect() -} - -/// the byte sums of some data -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct ByteSums { - /// Σ (b + 1) - pub bytes: u64, - /// Σ (256 − b) - pub reversed: u64, - /// Σ digit_sum(b + 1) - pub digits: u64, - /// how many bytes were summed - pub len: u64, -} - -impl ByteSums { - /// the digit roots of `bytes`, `reversed` and `digits` (0 for 0, else 1 + (n − 1) mod 9) - pub fn roots(&self) -> [u64; 3] { - [digit_root(self.bytes), digit_root(self.reversed), digit_root(self.digits)] - } - - /// the mean byte value, 1 ..= 256 (0 for no bytes) - pub fn mean(&self) -> f64 { - if self.len == 0 { 0.0 } else { self.bytes as f64 / self.len as f64 } - } -} - -/// the value of one byte: 1 ..= 256 -pub fn value(b: u8) -> u64 { - b as u64 + 1 -} - -/// the sum of n's decimal digits -pub fn digit_sum(mut n: u64) -> u64 { - let mut s = 0; - while n > 0 { - s += n % 10; - n /= 10; - } - s -} + if !greeting.is_empty() { eprintln!("{greeting}"); } -/// 0 for 0, else 1 + (n − 1) mod 9 -pub fn digit_root(n: u64) -> u64 { - if n == 0 { 0 } else { 1 + (n - 1) % 9 } -} - -const DIGITS: [u8; 256] = { - let mut t = [0u8; 256]; - let mut i = 0; - while i < 256 { - let mut n = i + 1; - let mut s = 0; - while n > 0 { - s += n % 10; - n /= 10; - } - t[i] = s as u8; - i += 1; - } - t -}; - -/// every byte sum of `data` -pub fn byte_sums(data: &[u8]) -> ByteSums { - let (mut bytes, mut digits) = (0u64, 0u64); - for &b in data { - bytes += b as u64; - digits += DIGITS[b as usize] as u64; - } - let len = data.len() as u64; - bytes += len; - ByteSums {bytes, reversed: 257 * len - bytes, digits, len} -} - -/// Σ (b + 1) alone -pub fn byte_sum(data: &[u8]) -> u64 { - data.iter().map(|&b| b as u64).sum::() + data.len() as u64 -} - -pub fn prime_factors(mut n: u64) -> Vec { - let mut found = Vec::new(); - let mut d = 2; - while n > 1 { - while n.is_multiple_of(d) { - found.push(d); - n /= d; - } - d += 1; - if d * d > n && n > 1 { - found.push(n); - break; - } - } - found -} - -fn pf_sum(n: u64) -> u64 { - prime_factors(n).iter().sum() -} - -pub fn pf_reduction(s: u64) -> (Vec, u64, u64) { - const LIMIT: usize = 32; - let pf = prime_factors(s); - let pfr: u64 = pf.iter().sum(); - let ds = digit_sum(s); - let dp = ds * pfr; - let mut chain: Vec = Vec::new(); - for _ in 0..=LIMIT { - let xn = match chain.last() { - Some(&last) => pf_sum(last) * digit_sum(last), - None => pf_sum(dp) * ds, - }; - if chain.contains(&xn) { break; } - chain.push(xn); - } - let total = chain.iter().sum::() + pfr; - (chain, total, digit_sum(total)) -} - -#[allow(dead_code)] -fn vowel(c: u8) -> u64 { - match c { - b'a' => 1, b'e' => 2, - b'o' => 3, b'u' => 4, - b'i' => 5, b'y' => 6, - _ => 0, - } -} + let neum = app() + .with_title("Neum") // .with_decorations(Decorations::NoTitleBar) + .with_stylesheet(STYLE) + .with_size(666.0, 444.0); -#[allow(dead_code)] -fn consonant(c: u8) -> u64 { - match c { - b'b' => 1, b'c' => 2, b'd' => 3, b'f' => 4, - b'g' => 5, b'h' => 6, b'j' => 7, b'k' => 8, - b'l' => 9, b'm' => 10, b'n' => 11, b'p' => 12, - b'q' => 13, b'r' => 14, b's' => 15, b't' => 16, - b'v' => 17, b'w' => 18, b'x' => 19, b'z' => 20, - _ => 0, - } + neum.run(move || { + let (settings, greeting) = (settings.clone(), greeting.clone()); + view! { Gematria(settings = settings, greeting = greeting) } + })?; + store::close(); + Ok(()) } diff --git a/src/settings.rs b/src/settings.rs new file mode 100644 index 0000000..86eb76e --- /dev/null +++ b/src/settings.rs @@ -0,0 +1,139 @@ +//! what neum remembers between runs, as JSON in the config folder + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::gematria::Mode; + +/// how many saves the escape menu keeps at hand +pub const RECENT: usize = 8; + +/// the thresholds the escape menu cycles through, in percent +pub const THRESHOLDS: [u8; 4] = [50, 75, 90, 100]; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Settings { + /// light mode instead of dark + pub light: bool, + /// the memory to open at start (the last one open) + pub memory: String, + /// memories opened before, newest first: the saves + pub recent: Vec, + /// compact a memory's file when opening it + pub compact_on_open: bool, + /// whether the saturate toggle starts on + pub saturate: bool, + /// which sum the middle letter starts on + pub mode: Mode, + /// percent of a query's distinct words a memory must hold for search to find it + pub threshold: u8, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + light: false, + memory: String::new(), + recent: vec![], + compact_on_open: true, + saturate: true, + mode: Mode::Ordinal, + threshold: 75, + } + } +} + +impl Settings { + /// where settings live: $NEUM_CONFIG, else neum/settings.json in the platform's config + /// folder ($XDG_CONFIG_HOME, ~/.config, %APPDATA%), else next to the working directory + pub fn path() -> PathBuf { + if let Some(p) = std::env::var_os("NEUM_CONFIG").filter(|p| !p.is_empty()) { + return PathBuf::from(p); + } + let base = std::env::var_os("XDG_CONFIG_HOME") + .filter(|p| !p.is_empty()) + .map(PathBuf::from) + .or_else(|| std::env::var_os("APPDATA").filter(|p| !p.is_empty()).map(PathBuf::from)) + .or_else(|| std::env::var_os("HOME").filter(|p| !p.is_empty()).map(|h| PathBuf::from(h).join(".config"))); + match base { + Some(b) => b.join("neum").join("settings.json"), + None => PathBuf::from("neum-settings.json"), + } + } + + /// settings from `path`; the defaults if there are none there or they don't parse + pub fn load_from(path: &Path) -> Settings { + match std::fs::read_to_string(path) { + Ok(s) => serde_json::from_str::(&s) + .map(Settings::sane) + .unwrap_or_else(|e| { + eprintln!("settings at {} don't parse ({e}), using defaults", path.display()); + Settings::default() + }), + Err(_) => Settings::default(), + } + } + + pub fn load() -> Settings { + Settings::load_from(&Settings::path()) + } + + pub fn save_to(&self, path: &Path) -> anyhow::Result<()> { + if let Some(dir) = path.parent() + && !dir.as_os_str().is_empty() + { + std::fs::create_dir_all(dir)?; + } + // written whole then moved, so a crash mid-write leaves the old settings + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, serde_json::to_string_pretty(self)?)?; + std::fs::rename(&tmp, path)?; + Ok(()) + } + + pub fn save(&self) -> anyhow::Result<()> { + self.save_to(&Settings::path()) + } + + /// the threshold as a fraction, for search + pub fn threshold_fraction(&self) -> f32 { + self.threshold.clamp(1, 100) as f32 / 100.0 + } + + /// the next threshold in [`THRESHOLDS`] after this one + pub fn next_threshold(&self) -> u8 { + THRESHOLDS.iter().copied().find(|&t| t > self.threshold).unwrap_or(THRESHOLDS[0]) + } + + /// `path` is the memory open now: first among the saves, and the one to open next time + pub fn opened(&mut self, path: &str) { + self.memory = path.to_string(); + self.recent.retain(|p| p != path); + self.recent.insert(0, path.to_string()); + self.recent.truncate(RECENT); + } + + /// take a save off the list (the file stays) + pub fn forget_save(&mut self, path: &str) { + self.recent.retain(|p| p != path); + } + + fn sane(mut self) -> Settings { + self.threshold = self.threshold.clamp(1, 100); + self.recent.retain(|p| !p.is_empty()); + self.recent.dedup(); + self.recent.truncate(RECENT); + self + } +} + +/// which memory to open at start: the first argument, else $NEUM_MEMORY, else the last one +/// open, else `./memory` +pub fn startup_memory(arg: Option, settings: &Settings) -> String { + arg.filter(|a| !a.is_empty()) + .or_else(|| std::env::var("NEUM_MEMORY").ok().filter(|p| !p.is_empty())) + .or_else(|| Some(settings.memory.clone()).filter(|p| !p.is_empty())) + .unwrap_or_else(|| "./memory".into()) +} diff --git a/src/store.rs b/src/store.rs new file mode 100644 index 0000000..781d25d --- /dev/null +++ b/src/store.rs @@ -0,0 +1,821 @@ +//! the memory: a redb file of memories, their indexes, lists and notes. +//! +//! One memory is open at a time, like a game save: [`open`] swaps it for another file, +//! [`close`] puts it away. [`Memory`] is the same thing without the global, which is what +//! tests use. + +use anyhow::anyhow; + +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, + sync::{Arc, RwLock}, +}; + +use rayon::prelude::*; +use redb::{ + Database, MultimapTableDefinition, ReadableDatabase, ReadableMultimapTable, ReadableTable, + ReadableTableMetadata, TableDefinition, WriteTransaction, +}; + +use serde_json::{Value, json}; + +use crate::gematria::{Mode, byte_sum, sums, unique_tokens}; + +const ALLOCATED: &[u8] = b"allocated".as_slice(); +const VERSION: &[u8] = b"version".as_slice(); + +/// 1: memories, indexes, lists and notes. 2: the byte sum bucket. +pub const SCHEMA: u8 = 2; + +/// how much of a query's distinct tokens a memory must hold to be found, unless told otherwise +pub const DEFAULT_THRESHOLD: f32 = 0.75; + +macro_rules! multimap { + ($i:ident, $k:ty, $v:ty) => { + const $i: MultimapTableDefinition<$k, $v> = MultimapTableDefinition::new(stringify!($i)); + }; +} + +macro_rules! map { + ($i:ident, $k:ty, $v:ty) => { + const $i: TableDefinition<$k, $v> = TableDefinition::new(stringify!($i)); + }; +} + +map!(STATE, &[u8], &[u8]); +map!(MEMORIES, &[u8], &[u8]); +multimap!(MEMORY_INDEX, &[u8], &[u8]); + +multimap!(TO_REUSE, &[u8], &[u8]); + +multimap!(LISTS, &[u8], &[u8]); +multimap!(LISTED_IN, &[u8], &[u8]); + +multimap!(NOTES, &[u8], &[u8]); +multimap!(NOTED_ON, &[u8], &[u8]); + +multimap!(MEMORY_GEMATRIA, u64, &[u8]); +multimap!(MEMORY_GEMATRIA_REVERSED, u64, &[u8]); +multimap!(MEMORY_BYTE_SUM, u64, &[u8]); + +/// memories by id +pub type Found = HashMap, Vec>; + +/// what the escape menu says about a memory +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Stats { + pub path: PathBuf, + /// the file's size on disk + pub file_bytes: u64, + pub memories: u64, + /// the bytes of every memory added up + pub stored_bytes: u64, + /// distinct lists, and how many memories are in them altogether + pub lists: u64, + pub listed: u64, + /// note links, and how many memories have at least one note on them + pub notes: u64, + pub noted: u64, + /// the last id handed out, and how many freed ids wait to be handed out again + pub allocated: Option>, + pub reusable: u64, + pub schema: u8, +} + +pub struct Memory { + db: Database, + path: PathBuf, +} + +impl std::fmt::Debug for Memory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Memory").field("path", &self.path).finish() + } +} + +fn ensure_tables(wx: &WriteTransaction) -> anyhow::Result<()> { + wx.open_table(STATE)?; + wx.open_table(MEMORIES)?; + for t in [MEMORY_INDEX, TO_REUSE, LISTS, LISTED_IN, NOTES, NOTED_ON] { + wx.open_multimap_table(t)?; + } + for t in [MEMORY_GEMATRIA, MEMORY_GEMATRIA_REVERSED, MEMORY_BYTE_SUM] { + wx.open_multimap_table(t)?; + } + Ok(()) +} + +fn schema_of(wx: &WriteTransaction) -> anyhow::Result { + Ok(wx.open_table(STATE)?.get(VERSION)?.and_then(|g| g.value().first().copied()).unwrap_or(1)) +} + +// ---- ids ---- + +/// ids are big-endian counters that grow a byte when they run out: [0]..[255], [0,0].., +/// up to 4 bytes (the same sequence your `saturate` made) +pub fn next_id(prev: &[u8]) -> anyhow::Result> { + let mut b = prev.to_vec(); + for i in (0..b.len()).rev() { + if b[i] == u8::MAX { + b[i] = 0; + } else { + b[i] += 1; + return Ok(b); + } + } + if b.len() >= 4 { + return Err(anyhow!("ran out of 4-byte ids")); + } + Ok(vec![0; b.len() + 1]) +} + +/// which of two ids came later in that sequence +pub fn id_order(id: &[u8]) -> (usize, &[u8]) { + (id.len(), id) +} + +fn exists(wx: &WriteTransaction, id: &[u8]) -> anyhow::Result { + Ok(wx.open_table(MEMORIES)?.get(id)?.is_some()) +} + +/// a fresh id: the smallest one to reuse, else one past ALLOCATED, else [0] +fn allocate(wx: &WriteTransaction) -> anyhow::Result> { + let mut reusables = wx.open_multimap_table(TO_REUSE)?; + let reuse = match reusables.get(ALLOCATED)?.next() { + Some(r) => Some(r?.value().to_vec()), + None => None, + }; + if let Some(id) = reuse { + reusables.remove(ALLOCATED, id.as_slice())?; + return Ok(id); + } + let mut state = wx.open_table(STATE)?; + let last = state.get(ALLOCATED)?.map(|g| g.value().to_vec()); + let id = match last { + Some(last) => next_id(&last)?, + None => vec![0], + }; + state.insert(ALLOCATED, id.as_slice())?; + Ok(id) +} + +// ---- indexing ---- + +fn index(wx: &WriteTransaction, id: &[u8], value: &[u8]) -> anyhow::Result<()> { + let (f, r) = sums(value); + wx.open_multimap_table(MEMORY_GEMATRIA)?.insert(f, id)?; + wx.open_multimap_table(MEMORY_GEMATRIA_REVERSED)?.insert(r, id)?; + wx.open_multimap_table(MEMORY_BYTE_SUM)?.insert(byte_sum(value), id)?; + let mut memi = wx.open_multimap_table(MEMORY_INDEX)?; + for tk in unique_tokens(value) { + memi.insert(tk, id)?; + } + Ok(()) +} + +fn unindex(wx: &WriteTransaction, id: &[u8], value: &[u8]) -> anyhow::Result<()> { + let (f, r) = sums(value); + wx.open_multimap_table(MEMORY_GEMATRIA)?.remove(f, id)?; + wx.open_multimap_table(MEMORY_GEMATRIA_REVERSED)?.remove(r, id)?; + wx.open_multimap_table(MEMORY_BYTE_SUM)?.remove(byte_sum(value), id)?; + let mut memi = wx.open_multimap_table(MEMORY_INDEX)?; + for tk in unique_tokens(value) { + memi.remove(tk, id)?; + } + Ok(()) +} + +/// every value of a multimap key, removed, collected (so the table can be used again) +fn take_all(wx: &WriteTransaction, t: MultimapTableDefinition<&[u8], &[u8]>, k: &[u8]) -> anyhow::Result>> { + let mut table = wx.open_multimap_table(t)?; + let mut out = vec![]; + for r in table.remove_all(k)? { + out.push(r?.value().to_vec()); + } + Ok(out) +} + +fn link_note(wx: &WriteTransaction, note: &[u8], on: &[u8]) -> anyhow::Result<()> { + if !exists(wx, note)? || !exists(wx, on)? { + return Err(anyhow!("a note and what it is on must both exist")); + } + if note == on { + return Err(anyhow!("a memory can't be a note on itself")); + } + wx.open_multimap_table(NOTES)?.insert(on, note)?; + wx.open_multimap_table(NOTED_ON)?.insert(note, on)?; + Ok(()) +} + +impl Memory { + /// open the memory at `path`, making it (and its folder) if it isn't there. Older + /// schemas are brought up to date; with `compact`, the file is compacted too. + pub fn open(path: impl AsRef, compact: bool) -> anyhow::Result { + let path = path.as_ref().to_path_buf(); + if path.as_os_str().is_empty() { + return Err(anyhow!("a memory needs a path")); + } + if path.is_dir() { + return Err(anyhow!("{} is a folder, a memory is a file", path.display())); + } + if let Some(dir) = path.parent() + && !dir.as_os_str().is_empty() && !dir.exists() + { + std::fs::create_dir_all(dir)?; + } + let mut db = Database::create(&path).map_err(|e| match e { + redb::DatabaseError::DatabaseAlreadyOpen => anyhow!("{} is already open (in another neum?)", path.display()), + e => anyhow!("couldn't open {}: {e}", path.display()), + })?; + let wx = db.begin_write()?; + ensure_tables(&wx)?; + if schema_of(&wx)? < SCHEMA { + // the byte sum bucket (and anything else an index can be rebuilt from) + repair_in(&wx)?; + wx.open_table(STATE)?.insert(VERSION, [SCHEMA].as_slice())?; + } + wx.commit()?; + if compact { + db.compact()?; + } + Ok(Memory { db, path }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + // ---- memories ---- + + /// with `None`, store a new memory under a fresh id; with `Some(id)`, put this + /// value at that id, moving its index entries from the old value (if there was one) to + /// the new. Returns the id. + pub fn saturate(&self, quidity: &[u8], t: Option>) -> anyhow::Result> { + let wx = self.db.begin_write()?; + let id = match t { + None => allocate(&wx)?, + Some(id) => { + if id.is_empty() || id.len() > 4 { + return Err(anyhow!("ids are 1 to 4 bytes")); + } + let old = wx.open_table(MEMORIES)?.get(id.as_slice())?.map(|g| g.value().to_vec()); + if let Some(old) = old { + unindex(&wx, &id, &old)?; + } // an id written by hand is in use now: not to be handed out again + wx.open_multimap_table(TO_REUSE)?.remove(ALLOCATED, id.as_slice())?; + let mut state = wx.open_table(STATE)?; + let last = state.get(ALLOCATED)?.map(|g| g.value().to_vec()); + if last.as_deref().is_none_or(|l| id_order(&id) > id_order(l)) { + state.insert(ALLOCATED, id.as_slice())?; + } + id + } + }; + wx.open_table(MEMORIES)?.insert(id.as_slice(), quidity)?; + index(&wx, &id, quidity)?; + wx.commit()?; + Ok(id) + } + + pub fn remember(&self, this: &[u8]) -> anyhow::Result> { + match self.db.begin_read()?.open_table(MEMORIES)?.get(this)? { + Some(ag) => Ok(ag.value().to_vec()), + None => Err(anyhow!("didn't find that")), + } + } + + /// forget a memory and every link to it. Notes are memories of their own, so the + /// notes on it stay; they are only no longer linked to it. + pub fn forget(&self, this: &[u8]) -> anyhow::Result> { + let wx = self.db.begin_write()?; + let old = wx.open_table(MEMORIES)?.remove(this)?.map(|g| g.value().to_vec()); + let Some(old) = old else { + wx.abort()?; + return Err(anyhow!("there was nothing with that name anyways")); + }; + unindex(&wx, this, &old)?; + wx.open_multimap_table(TO_REUSE)?.insert(ALLOCATED, this)?; + // the notes on it + for note in take_all(&wx, NOTES, this)? { + wx.open_multimap_table(NOTED_ON)?.remove(note.as_slice(), this)?; + } + // what it was a note on + for on in take_all(&wx, NOTED_ON, this)? { + wx.open_multimap_table(NOTES)?.remove(on.as_slice(), this)?; + } + // the lists it was in + for list in take_all(&wx, LISTED_IN, this)? { + wx.open_multimap_table(LISTS)?.remove(list.as_slice(), this)?; + } + // and, if it was a list, its members' way back to it + for member in take_all(&wx, LISTS, this)? { + wx.open_multimap_table(LISTED_IN)?.remove(member.as_slice(), this)?; + } + wx.commit()?; + Ok(old) + } + + // ---- lists ---- + + /// put a memory into a list (it must exist, so lists never point at nothing) + pub fn add_to_list(&self, ln: &[u8], v: &[u8]) -> anyhow::Result<()> { + let wx = self.db.begin_write()?; + if !exists(&wx, v)? { + wx.abort()?; + return Err(anyhow!("no memory with that id to list")); + } + wx.open_multimap_table(LISTS)?.insert(ln, v)?; + wx.open_multimap_table(LISTED_IN)?.insert(v, ln)?; + wx.commit()?; + Ok(()) + } + + pub fn rm_from_list(&self, ln: &[u8], v: &[u8]) -> anyhow::Result<()> { + let wx = self.db.begin_write()?; + wx.open_multimap_table(LISTS)?.remove(ln, v)?; + wx.open_multimap_table(LISTED_IN)?.remove(v, ln)?; + wx.commit()?; + Ok(()) + } + + /// a list's memories by id (empty if the list is empty or unknown) + pub fn get_list(&self, ln: &[u8]) -> anyhow::Result { + let rx = self.db.begin_read()?; + let lists = rx.open_multimap_table(LISTS)?; + let memories = rx.open_table(MEMORIES)?; + let mut hm = HashMap::new(); + for r in lists.get(ln)? { + let k = r?; + if let Some(v) = memories.get(k.value())? { + hm.insert(k.value().to_vec(), v.value().to_vec()); + } + } + Ok(hm) + } + + /// the lists a memory is in + pub fn lists_of(&self, v: &[u8]) -> anyhow::Result>> { + let rx = self.db.begin_read()?; + let listed = rx.open_multimap_table(LISTED_IN)?; + let mut out = vec![]; + for r in listed.get(v)? { + out.push(r?.value().to_vec()); + } + Ok(out) + } + + // ---- notes ---- + + /// `note` (a memory) is now a note on `on` (another) + pub fn note_on(&self, note: &[u8], on: &[u8]) -> anyhow::Result<()> { + let wx = self.db.begin_write()?; + link_note(&wx, note, on)?; + wx.commit()?; + Ok(()) + } + + /// many (on, note) pairs in one transaction (one commit, so one fsync: use this for imports) + pub fn notes_on<'a>(&self, pairs: impl IntoIterator) -> anyhow::Result<()> { + let wx = self.db.begin_write()?; + for (on, note) in pairs { + link_note(&wx, note, on)?; + } + wx.commit()?; + Ok(()) + } + + pub fn rm_note(&self, note: &[u8], on: &[u8]) -> anyhow::Result<()> { + let wx = self.db.begin_write()?; + wx.open_multimap_table(NOTES)?.remove(on, note)?; + wx.open_multimap_table(NOTED_ON)?.remove(note, on)?; + wx.commit()?; + Ok(()) + } + + /// get notes on a memory, by key (empty if none) + pub fn get_notes_on(&self, this: &[u8]) -> anyhow::Result { + let rx = self.db.begin_read()?; + let notes = rx.open_multimap_table(NOTES)?; + let memories = rx.open_table(MEMORIES)?; + let mut found = HashMap::new(); + for r in notes.get(this)? { + let k = r?; + if let Some(v) = memories.get(k.value())? { + found.insert(k.value().to_vec(), v.value().to_vec()); + } + } + Ok(found) + } + + /// what a note is on + pub fn noted_on(&self, note: &[u8]) -> anyhow::Result>> { + let rx = self.db.begin_read()?; + let noted = rx.open_multimap_table(NOTED_ON)?; + let mut out = vec![]; + for r in noted.get(note)? { + out.push(r?.value().to_vec()); + } + Ok(out) + } + + // ---- search ---- + + /// [`Memory::search_with`] at the default threshold + pub fn search(&self, query: &[u8], also_saturate: bool) -> anyhow::Result<(Found, Option>)> { + self.search_with(query, also_saturate, DEFAULT_THRESHOLD) + } + + /// memories holding at least `threshold` (0..=1) of the query's distinct tokens. With + /// `also_saturate`, the query is stored as a memory too unless one already equals it. + /// The id of the memory equal to the query, if there is one (new or old), comes back + /// second (and it is in the results). + pub fn search_with(&self, query: &[u8], also_saturate: bool, threshold: f32) -> anyhow::Result<(Found, Option>)> { + if query.is_empty() { return Err(anyhow!("can't look for nothing")); } + let tkns = unique_tokens(query); + let mut out = HashMap::new(); + let mut that_one: Option> = None; + if !tkns.is_empty() { + let threshold = ((tkns.len() as f32 * threshold.clamp(0.0, 1.0)).ceil() as usize).max(1); + let rx = self.db.begin_read()?; + let mi = rx.open_multimap_table(MEMORY_INDEX)?; + let mem = rx.open_table(MEMORIES)?; + let mut scores: HashMap, usize> = HashMap::new(); + for tkn in &tkns { + for r in mi.get(*tkn)? { + *scores.entry(r?.value().to_vec()).or_insert(0) += 1; + } + } + for (k, hits) in scores { + if hits >= threshold + && let Some(v) = mem.get(k.as_slice())? + { + let val = v.value(); + if val == query { that_one = Some(k.clone()); } + out.insert(k, val.to_vec()); + } + } + } + if that_one.is_none() { + that_one = if also_saturate && out.par_iter().find_any(|v| v.1 == query).is_none() { + let id = self.saturate(query, None)?; + out.insert(id.clone(), query.to_vec()); + Some(id) + } else { None }; + } + Ok((out, that_one)) + } + + /// memories whose ordinal, reversed or byte sum is `sum` + pub fn gsearch(&self, sum: u64, mode: Mode) -> anyhow::Result { + let rx = self.db.begin_read()?; + let mem = rx.open_table(MEMORIES)?; + let memg = rx.open_multimap_table(match mode { + Mode::Ordinal => MEMORY_GEMATRIA, + Mode::Reversed => MEMORY_GEMATRIA_REVERSED, + Mode::Bytes => MEMORY_BYTE_SUM, + })?; + let mut out = HashMap::new(); + for r in memg.get(sum)? { + let k = r?; + if let Some(v) = mem.get(k.value())? { + out.insert(k.value().to_vec(), v.value().to_vec()); + } + } + Ok(out) + } + + /// the id of the memory that is exactly `value`, if there is one + pub fn find(&self, value: &[u8]) -> anyhow::Result>> { + let found = self.gsearch(byte_sum(value), Mode::Bytes)?; + Ok(found.into_iter().find(|(_, v)| v == value).map(|(k, _)| k)) + } + + /// the id of the memory that is exactly `value`, saturating it first if there isn't one + pub fn find_or_saturate(&self, value: &[u8]) -> anyhow::Result> { + match self.find(value)? { + Some(id) => Ok(id), + None => self.saturate(value, None), + } + } + + // ---- repair ---- + + /// rebuild what can be rebuilt and drop what points at nothing + pub fn repair(&self) -> anyhow::Result<()> { + let wx = self.db.begin_write()?; + repair_in(&wx)?; + wx.commit()?; + Ok(()) + } + + /// compact the file, if redb finds anything to give back + pub fn compact(&mut self) -> anyhow::Result { + Ok(self.db.compact()?) + } + + // ---- stats ---- + + pub fn stats(&self) -> anyhow::Result { + let rx = self.db.begin_read()?; + let mem = rx.open_table(MEMORIES)?; + let mut stored_bytes = 0; + for r in mem.iter()? { + stored_bytes += r?.1.value().len() as u64; + } + let keys_and_len = |t: MultimapTableDefinition<&[u8], &[u8]>| -> anyhow::Result<(u64, u64)> { + let table = rx.open_multimap_table(t)?; + let (mut keys, mut len) = (0, 0); + for r in table.iter()? { + let (_, vs) = r?; + keys += 1; + len += vs.len(); + } + Ok((keys, len)) + }; + let (lists, listed) = keys_and_len(LISTS)?; + let (noted, notes) = keys_and_len(NOTES)?; + let (_, reusable) = keys_and_len(TO_REUSE)?; + let state = rx.open_table(STATE)?; + let allocated = state.get(ALLOCATED)?.map(|g| g.value().to_vec()); + let schema = state.get(VERSION)?.and_then(|g| g.value().first().copied()).unwrap_or(1); + Ok(Stats { + path: self.path.clone(), + file_bytes: std::fs::metadata(&self.path).map(|m| m.len()).unwrap_or(0), + memories: mem.len()?, + stored_bytes, + lists, + listed, + notes, + noted, + allocated, + reusable, + schema, + }) + } + + // ---- backup ---- + + /// everything needed to restore, as JSON at `path`. Returns how many memories were written. + pub fn backup_to_file(&self, path: impl AsRef) -> anyhow::Result { + let rx = self.db.begin_read()?; + let mem = rx.open_table(MEMORIES)?; + let mut memories = vec![]; + for r in mem.iter()? { + let (k, v) = r?; + memories.push(json!({ "id": hex(k.value()), "value": enc(v.value()) })); + } + let pairs = |t: MultimapTableDefinition<&[u8], &[u8]>| -> anyhow::Result, Vec)>> { + let table = rx.open_multimap_table(t)?; + let mut out = vec![]; + for r in table.iter()? { + let (k, vs) = r?; + for v in vs { + out.push((k.value().to_vec(), v?.value().to_vec())); + } + } + Ok(out) + }; + let lists: Vec = pairs(LISTS)?.iter().map(|(ln, id)| json!({ "list": enc(ln), "id": hex(id) })).collect(); + let notes: Vec = pairs(NOTES)?.iter().map(|(on, note)| json!({ "on": hex(on), "note": hex(note) })).collect(); + let reuse: Vec = pairs(TO_REUSE)?.iter().map(|(_, id)| hex(id)).collect(); + let allocated = rx.open_table(STATE)?.get(ALLOCATED)?.map(|g| hex(g.value())); + let all = json!({ + "neum": SCHEMA, + "allocated": allocated, + "reuse": reuse, + "memories": memories, + "lists": lists, + "notes": notes, + }); + if let Some(dir) = path.as_ref().parent() + && !dir.as_os_str().is_empty() && !dir.exists() + { + std::fs::create_dir_all(dir)?; + } + std::fs::write(path, serde_json::to_string(&all)?)?; + Ok(mem.len()?) + } + + /// read a backup into an empty memory (refuses to mix one into memories already there) + pub fn restore_from_file(&self, path: impl AsRef) -> anyhow::Result { + let all: Value = serde_json::from_str(&std::fs::read_to_string(path)?)?; + let wx = self.db.begin_write()?; + if !wx.open_table(MEMORIES)?.is_empty()? { + wx.abort()?; + return Err(anyhow!("restore wants an empty memory (open a new one first)")); + } + let arr = |k: &str| all.get(k).and_then(Value::as_array).cloned().unwrap_or_default(); + let field = |v: &Value, k: &str| -> anyhow::Result { + Ok(v.get(k).and_then(Value::as_str).ok_or_else(|| anyhow!("missing {k}"))?.to_string()) + }; + let mut n = 0; + for m in arr("memories") { + let id = unhex(&field(&m, "id")?)?; + let v = dec(m.get("value").ok_or_else(|| anyhow!("missing value"))?)?; + wx.open_table(MEMORIES)?.insert(id.as_slice(), v.as_slice())?; + n += 1; + } + for l in arr("lists") { + let ln = dec(l.get("list").ok_or_else(|| anyhow!("missing list"))?)?; + wx.open_multimap_table(LISTS)?.insert(ln.as_slice(), unhex(&field(&l, "id")?)?.as_slice())?; + } + for x in arr("notes") { + let (on, note) = (unhex(&field(&x, "on")?)?, unhex(&field(&x, "note")?)?); + wx.open_multimap_table(NOTES)?.insert(on.as_slice(), note.as_slice())?; + } + for id in arr("reuse") { + let id = unhex(id.as_str().ok_or_else(|| anyhow!("reuse ids are hex strings"))?)?; + wx.open_multimap_table(TO_REUSE)?.insert(ALLOCATED, id.as_slice())?; + } + if let Some(a) = all.get("allocated").and_then(Value::as_str) { + wx.open_table(STATE)?.insert(ALLOCATED, unhex(a)?.as_slice())?; + } + // indexes, reverse tables and the counter follow from what was read + repair_in(&wx)?; + wx.commit()?; + Ok(n) + } +} + +fn repair_in(wx: &WriteTransaction) -> anyhow::Result<()> { + let mut all: Vec<(Vec, Vec)> = vec![]; + for r in wx.open_table(MEMORIES)?.iter()? { + let (k, v) = r?; + all.push((k.value().to_vec(), v.value().to_vec())); + } + let live: HashSet> = all.iter().map(|(k, _)| k.clone()).collect(); + + // indexes from scratch + wx.delete_multimap_table(MEMORY_INDEX)?; + wx.delete_multimap_table(MEMORY_GEMATRIA)?; + wx.delete_multimap_table(MEMORY_GEMATRIA_REVERSED)?; + wx.delete_multimap_table(MEMORY_BYTE_SUM)?; + for (k, v) in &all { + index(wx, k, v)?; + } + + // lists: keep live members, rebuild the reverse + let pairs = multimap_pairs(wx, LISTS)?; + wx.delete_multimap_table(LISTS)?; + wx.delete_multimap_table(LISTED_IN)?; + { + let mut lists = wx.open_multimap_table(LISTS)?; + let mut listed = wx.open_multimap_table(LISTED_IN)?; + for (ln, v) in pairs.into_iter().filter(|(_, v)| live.contains(v)) { + lists.insert(ln.as_slice(), v.as_slice())?; + listed.insert(v.as_slice(), ln.as_slice())?; + } + } + + // notes: every link known from either side, if both ends are live + let mut links: HashSet<(Vec, Vec)> = multimap_pairs(wx, NOTES)?.into_iter().collect(); + links.extend(multimap_pairs(wx, NOTED_ON)?.into_iter().map(|(note, on)| (on, note))); + wx.delete_multimap_table(NOTES)?; + wx.delete_multimap_table(NOTED_ON)?; + { + let mut notes = wx.open_multimap_table(NOTES)?; + let mut noted = wx.open_multimap_table(NOTED_ON)?; + for (on, note) in links.into_iter().filter(|(on, note)| on != note && live.contains(on) && live.contains(note)) { + notes.insert(on.as_slice(), note.as_slice())?; + noted.insert(note.as_slice(), on.as_slice())?; + } + } + + // ids to reuse: only the ones not in use + let reusable: Vec> = multimap_pairs(wx, TO_REUSE)?.into_iter().map(|(_, id)| id).collect(); + wx.delete_multimap_table(TO_REUSE)?; + { + let mut reuse = wx.open_multimap_table(TO_REUSE)?; + for id in reusable.into_iter().filter(|id| !live.contains(id)) { + reuse.insert(ALLOCATED, id.as_slice())?; + } + } + + // the counter at least as far as the largest id in use + let mut state = wx.open_table(STATE)?; + let last = state.get(ALLOCATED)?.map(|g| g.value().to_vec()); + let top = live.iter().max_by(|a, b| id_order(a).cmp(&id_order(b))).cloned(); + if let Some(top) = top + && last.as_deref().is_none_or(|l| id_order(&top) > id_order(l)) + { + state.insert(ALLOCATED, top.as_slice())?; + } + drop(state); + // tables emptied above come back even when nothing was put in them + ensure_tables(wx) +} + +fn multimap_pairs(wx: &WriteTransaction, t: MultimapTableDefinition<&[u8], &[u8]>) -> anyhow::Result, Vec)>> { + let table = wx.open_multimap_table(t)?; + let mut out = vec![]; + for r in table.iter()? { + let (k, vs) = r?; + for v in vs { + out.push((k.value().to_vec(), v?.value().to_vec())); + } + } + Ok(out) +} + +// ---- backup encoding ---- + +/// bytes as JSON: {"text": ..} when they are UTF-8, else {"hex": ..} +pub fn enc(b: &[u8]) -> Value { + match std::str::from_utf8(b) { + Ok(s) => json!({ "text": s }), + Err(_) => json!({ "hex": hex(b) }), + } +} + +pub fn dec(v: &Value) -> anyhow::Result> { + if let Some(s) = v.get("text").and_then(Value::as_str) { + return Ok(s.as_bytes().to_vec()); + } + let h = v.get("hex").and_then(Value::as_str).ok_or_else(|| anyhow!("neither text nor hex: {v}"))?; + unhex(h) +} + +pub fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +pub fn unhex(h: &str) -> anyhow::Result> { + if !h.len().is_multiple_of(2) { + return Err(anyhow!("odd hex length")); + } + if !h.is_ascii() { + return Err(anyhow!("hex is ascii")); + } + (0..h.len()).step_by(2).map(|i| Ok(u8::from_str_radix(&h[i..i + 2], 16)?)).collect() +} + +// ---- the open memory ---- + +static CURRENT: RwLock>> = RwLock::new(None); + +/// the memory that's open, or why there isn't one +pub fn current() -> anyhow::Result> { + CURRENT + .read() + .unwrap_or_else(|e| e.into_inner()) + .clone() + .ok_or_else(|| anyhow!("no memory open (esc to open one)")) +} + +pub fn is_open() -> bool { + CURRENT.read().unwrap_or_else(|e| e.into_inner()).is_some() +} + +/// where the open memory lives +pub fn current_path() -> Option { + CURRENT.read().unwrap_or_else(|e| e.into_inner()).as_ref().map(|m| m.path.clone()) +} + +/// open the memory at `path` in place of whichever was open. The old one stays open if the +/// new one can't be. +pub fn open(path: impl AsRef, compact: bool) -> anyhow::Result> { + let path = path.as_ref(); + let mut cur = CURRENT.write().unwrap_or_else(|e| e.into_inner()); + if let Some(m) = cur.as_ref() + && same_file(m.path(), path) + { + return Ok(m.clone()); + } + let m = Arc::new(Memory::open(path, compact)?); + *cur = Some(m.clone()); + Ok(m) +} + +/// put the open memory away. Returns where it was. +pub fn close() -> Option { + let old = CURRENT.write().unwrap_or_else(|e| e.into_inner()).take(); + old.map(|m| m.path.clone()) +} + +fn same_file(a: &Path, b: &Path) -> bool { + a == b || matches!((a.canonicalize(), b.canonicalize()), (Ok(a), Ok(b)) if a == b) +} + +// the same calls on whichever memory is open + +pub fn saturate(quidity: &[u8], t: Option>) -> anyhow::Result> { current()?.saturate(quidity, t) } +pub fn remember(this: &[u8]) -> anyhow::Result> { current()?.remember(this) } +pub fn forget(this: &[u8]) -> anyhow::Result> { current()?.forget(this) } +pub fn add_to_list(ln: &[u8], v: &[u8]) -> anyhow::Result<()> { current()?.add_to_list(ln, v) } +pub fn rm_from_list(ln: &[u8], v: &[u8]) -> anyhow::Result<()> { current()?.rm_from_list(ln, v) } +pub fn get_list(ln: &[u8]) -> anyhow::Result { current()?.get_list(ln) } +pub fn lists_of(v: &[u8]) -> anyhow::Result>> { current()?.lists_of(v) } +pub fn note_on(note: &[u8], on: &[u8]) -> anyhow::Result<()> { current()?.note_on(note, on) } +pub fn rm_note(note: &[u8], on: &[u8]) -> anyhow::Result<()> { current()?.rm_note(note, on) } +pub fn get_notes_on(this: &[u8]) -> anyhow::Result { current()?.get_notes_on(this) } +pub fn noted_on(note: &[u8]) -> anyhow::Result>> { current()?.noted_on(note) } +pub fn search(query: &[u8], also_saturate: bool) -> anyhow::Result<(Found, Option>)> { current()?.search(query, also_saturate) } +pub fn search_with(query: &[u8], also_saturate: bool, threshold: f32) -> anyhow::Result<(Found, Option>)> { current()?.search_with(query, also_saturate, threshold) } +pub fn gsearch(sum: u64, mode: Mode) -> anyhow::Result { current()?.gsearch(sum, mode) } +pub fn find(value: &[u8]) -> anyhow::Result>> { current()?.find(value) } +pub fn find_or_saturate(value: &[u8]) -> anyhow::Result> { current()?.find_or_saturate(value) } +pub fn repair() -> anyhow::Result<()> { current()?.repair() } +pub fn stats() -> anyhow::Result { current()?.stats() } +pub fn backup_memories_to_file(path: impl AsRef) -> anyhow::Result { current()?.backup_to_file(path) } +pub fn restore_from_file(path: impl AsRef) -> anyhow::Result { current()?.restore_from_file(path) } diff --git a/tests/current.rs b/tests/current.rs new file mode 100644 index 0000000..9a33926 --- /dev/null +++ b/tests/current.rs @@ -0,0 +1,45 @@ +//! the one open memory: opening, swapping and closing like game saves. One test, since +//! the open memory is shared by the whole test binary. + +use neum::{Mode, gematria::byte_sum, store}; + +#[test] +fn saves() { + let d = tempfile::tempdir().unwrap(); + let (one, two) = (d.path().join("one"), d.path().join("two")); + + assert!(!store::is_open()); + assert!(store::saturate(b"nowhere", None).is_err(), "nothing open, nothing done"); + assert!(store::current().unwrap_err().to_string().contains("no memory open")); + + store::open(&one, false).unwrap(); + assert_eq!(store::current_path().as_deref(), Some(one.as_path())); + let a = store::saturate(b"in one", None).unwrap(); + + // the same file again is the same memory + let again = store::open(&one, false).unwrap(); + assert_eq!(again.path(), one.as_path()); + // a handle kept past a swap keeps its file locked, so let it go + drop(again); + + // a second save, then back + store::open(&two, false).unwrap(); + assert!(store::remember(&a).is_err(), "two has nothing yet"); + let b = store::saturate(b"in two", None).unwrap(); + assert_eq!(a, b, "each save counts its own ids"); + store::open(&one, false).unwrap(); + assert_eq!(store::remember(&a).unwrap(), b"in one"); + assert!(store::gsearch(byte_sum(b"in one"), Mode::Bytes).unwrap().contains_key(&a)); + + // one that can't open leaves the open one be + assert!(store::open(d.path(), false).is_err()); + assert_eq!(store::current_path().as_deref(), Some(one.as_path())); + + assert_eq!(store::close().as_deref(), Some(one.as_path())); + assert!(!store::is_open()); + assert_eq!(store::close(), None); + + // closed means the file is free for anyone + let m = neum::Memory::open(&one, false).unwrap(); + assert_eq!(m.remember(&a).unwrap(), b"in one"); +} diff --git a/tests/settings.rs b/tests/settings.rs new file mode 100644 index 0000000..194cdf6 --- /dev/null +++ b/tests/settings.rs @@ -0,0 +1,74 @@ +use neum::{Mode, Settings, settings::{RECENT, THRESHOLDS, startup_memory}}; + +#[test] +fn defaults_when_missing_or_broken() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("settings.json"); + assert_eq!(Settings::load_from(&p), Settings::default()); + std::fs::write(&p, "{ not json").unwrap(); + assert_eq!(Settings::load_from(&p), Settings::default()); + // fields it doesn't know are dropped, fields it misses are defaults + std::fs::write(&p, r#"{"light": true, "whatever": 1}"#).unwrap(); + let s = Settings::load_from(&p); + assert!(s.light); + assert_eq!(s.threshold, 75); +} + +#[test] +fn round_trip() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("deeper").join("settings.json"); + let mut s = Settings { light: true, mode: Mode::Bytes, threshold: 90, ..Default::default() }; + s.opened("/a"); + s.save_to(&p).unwrap(); + assert_eq!(Settings::load_from(&p), s); + assert!(std::fs::read_to_string(&p).unwrap().contains(r#""mode": "bytes""#)); +} + +#[test] +fn saves_are_newest_first_and_capped() { + let mut s = Settings::default(); + for i in 0..RECENT + 3 { + s.opened(&format!("/m{i}")); + } + s.opened("/m5"); + assert_eq!(s.recent.len(), RECENT); + assert_eq!(s.recent[0], "/m5"); + assert_eq!(s.memory, "/m5"); + assert_eq!(s.recent.iter().filter(|p| *p == "/m5").count(), 1); + s.forget_save("/m5"); + assert!(!s.recent.contains(&"/m5".to_string())); +} + +#[test] +fn thresholds_cycle() { + let mut s = Settings::default(); + let mut seen = vec![]; + for _ in 0..THRESHOLDS.len() { + s.threshold = s.next_threshold(); + seen.push(s.threshold); + } + assert_eq!(seen, [90, 100, 50, 75]); + assert_eq!(s.threshold_fraction(), 0.75); +} + +#[test] +fn which_memory_opens() { + let mut s = Settings::default(); + // NEUM_MEMORY isn't set by the test runner; if a developer has it set, skip what it decides + if std::env::var("NEUM_MEMORY").is_ok() { + return; + } + assert_eq!(startup_memory(None, &s), "./memory"); + s.memory = "/last".into(); + assert_eq!(startup_memory(None, &s), "/last"); + assert_eq!(startup_memory(Some("/arg".into()), &s), "/arg"); +} + +#[test] +fn human_bytes() { + assert_eq!(neum::human_bytes(0), "0 B"); + assert_eq!(neum::human_bytes(1023), "1023 B"); + assert_eq!(neum::human_bytes(1536), "1.5 KiB"); + assert_eq!(neum::human_bytes(5 * 1024 * 1024), "5.0 MiB"); +} diff --git a/tests/store.rs b/tests/store.rs new file mode 100644 index 0000000..46c4dc6 --- /dev/null +++ b/tests/store.rs @@ -0,0 +1,239 @@ +//! the memory, end to end, on files in a temporary folder + +use neum::{Memory, Mode, gematria::{byte_sum, sums}, store}; +use tempfile::TempDir; + +fn fresh() -> (TempDir, Memory) { + let dir = tempfile::tempdir().unwrap(); + let m = Memory::open(dir.path().join("memory"), false).unwrap(); + (dir, m) +} + +#[test] +fn saturate_remember_forget() { + let (_d, m) = fresh(); + let a = m.saturate(b"count your says", None).unwrap(); + let b = m.saturate(b"just a regular friday", None).unwrap(); + assert_eq!(a, vec![0]); + assert_eq!(b, vec![1]); + assert_eq!(m.remember(&a).unwrap(), b"count your says"); + assert_eq!(m.forget(&a).unwrap(), b"count your says"); + assert!(m.remember(&a).is_err()); + assert!(m.forget(&a).is_err(), "forgetting twice"); + // the freed id is handed out again before a new one + assert_eq!(m.saturate(b"again", None).unwrap(), vec![0]); + assert_eq!(m.saturate(b"new", None).unwrap(), vec![2]); +} + +#[test] +fn ids_grow_a_byte_when_they_run_out() { + assert_eq!(store::next_id(&[0]).unwrap(), vec![1]); + assert_eq!(store::next_id(&[255]).unwrap(), vec![0, 0]); + assert_eq!(store::next_id(&[0, 255]).unwrap(), vec![1, 0]); + assert_eq!(store::next_id(&[255, 255, 255]).unwrap(), vec![0, 0, 0, 0]); + assert!(store::next_id(&[255, 255, 255, 255]).is_err()); + assert!(store::id_order(&[0, 0]) > store::id_order(&[255])); +} + +#[test] +fn saturate_at_an_id_moves_the_index() { + let (_d, m) = fresh(); + let id = m.saturate(b"abc", None).unwrap(); + m.saturate(b"zzz", Some(id.clone())).unwrap(); + assert!(m.gsearch(sums(b"abc").0, Mode::Ordinal).unwrap().is_empty()); + assert_eq!(m.gsearch(sums(b"zzz").0, Mode::Ordinal).unwrap()[&id], b"zzz"); + // a hand-written id past the counter moves the counter + m.saturate(b"far", Some(vec![9, 9])).unwrap(); + assert_eq!(m.saturate(b"next", None).unwrap(), vec![9, 10]); + assert!(m.saturate(b"bad", Some(vec![])).is_err()); + assert!(m.saturate(b"bad", Some(vec![1, 2, 3, 4, 5])).is_err()); +} + +#[test] +fn every_sum_has_a_bucket() { + let (_d, m) = fresh(); + let id = m.saturate(b"ordinary cunt", None).unwrap(); + let other = m.saturate("ünï".as_bytes(), None).unwrap(); + let (o, r) = sums(b"ordinary cunt"); + assert!(m.gsearch(o, Mode::Ordinal).unwrap().contains_key(&id)); + assert!(m.gsearch(r, Mode::Reversed).unwrap().contains_key(&id)); + assert!(m.gsearch(byte_sum(b"ordinary cunt"), Mode::Bytes).unwrap().contains_key(&id)); + // non-letters still have a byte sum + assert!(m.gsearch(byte_sum("ünï".as_bytes()), Mode::Bytes).unwrap().contains_key(&other)); + assert!(m.gsearch(12345, Mode::Bytes).unwrap().is_empty()); + // raw, non UTF-8 bytes too + let raw = m.saturate(&[0, 159, 146, 150], None).unwrap(); + assert!(m.gsearch(byte_sum(&[0, 159, 146, 150]), Mode::Bytes).unwrap().contains_key(&raw)); +} + +#[test] +fn search_by_words() { + let (_d, m) = fresh(); + let a = m.saturate(b"the cat sat", None).unwrap(); + let b = m.saturate(b"the dog sat", None).unwrap(); + let (found, that_one) = m.search(b"the cat sat", false).unwrap(); + assert!(found.contains_key(&a) && !found.contains_key(&b), "3 of 3 words needed at 75%"); + assert_eq!(that_one, Some(a.clone())); + let (found, _) = m.search_with(b"the cat sat", false, 0.5).unwrap(); + assert!(found.contains_key(&a) && found.contains_key(&b), "2 of 3 is enough at 50%"); + // saturating search stores what wasn't there, once + let (found, new) = m.search(b"a bird", true).unwrap(); + let new = new.unwrap(); + assert_eq!(found[&new], b"a bird"); + let (_, again) = m.search(b"a bird", true).unwrap(); + assert_eq!(again, Some(new)); + assert!(m.search(b"", false).is_err()); +} + +#[test] +fn find_and_find_or_saturate() { + let (_d, m) = fresh(); + assert_eq!(m.find(b"x").unwrap(), None); + let id = m.find_or_saturate(b"x").unwrap(); + assert_eq!(m.find(b"x").unwrap(), Some(id.clone())); + assert_eq!(m.find_or_saturate(b"x").unwrap(), id); +} + +#[test] +fn lists() { + let (_d, m) = fresh(); + let list = m.saturate(b"songs", None).unwrap(); + let a = m.saturate(b"wasn't our flow", None).unwrap(); + let b = m.saturate(b"say what you want", None).unwrap(); + m.add_to_list(&list, &a).unwrap(); + m.add_to_list(&list, &b).unwrap(); + assert!(m.add_to_list(&list, &[77]).is_err(), "lists never point at nothing"); + assert_eq!(m.get_list(&list).unwrap().len(), 2); + assert_eq!(m.lists_of(&a).unwrap(), vec![list.clone()]); + m.rm_from_list(&list, &a).unwrap(); + assert_eq!(m.get_list(&list).unwrap().len(), 1); + assert!(m.lists_of(&a).unwrap().is_empty()); + // forgetting a member takes it off its lists + m.forget(&b).unwrap(); + assert!(m.get_list(&list).unwrap().is_empty()); + // forgetting the list lets its members go + let c = m.saturate(b"c", None).unwrap(); + m.add_to_list(&list, &c).unwrap(); + m.forget(&list).unwrap(); + assert!(m.lists_of(&c).unwrap().is_empty()); +} + +#[test] +fn notes() { + let (_d, m) = fresh(); + let on = m.saturate(b"friday", None).unwrap(); + let n1 = m.saturate(b"get very nervous", None).unwrap(); + let n2 = m.saturate(b"can't have it both ways", None).unwrap(); + m.note_on(&n1, &on).unwrap(); + m.notes_on([(on.as_slice(), n2.as_slice())]).unwrap(); + assert!(m.note_on(&on, &on).is_err(), "not a note on itself"); + assert!(m.note_on(&[99], &on).is_err(), "notes must exist"); + let notes = m.get_notes_on(&on).unwrap(); + assert_eq!(notes.len(), 2); + assert_eq!(notes[&n1], b"get very nervous"); + assert_eq!(m.noted_on(&n1).unwrap(), vec![on.clone()]); + m.rm_note(&n1, &on).unwrap(); + assert_eq!(m.get_notes_on(&on).unwrap().len(), 1); + assert!(m.noted_on(&n1).unwrap().is_empty()); + // forgetting what notes are on keeps the notes, unlinked + m.forget(&on).unwrap(); + assert_eq!(m.remember(&n2).unwrap(), b"can't have it both ways"); + assert!(m.noted_on(&n2).unwrap().is_empty()); +} + +#[test] +fn stats_count_what_is_there() { + let (_d, m) = fresh(); + let s = m.stats().unwrap(); + assert_eq!((s.memories, s.lists, s.notes, s.reusable), (0, 0, 0, 0)); + assert_eq!(s.schema, store::SCHEMA); + assert!(s.file_bytes > 0); + let l = m.saturate(b"l", None).unwrap(); + let a = m.saturate(b"aa", None).unwrap(); + let b = m.saturate(b"bbb", None).unwrap(); + m.add_to_list(&l, &a).unwrap(); + m.add_to_list(&l, &b).unwrap(); + m.note_on(&a, &b).unwrap(); + let gone = m.saturate(b"gone", None).unwrap(); + m.forget(&gone).unwrap(); + let s = m.stats().unwrap(); + assert_eq!(s.memories, 3); + assert_eq!(s.stored_bytes, 6); + assert_eq!((s.lists, s.listed), (1, 2)); + assert_eq!((s.notes, s.noted), (1, 1)); + assert_eq!(s.reusable, 1); + assert_eq!(s.allocated, Some(gone)); +} + +#[test] +fn backup_and_restore() { + let (d, m) = fresh(); + let l = m.saturate(b"list", None).unwrap(); + let a = m.saturate(b"text", None).unwrap(); + let raw = m.saturate(&[0xff, 0x00, 0xfe], None).unwrap(); + m.add_to_list(&l, &a).unwrap(); + m.note_on(&raw, &a).unwrap(); + let gone = m.saturate(b"gone", None).unwrap(); + m.forget(&gone).unwrap(); + let json = d.path().join("backups").join("all.json"); + assert_eq!(m.backup_to_file(&json).unwrap(), 3); + + let other = Memory::open(d.path().join("other"), false).unwrap(); + assert_eq!(other.restore_from_file(&json).unwrap(), 3); + assert_eq!(other.remember(&raw).unwrap(), vec![0xff, 0x00, 0xfe]); + assert_eq!(other.get_list(&l).unwrap().len(), 1); + assert_eq!(other.noted_on(&raw).unwrap(), vec![a.clone()]); + assert!(other.gsearch(byte_sum(b"text"), Mode::Bytes).unwrap().contains_key(&a), "indexes rebuilt"); + assert_eq!(other.stats().unwrap(), neum::Stats { path: other.path().into(), file_bytes: other.stats().unwrap().file_bytes, ..m.stats().unwrap() }); + // the freed id came along + assert_eq!(other.saturate(b"new", None).unwrap(), gone); + // and it won't mix into a memory that has things in it + assert!(other.restore_from_file(&json).is_err()); +} + +#[test] +fn repair_rebuilds_and_drops() { + let (_d, m) = fresh(); + let a = m.saturate(b"alpha beta", None).unwrap(); + m.repair().unwrap(); + assert!(m.search(b"alpha beta", false).unwrap().0.contains_key(&a)); + assert!(m.gsearch(byte_sum(b"alpha beta"), Mode::Bytes).unwrap().contains_key(&a)); +} + +#[test] +fn reopening_keeps_everything_and_compacts() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("nested").join("folder").join("memory"); + let id = { + let m = Memory::open(&p, true).unwrap(); + m.saturate(b"kept", None).unwrap() + }; + let m = Memory::open(&p, true).unwrap(); + assert_eq!(m.remember(&id).unwrap(), b"kept"); + assert!(Memory::open(&p, false).is_err(), "a memory opens once"); + assert!(Memory::open(d.path(), false).is_err(), "a folder isn't a memory"); + assert!(Memory::open("", false).is_err()); +} + +#[test] +fn a_schema_1_memory_gets_its_byte_bucket() { + use redb::{Database, TableDefinition}; + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("old"); + { + // what 0.1 wrote: memories, no byte sum bucket, no version + const MEMORIES: TableDefinition<&[u8], &[u8]> = TableDefinition::new("MEMORIES"); + const STATE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("STATE"); + let db = Database::create(&p).unwrap(); + let wx = db.begin_write().unwrap(); + wx.open_table(MEMORIES).unwrap().insert([0u8].as_slice(), b"old times".as_slice()).unwrap(); + wx.open_table(STATE).unwrap().insert(b"allocated".as_slice(), [0u8].as_slice()).unwrap(); + wx.commit().unwrap(); + } + let m = Memory::open(&p, false).unwrap(); + assert_eq!(m.stats().unwrap().schema, store::SCHEMA); + assert!(m.gsearch(byte_sum(b"old times"), Mode::Bytes).unwrap().contains_key(&vec![0])); + assert!(m.gsearch(sums(b"old times").0, Mode::Ordinal).unwrap().contains_key(&vec![0])); + assert!(m.search(b"old times", false).unwrap().0.contains_key(&vec![0])); + assert_eq!(m.saturate(b"new", None).unwrap(), vec![1], "the counter carried on"); +}