diff --git a/.config/hakari.toml b/.config/hakari.toml new file mode 100644 index 00000000..bb5d464e --- /dev/null +++ b/.config/hakari.toml @@ -0,0 +1,27 @@ +# This file contains settings for `cargo hakari`. +# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options. + +hakari-package = "workspace-hack" + +# Format version for hakari's output. Version 4 requires cargo-hakari 0.9.22 or above. +dep-format-version = "4" + +# Setting workspace.resolver = "2" or higher in the root Cargo.toml is HIGHLY recommended. +# Hakari works much better with the v2 resolver. (The v2 and v3 resolvers are identical from +# hakari's perspective, so you're welcome to set either.) +# +# For more about the new feature resolver, see: +# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver +resolver = "3" + +# Add triples corresponding to platforms commonly used by developers here. +# https://doc.rust-lang.org/rustc/platform-support.html +platforms = [ + "x86_64-unknown-linux-gnu", + # "x86_64-apple-darwin", + # "aarch64-apple-darwin", + # "x86_64-pc-windows-msvc", +] + +# Write out exact versions rather than a semver range. (Defaults to false.) +# exact-versions = true diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..07be7bcf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Avoid putting conflict markers in the generated Cargo.toml file, since their presence breaks +# Cargo. +Cargo.lock merge=binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1073282e..ef0db1a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,15 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 - - # c.f. https://github.com/extractions/setup-just/issues/23 - - uses: extractions/setup-crate@4993624604c307fbca528d28a3c8b60fa5ecc859 # v1.4.0 - with: - repo: casey/just - version: 1.42.4 - - uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670 # v1.2.2 - with: - bins: cargo-nextest, cargo-insta@1.28.0 + - uses: ./.github/actions/setup-nix-direnv-rust - run: just test-unit test-integration: name: Integration test (& build all in dev-mode) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 250aa835..435cd524 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -3,8 +3,8 @@ name: "Opsqueue CI" agent: machine: - type: "f1-standard-2" - os_image: "ubuntu2204" + type: "f1-standard-4" + os_image: "ubuntu2404" # If we push a new build to some branch that isn't master, and another build is # already running, we cancel it. @@ -60,17 +60,14 @@ blocks: when: "branch = 'master'" task: jobs: - - name: "Build Opsqueue binary (in release mode) & run unit tests" + - name: "Build Opsqueue binary & Python library (in release mode) & run unit tests" commands: - - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-bin)) | grep -v '\\.drv$' | cachix push channable" + - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build)) | grep -v '\\.drv$' | cachix push channable" # Once a day, we update the devenv cache. # Except when fresh, this takes < 1 second # but it cannot live in the prologue # as that might corrupt it - cache store nix-store-$(date -u -Idate) /nix - - name: "Build Python library (in release mode)" - commands: - - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-python)) | grep -v '\\.drv$' | cachix push channable" - name: "Integration" dependencies: - "Build" diff --git a/Cargo.lock b/Cargo.lock index 4ea740a9..668797cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,15 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloca" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" -dependencies = [ - "cc", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -50,12 +41,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "1.0.0" @@ -121,12 +106,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "assert_matches" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" - [[package]] name = "async-channel" version = "2.5.0" @@ -367,12 +346,6 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.3.0" @@ -595,42 +568,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" -[[package]] -name = "criterion" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" -dependencies = [ - "alloca", - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "itertools 0.13.0", - "num-traits", - "oorandom", - "page_size", - "plotters", - "rayon", - "regex", - "serde", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" -dependencies = [ - "cast", - "itertools 0.13.0", -] - [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -640,16 +577,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -1516,15 +1443,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2082,12 +2000,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "openssl-probe" version = "0.2.1" @@ -2194,14 +2106,12 @@ version = "0.36.0" dependencies = [ "anyhow", "arc-swap", - "assert_matches", "axum", "axum-prometheus", "backon", "chrono", "ciborium", "clap", - "criterion", "crossbeam-skiplist", "dashmap", "either", @@ -2225,7 +2135,6 @@ dependencies = [ "sentry", "sentry-tracing", "serde", - "serde-error", "serde_json", "snowflaked", "sqlformat", @@ -2241,6 +2150,7 @@ dependencies = [ "url", "uuid", "ux", + "workspace-hack", ] [[package]] @@ -2260,6 +2170,7 @@ dependencies = [ "tracing", "uuid", "ux", + "workspace-hack", ] [[package]] @@ -2278,16 +2189,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking" version = "2.2.1" @@ -2355,34 +2256,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "portable-atomic" version = "1.14.0" @@ -2724,26 +2597,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -3139,15 +2992,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-error" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342110fb7a5d801060c885da03bf91bfa7c7ca936deafcc64bb6706375605d47" -dependencies = [ - "serde", -] - [[package]] name = "serde_core" version = "1.0.229" @@ -3390,6 +3234,7 @@ dependencies = [ "log", "memchr", "percent-encoding", + "rustls", "serde", "serde_json", "sha2 0.10.9", @@ -3399,6 +3244,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "webpki-roots", ] [[package]] @@ -3693,16 +3539,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.12.0" @@ -4521,6 +4357,60 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "workspace-hack" +version = "0.1.0" +dependencies = [ + "aws-lc-rs", + "base64", + "bitflags", + "cc", + "chrono", + "crossbeam-epoch", + "crossbeam-utils", + "either", + "event-listener", + "fastrand", + "futures-channel", + "futures-io", + "futures-sink", + "futures-util", + "hyper", + "hyper-util", + "libc", + "libsqlite3-sys", + "log", + "num-traits", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", + "rand 0.9.5", + "regex-automata", + "regex-syntax", + "reqwest", + "rustls-pki-types", + "serde", + "serde_core", + "serde_json", + "sha2 0.10.9", + "slab", + "smallvec", + "sqlx-core", + "sqlx-sqlite", + "syn 3.0.2", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tracing-core", + "typenum", + "url", + "uuid", + "zerocopy", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index a16b5baa..4baa1496 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,11 +6,73 @@ resolver = "3" members = [ "opsqueue/", "libs/opsqueue_python", + "workspace-hack", ] +[workspace.metadata.hakari] +hakari-package = "workspace-hack" + [workspace.package] +edition = "2024" version = "0.36.0" +[workspace.dependencies] +anyhow = { version = "1.0.102" } +arc-swap = { version = "1.9.1" } +assert_matches = { version = "1.5.0" } +axum = { version = "0.8.9", features = ["ws", "macros"] } +axum-prometheus = { version = "0.10.0", features = ["prometheus"] } +backon = { version = "1.3.0", features = ["tokio-sleep"] } +chrono = { version = "0.4.44", features = ["serde"] } +ciborium = { version = "0.2.2", features = ["std"] } +clap = { version = "4.5.60", features = ["derive", "std"] } +criterion = { version = "0.8", features = ["async_tokio"] } +crossbeam-skiplist = { version = "0.1.3", features = ["std"] } +dashmap = { version = "6.1.0" } +either = { version = "1.13.0" } +futures = { version = "0.3.32" } +http = { version = "1.4.0" } +humantime = { version = "2.1.0" } +insta = { version = "1.47.2" } +itertools = { version = "0.15.0" } +moka = { version = "0.12.15", features = ["sync"] } +moro-local = { version = "0.4.0" } +object_store = { version = "0.14.0", features = ["gcp", "http"] } +once_cell = { version = "1.21.4" } +opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } +opentelemetry-http = { version = "0.32" } +opentelemetry-otlp = { version = "0.32", features = ["grpc-tonic"] } +opentelemetry-resource-detectors = { version = "0.11.0" } +opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } +opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "rt-tokio"] } +opsqueue = { path = "opsqueue", default-features = false } +pyo3 = { version = "0.28.3", features = ["chrono"] } +pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime", "unstable-streams"] } +pyo3-log = { version = "0.13.3" } +rand = { version = "0.10.1" } +reqwest = { version = "0.13.2", default-features = false, features = ["json"] } +rustc-hash = { version = "2.1.2" } +sentry = { version = "0.48", default-features = false, features = ["rustls", "reqwest", "anyhow", "backtrace", "contexts", "tracing", "panic"] } +sentry-tracing = { version = "0.48" } +serde = { version = "1.0.203", features = ["derive"] } +serde-error = { version = "0.1.3" } +serde_json = { version = "1.0.149" } +snowflaked = { version = "1.0.3", features = ["sync"] } +sqlformat = { version = "0.5.0" } +sqlx = { version = "0.9.0", features = ["chrono", "derive", "json", "macros", "migrate", "runtime-tokio", "sqlite", "tls-rustls-aws-lc-rs"] } +thiserror = { version = "2.0.18" } +tokio = { version = "1.52.1", features = ["macros", "signal", "rt-multi-thread"] } +tokio-tungstenite = { version = "0.30.0", features = ["connect", "stream"] } +tokio-util = { version = "0.7.18", features = ["io", "rt", "time"] } +tower-http = { version = "0.7.0", features = ["trace", "catch-panic"] } +tracing = { version = "0.1.41", features = ["log"] } +tracing-opentelemetry = { version = "0.33.0" } +tracing-subscriber = { version = "0.3", features = ["std", "env-filter", "fmt"] } +url = { version = "2.5.2" } +uuid = { version = "1.23.1", features = ["v7", "fast-rng", "serde"] } +ux = { version = "0.1.6" } +workspace-hack = { version = "0.1", path = "workspace-hack" } + [workspace.lints.clippy] cargo = { level = "deny", priority = -1 } cargo_common_metadata = "allow" diff --git a/clippy.toml b/clippy.toml index 274bb7be..cb9f6293 100644 --- a/clippy.toml +++ b/clippy.toml @@ -5,6 +5,7 @@ allowed-duplicate-crates = [ "digest", "getrandom", "hashbrown", + "itertools", "r-efi", "rand", "rand_core", diff --git a/default.nix b/default.nix index 2da1a739..ef1e7aed 100644 --- a/default.nix +++ b/default.nix @@ -48,9 +48,12 @@ let pkgs.cargo-audit pkgs.cargo-edit pkgs.cargo-insta + pkgs.cargo-hakari pkgs.cargo-nextest pkgs.maturin + # Resolve native sqlite from Nix for libsqlite3-sys + pkgs.pkg-config # sqlite3 binary, for easy debugging/introspection pkgs.sqlite ]; @@ -63,6 +66,12 @@ let # For the shell, libpython needs to be in the search path. pythonEnv ]; + shellHook = '' + # We need Cargo to re-run if the sqlite3 dev package changes, so we set this env var to point to it. + export LIBSQLITE3_SYS_USE_PKG_CONFIG="${pkgs.sqlite.dev}" + # This is needed for the libsqlite3-sys crate to find the correct sqlite3. + export PKG_CONFIG_PATH="${pkgs.sqlite.dev}/lib/pkgconfig''${PKG_CONFIG_PATH:+:''${PKG_CONFIG_PATH}}" + ''; }; }; in diff --git a/justfile b/justfile index 0cba2f34..7c8ad88e 100644 --- a/justfile +++ b/justfile @@ -81,7 +81,12 @@ lint-light *ARGS: # Run the slow linters/static analysers that need to look at everything [group('lint')] -lint-heavy: clippy mypy +lint-heavy: clippy hakari mypy + +# Verify the workspace-hack crate is up to date +[group('lint')] +hakari: + cargo hakari verify # Rust static analysis [group('lint')] diff --git a/libs/opsqueue_python/Cargo.toml b/libs/opsqueue_python/Cargo.toml index 67d10548..4cd01f26 100644 --- a/libs/opsqueue_python/Cargo.toml +++ b/libs/opsqueue_python/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "opsqueue_python" version.workspace = true -edition = "2024" +edition.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] @@ -11,32 +11,31 @@ crate-type = ["cdylib"] [dependencies] # Opsqueue itself: # NOTE: We compile without the default `sqlx` feature, reducing the amount of dependencies -opsqueue = {path = "../../opsqueue/", default-features = false, features = ["client-logic"] } +opsqueue = { workspace = true, features = ["client-logic"] } # General-purpose datatypes: -uuid = {version = "1.23.1", features = [ - "v7", # Temporally orderable UUIDs with random component - "fast-rng", # Use a faster (but still sufficiently random) RNG - ]} -chrono = { version = "0.4.44"} -ux = "0.1.6" +uuid.workspace = true +chrono.workspace = true +ux.workspace = true # Concurrency: -tokio = {version = "1.52", features = ["macros", "rt-multi-thread"]} -futures = "0.3.32" +tokio.workspace = true +futures.workspace = true # Error handling: -anyhow = "1.0.102" -thiserror = "2.0.18" +anyhow.workspace = true +thiserror.workspace = true # Python FFI: -pyo3 = { version = "0.28.3", features = ["chrono"] } -pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime", "unstable-streams"] } -once_cell = "1.21.4" # Only currently used for `unsync::OnceCell` as part of PyO3 async helpers. +pyo3.workspace = true +pyo3-async-runtimes.workspace = true +once_cell.workspace = true # Only currently used for `unsync::OnceCell` as part of PyO3 async helpers. # Logging/tracing: -pyo3-log = "0.13.3" -tracing = { version = "0.1.41", features = ["log"] } +pyo3-log.workspace = true +tracing.workspace = true + +workspace-hack.workspace = true [lints] workspace = true diff --git a/libs/opsqueue_python/opsqueue_python.nix b/libs/opsqueue_python/opsqueue_python.nix index 57a3e89f..3645f38e 100644 --- a/libs/opsqueue_python/opsqueue_python.nix +++ b/libs/opsqueue_python/opsqueue_python.nix @@ -32,6 +32,7 @@ let sources = import ../../nix/sources.nix; crane = import sources.crane { pkgs = pkgs; }; craneLib = crane.overrideToolchain (pkgs: rustToolchain); + sqlitePkgConfigPath = lib.makeSearchPathOutput "dev" "lib/pkgconfig" [ pkgs.sqlite ]; # Only the files necessary to build the Rust-side and cache dependencies sqlFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; @@ -59,7 +60,11 @@ let inherit version pname; src = depsSrc; strictDeps = true; - nativeBuildInputs = [ python ]; + nativeBuildInputs = [ + python + pkgs.pkg-config + ]; + buildInputs = [ pkgs.sqlite ]; cargoExtraArgs = "--package opsqueue_python"; doCheck = false; }; @@ -77,6 +82,9 @@ let // { inherit cargoArtifacts; src = wheelSrc; + env.DATABASE_URL = "sqlite://${depsSrc}/opsqueue/opsqueue_example_database_schema.db"; + env.LIBSQLITE3_SYS_USE_PKG_CONFIG = "${pkgs.sqlite.dev}"; + env.PKG_CONFIG_PATH = sqlitePkgConfigPath; } )).overrideAttrs (old: { diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 59e3fbae..310eb645 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "opsqueue" version.workspace = true -edition = "2024" +edition.workspace = true description = "lightweight batch processing queue for heavy loads" repository = "https://github.com/channable/opsqueue" license = "MIT" @@ -17,65 +17,61 @@ required-features = ["server-logic"] [dependencies] # Datatypes and concurrency: -itertools = "0.15.0" -arc-swap = {version = "1.9.1", optional = true} -moka = { version = "0.12.15", features = ["sync"], optional = true } -chrono = { version = "0.4.44", features = ["serde"]} -futures = "0.3.32" -tokio = { version = "1.52.1", features = ["macros", "signal", "rt-multi-thread"] } -uuid = {version = "1.23.1", features = [ - "v7", # Timestamp-sortable UUIDs with random component - "fast-rng", # Use a faster (but still sufficiently random) RNG - "serde", - ]} -ux = "0.1.6" +itertools.workspace = true +arc-swap = { workspace = true, optional = true } +moka = { workspace = true, optional = true } +chrono.workspace = true +futures.workspace = true +tokio.workspace = true +uuid.workspace = true +ux.workspace = true # Error handling: -anyhow = "1.0.102" +anyhow.workspace = true # Database: -sqlx = { version = "0.9.0", features = ["sqlite", "runtime-tokio", "chrono"], optional = true } +sqlx = { workspace = true, optional = true } # Serialization: -serde = { version = "1.0.203", features = ["derive"] } -serde_json = "1.0.149" -ciborium = "0.2.2" +serde.workspace = true +serde_json.workspace = true +ciborium.workspace = true # Webservers/clients -http = "1.4.0" -object_store = {version = "0.14.0", features = ["gcp", "http"]} -snowflaked = {version = "1.0.3", features = ["sync"] } -tokio-tungstenite = {version = "0.30.0", optional = true} -axum = { version = "0.8.9", features = ["ws", "macros"], optional = true } -reqwest = { version = "0.13.2", default-features = false, features = ["json"], optional = true } -url = {version = "2.5.2"} -tokio-util = { version = "0.7.18", features = ["io", "rt", "time"] } -tower-http = { version = "0.7.0", features = ["trace", "catch-panic"], optional = true } +http.workspace = true +object_store.workspace = true +snowflaked.workspace = true +tokio-tungstenite = { workspace = true, optional = true } +axum = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } +url.workspace = true +tokio-util.workspace = true +tower-http = { workspace = true, optional = true } # Logging and tracing: -tracing = {version = "0.1", features = ["log"] } -tracing-subscriber = {version = "0.3", features = ["std", "env-filter"] } -sentry = {version = "0.48", optional = true, default-features=false, features=["rustls", "reqwest", "anyhow", "backtrace", "contexts", "tracing", "panic"]} -sentry-tracing = {version = "0.48", optional = true} +tracing.workspace = true +tracing-subscriber.workspace = true +sentry = { workspace = true, optional = true } +sentry-tracing = { workspace = true, optional = true } # Exporting traces to Opentelemetry: -opentelemetry = { version = "0.32", default-features = false, features = ["trace"] } -opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "rt-tokio"] } -opentelemetry-resource-detectors = "0.11.0" -opentelemetry-http = { version = "0.32" } -opentelemetry-otlp = { version = "0.32", optional = true, features = ["grpc-tonic"] } -tracing-opentelemetry = "0.33.0" -opentelemetry-semantic-conventions = {version = "0.32.0", features = ["semconv_experimental"], optional = true} -moro-local = "0.4.0" -thiserror = "2.0.18" -either = "1.13.0" -serde-error = "0.1.3" -backon = { version = "1.3.0", features = ["tokio-sleep"] } -rand = "0.10.1" -rustc-hash = "2.1.2" -axum-prometheus = {version = "0.10.0", optional = true} +opentelemetry.workspace = true +opentelemetry_sdk.workspace = true +opentelemetry-resource-detectors.workspace = true +opentelemetry-http.workspace = true +opentelemetry-otlp = { workspace = true, optional = true } +tracing-opentelemetry.workspace = true +opentelemetry-semantic-conventions = { workspace = true, optional = true } +moro-local.workspace = true +thiserror.workspace = true +either.workspace = true +backon.workspace = true +rand.workspace = true +rustc-hash.workspace = true +axum-prometheus = { workspace = true, optional = true } # Configuration: -clap = { version = "4.5.60", features = ["derive"] } -humantime = "2.1.0" +clap.workspace = true +humantime.workspace = true -dashmap = "6.1.0" -crossbeam-skiplist = "0.1.3" -sqlformat = "0.5.0" +dashmap.workspace = true +crossbeam-skiplist.workspace = true +sqlformat.workspace = true +workspace-hack.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -83,17 +79,7 @@ sqlformat = "0.5.0" workspace = true [dev-dependencies] -criterion = {version = "0.8", features = ["async_tokio"]} -insta = { version = "1.47.2" } -assert_matches = { version = "1.5.0" } - -# [[bench]] -# name = "chunks_select" -# harness = false - -# [[bench]] -# name = "submissions_insert" -# harness = false +insta.workspace = true [features] # Dependencies only in use by the server-logic: diff --git a/opsqueue/opsqueue.nix b/opsqueue/opsqueue.nix index 95e8ac77..c63c5610 100644 --- a/opsqueue/opsqueue.nix +++ b/opsqueue/opsqueue.nix @@ -16,6 +16,7 @@ let sources = import ../nix/sources.nix; crane = import sources.crane { pkgs = pkgs; }; craneLib = crane.overrideToolchain (pkgs: rustToolchain); + sqlitePkgConfigPath = lib.makeSearchPathOutput "dev" "lib/pkgconfig" [ pkgs.sqlite ]; # Only the files necessary to build the Rust-side and cache dependencies sqlFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; @@ -36,7 +37,8 @@ let commonArgs = { inherit src version pname; strictDeps = true; - nativeBuildInputs = [ ]; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.sqlite ]; cargoExtraArgs = "--package opsqueue"; doCheck = true; }; @@ -52,7 +54,9 @@ craneLib.buildPackage ( # Needed for the SQLx macros: env = { - DATABASE_URL = "sqlite:///build/opsqueue/opsqueue/opsqueue_example_database_schema.db"; + DATABASE_URL = "sqlite://${src}/opsqueue/opsqueue_example_database_schema.db"; + LIBSQLITE3_SYS_USE_PKG_CONFIG = "${pkgs.sqlite.dev}"; + PKG_CONFIG_PATH = sqlitePkgConfigPath; }; } diff --git a/opsqueue/src/consumer/strategy.rs b/opsqueue/src/consumer/strategy.rs index 17906e89..a725d138 100644 --- a/opsqueue/src/consumer/strategy.rs +++ b/opsqueue/src/consumer/strategy.rs @@ -309,13 +309,13 @@ pub mod test { 8, 2, CORRELATED SCALAR SUBQUERY 4 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 22, 8, LIST SUBQUERY 2 - 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 1, UNION ALL - 61, 58, SCAN chunks - 64, 58, CORRELATED SCALAR SUBQUERY 6 - 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 78, 64, LIST SUBQUERY 2 - 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: + 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 33, 22, CREATE BLOOM FILTER + 62, 1, UNION ALL + 65, 62, SCAN chunks + 68, 62, CORRELATED SCALAR SUBQUERY 6 + 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 80, 68, REUSE LIST SUBQUERY 2 "); } @@ -399,13 +399,13 @@ pub mod test { 8, 2, CORRELATED SCALAR SUBQUERY 4 12, 8, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 22, 8, LIST SUBQUERY 2 - 24, 22, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 1, UNION ALL - 61, 58, SCAN chunks - 64, 58, CORRELATED SCALAR SUBQUERY 6 - 68, 64, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 78, 64, LIST SUBQUERY 2 - 80, 78, SCAN json_each VIRTUAL TABLE INDEX 0: + 25, 22, SCAN json_each VIRTUAL TABLE INDEX 0: + 33, 22, CREATE BLOOM FILTER + 62, 1, UNION ALL + 65, 62, SCAN chunks + 68, 62, CORRELATED SCALAR SUBQUERY 6 + 72, 68, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 80, 68, REUSE LIST SUBQUERY 2 "); } @@ -498,27 +498,25 @@ pub mod test { 16, 4, CORRELATED SCALAR SUBQUERY 5 20, 16, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) 30, 16, LIST SUBQUERY 3 - 32, 30, SCAN json_each VIRTUAL TABLE INDEX 0: - 58, 3, UNION ALL - 61, 58, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 127, 115, CORRELATED SCALAR SUBQUERY 7 - 131, 127, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 141, 127, LIST SUBQUERY 3 - 143, 141, SCAN json_each VIRTUAL TABLE INDEX 0: - 169, 114, UNION ALL - 172, 169, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 121, 109, CORRELATED SCALAR SUBQUERY 7 + 125, 121, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 133, 121, REUSE LIST SUBQUERY 3 + 153, 108, UNION ALL + 156, 153, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 197, 185, CORRELATED SCALAR SUBQUERY 7 - 201, 197, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 211, 197, LIST SUBQUERY 3 - 213, 211, SCAN json_each VIRTUAL TABLE INDEX 0: - 231, 185, CORRELATED SCALAR SUBQUERY 11 - 235, 231, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 245, 231, LIST SUBQUERY 9 - 247, 245, SCAN json_each VIRTUAL TABLE INDEX 0: - 273, 184, UNION ALL - 276, 273, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 378, 366, CORRELATED SCALAR SUBQUERY 5 - 382, 378, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 392, 378, LIST SUBQUERY 3 - 394, 392, SCAN json_each VIRTUAL TABLE INDEX 0: - 412, 366, CORRELATED SCALAR SUBQUERY 13 - 416, 412, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 426, 412, LIST SUBQUERY 9 - 428, 426, SCAN json_each VIRTUAL TABLE INDEX 0: - 454, 365, UNION ALL - 457, 454, SEARCH chunks USING INDEX random_chunks_order (random_order?) - 557, 545, CORRELATED SCALAR SUBQUERY 7 - 561, 557, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 571, 557, LIST SUBQUERY 3 - 573, 571, SCAN json_each VIRTUAL TABLE INDEX 0: - 591, 545, CORRELATED SCALAR SUBQUERY 13 - 595, 591, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) - 605, 591, LIST SUBQUERY 9 - 607, 605, SCAN json_each VIRTUAL TABLE INDEX 0: - 633, 544, UNION ALL - 636, 633, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 213, 201, CORRELATED SCALAR SUBQUERY 7 + 217, 213, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 227, 213, LIST SUBQUERY 3 + 230, 227, SCAN json_each VIRTUAL TABLE INDEX 0: + 238, 227, CREATE BLOOM FILTER + 251, 201, CORRELATED SCALAR SUBQUERY 11 + 255, 251, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 265, 251, LIST SUBQUERY 9 + 268, 265, SCAN json_each VIRTUAL TABLE INDEX 0: + 276, 265, CREATE BLOOM FILTER + 297, 200, UNION ALL + 300, 297, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 410, 398, CORRELATED SCALAR SUBQUERY 5 + 414, 410, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 424, 410, LIST SUBQUERY 3 + 427, 424, SCAN json_each VIRTUAL TABLE INDEX 0: + 435, 424, CREATE BLOOM FILTER + 448, 398, CORRELATED SCALAR SUBQUERY 13 + 452, 448, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 462, 448, LIST SUBQUERY 9 + 465, 462, SCAN json_each VIRTUAL TABLE INDEX 0: + 473, 462, CREATE BLOOM FILTER + 494, 397, UNION ALL + 497, 494, SEARCH chunks USING INDEX random_chunks_order (random_order?) + 605, 593, CORRELATED SCALAR SUBQUERY 7 + 609, 605, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 619, 605, LIST SUBQUERY 3 + 622, 619, SCAN json_each VIRTUAL TABLE INDEX 0: + 630, 619, CREATE BLOOM FILTER + 643, 593, CORRELATED SCALAR SUBQUERY 13 + 647, 643, SEARCH submissions_metadata USING PRIMARY KEY (submission_id=? AND metadata_key=?) + 657, 643, LIST SUBQUERY 9 + 660, 657, SCAN json_each VIRTUAL TABLE INDEX 0: + 668, 657, CREATE BLOOM FILTER + 689, 592, UNION ALL + 692, 689, SEARCH chunks USING INDEX random_chunks_order (random_order