From db2f5b407d44ed8377ab6d0e3522533dd87c1002 Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Wed, 18 Nov 2026 10:08:01 +0100 Subject: [PATCH] Ship the corrected bundled SQLite in device components and pinned builds --- Cargo.lock | 173 ++++++++-------------------- Cargo.toml | 3 +- crates/agent-core/Cargo.toml | 2 +- crates/agent-windows/Cargo.toml | 2 +- crates/agent/Cargo.toml | 2 +- crates/persistence/Cargo.toml | 6 +- crates/persistence/src/lib.rs | 11 +- crates/privileged-helper/Cargo.toml | 2 +- crates/update-history/Cargo.toml | 2 +- crates/updater-worker/Cargo.toml | 2 +- docs/development.md | 2 +- tools/build.py | 38 +++++- 12 files changed, 100 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0de8b23..6d3bb0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,18 +37,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -234,9 +222,6 @@ name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -dependencies = [ - "serde_core", -] [[package]] name = "block-buffer" @@ -721,17 +706,6 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" -[[package]] -name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "spin", -] - [[package]] name = "fnv" version = "1.0.7" @@ -744,6 +718,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -967,15 +947,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - [[package]] name = "hashbrown" version = "0.15.5" @@ -984,7 +955,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -992,14 +963,8 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ - "hashbrown 0.14.5", + "foldhash 0.2.0", ] [[package]] @@ -1011,6 +976,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -1418,9 +1392,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.30.1" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "14a5f01b87b92705d07b433c06f96cbc764637d81a0bffb5734ce17cf0f9a2b3" dependencies = [ "cc", "pkg-config", @@ -2315,7 +2289,9 @@ name = "rmm-persistence" version = "0.1.0" dependencies = [ "anyhow", - "sqlx", + "sqlx-core", + "sqlx-macros", + "sqlx-postgres", "uuid", ] @@ -2538,17 +2514,27 @@ dependencies = [ ] [[package]] -name = "rusqlite" -version = "0.32.1" +name = "rsqlite-vfs" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +checksum = "353b74f9810d5b085b6c42c9d86a5f0fdb795553235cdd48753b32e3809d18a7" +dependencies = [ + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9bba6f3506135e120a6586df05be18b48d18896600646a4bec8f993d91a594" dependencies = [ "bitflags", "fallible-iterator", "fallible-streaming-iterator", - "hashlink 0.9.1", + "hashlink 0.12.2", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -2871,9 +2857,6 @@ name = "spin" version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" -dependencies = [ - "lock_api", -] [[package]] name = "spki" @@ -2885,6 +2868,16 @@ dependencies = [ "der", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cd504cecbfe7eef649988840f9a6cb768d11085efec138f8e99f160632d867" +dependencies = [ + "cc", + "rsqlite-vfs", +] + [[package]] name = "sqlx" version = "0.8.6" @@ -2893,9 +2886,7 @@ checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" dependencies = [ "sqlx-core", "sqlx-macros", - "sqlx-mysql", "sqlx-postgres", - "sqlx-sqlite", ] [[package]] @@ -2966,58 +2957,12 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sqlx-core", - "sqlx-mysql", "sqlx-postgres", - "sqlx-sqlite", "syn 2.0.119", "tokio", "url", ] -[[package]] -name = "sqlx-mysql" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" -dependencies = [ - "atoi", - "base64 0.22.1", - "bitflags", - "byteorder", - "bytes", - "chrono", - "crc", - "digest 0.10.7", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand 0.8.8", - "rsa", - "serde", - "sha1", - "sha2 0.10.9", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror", - "tracing", - "uuid", - "whoami", -] - [[package]] name = "sqlx-postgres" version = "0.8.6" @@ -3057,32 +3002,6 @@ dependencies = [ "whoami", ] -[[package]] -name = "sqlx-sqlite" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" -dependencies = [ - "atoi", - "chrono", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "serde_urlencoded", - "sqlx-core", - "thiserror", - "tracing", - "url", - "uuid", -] - [[package]] name = "sse-stream" version = "0.2.6" diff --git a/Cargo.toml b/Cargo.toml index 7d4ed38..118be08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,13 +18,14 @@ openssl = "0.10.75" prost = "0.14.1" rand = "0.9.2" reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls", "json"] } +rusqlite = { version = "0.41.1", features = ["bundled", "fallible_uint"] } rustls = { version = "0.23.35", default-features = false, features = ["ring", "std", "tls12"] } rustls-pemfile = "2.2.0" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" sha2 = "0.10.9" schemars = { version = "1", features = ["chrono04", "uuid1"] } -sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio-rustls", "postgres", "uuid", "chrono", "json", "migrate", "macros"] } +sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio-rustls", "postgres"] } tokio = { version = "1.48.0", features = ["full"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["ring", "tls12"] } tokio-tungstenite = { version = "0.28.0", features = ["rustls-tls-webpki-roots"] } diff --git a/crates/agent-core/Cargo.toml b/crates/agent-core/Cargo.toml index 1ce161f..13f119f 100644 --- a/crates/agent-core/Cargo.toml +++ b/crates/agent-core/Cargo.toml @@ -19,7 +19,7 @@ uuid.workspace = true rmm-domain = { path = "../domain" } rmm-protocol = { path = "../protocol" } rmm-updater-handoff = { path = "../updater-handoff" } -rusqlite = { version = "0.32.1", features = ["bundled"] } +rusqlite.workspace = true ed25519-dalek = "=2.2.0" [dev-dependencies] diff --git a/crates/agent-windows/Cargo.toml b/crates/agent-windows/Cargo.toml index 5860a4a..de52267 100644 --- a/crates/agent-windows/Cargo.toml +++ b/crates/agent-windows/Cargo.toml @@ -43,7 +43,7 @@ rmm-updater = { path = "../updater" } hex.workspace = true ed25519-dalek = "=2.2.0" rmm-update-history = { path = "../update-history" } -rusqlite = { version = "0.32.1", features = ["bundled"] } +rusqlite.workspace = true fs2 = "0.4.3" x509-parser = { version = "=0.18.1", features = ["verify"] } rcgen = { version = "=0.14.10", default-features = false, features = ["crypto", "ring", "pem"] } diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index 3c892e2..5137797 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -21,7 +21,7 @@ sha2.workspace = true tokio.workspace = true tokio-tungstenite.workspace = true uuid.workspace = true -rusqlite = { version = "0.32.1", features = ["bundled"] } +rusqlite.workspace = true fs2 = "0.4.3" rmm-domain = { path = "../domain" } rmm-protocol = { path = "../protocol" } diff --git a/crates/persistence/Cargo.toml b/crates/persistence/Cargo.toml index 8b06e42..bc356d4 100644 --- a/crates/persistence/Cargo.toml +++ b/crates/persistence/Cargo.toml @@ -5,5 +5,9 @@ edition.workspace = true license.workspace = true [dependencies] anyhow.workspace = true -sqlx.workspace = true +# Use the PostgreSQL driver directly. SQLx 0.8's generic migration and type +# features also resolve the obsolete SQLite driver in the shared lockfile. +sqlx = { package = "sqlx-core", version = "=0.8.6", features = ["migrate", "_rt-tokio", "_tls-rustls-ring-webpki"] } +sqlx-macros = { version = "=0.8.6", features = ["migrate"] } +sqlx-postgres = { version = "=0.8.6", features = ["migrate", "chrono", "uuid", "json"] } uuid.workspace = true diff --git a/crates/persistence/src/lib.rs b/crates/persistence/src/lib.rs index 712c383..533db85 100644 --- a/crates/persistence/src/lib.rs +++ b/crates/persistence/src/lib.rs @@ -1,17 +1,18 @@ use anyhow::{Result, ensure}; -use sqlx::{PgPool, Postgres, Transaction}; +use sqlx::{query::query, query_scalar::query_scalar}; +use sqlx_postgres::{PgPool, PgTransaction}; use uuid::Uuid; -pub async fn tenant_tx(pool: &PgPool, tenant: Uuid) -> Result> { +pub async fn tenant_tx(pool: &PgPool, tenant: Uuid) -> Result> { let mut tx = pool.begin().await?; - sqlx::query("SELECT set_config('app.tenant_id', $1, true)") + query("SELECT set_config('app.tenant_id', $1, true)") .bind(tenant.to_string()) .execute(&mut *tx) .await?; Ok(tx) } pub async fn verify_application_role(pool: &PgPool) -> Result<()> { - let unsafe_role: bool = sqlx::query_scalar( + let unsafe_role: bool = query_scalar( "SELECT rolsuper OR rolbypassrls OR EXISTS (SELECT 1 FROM pg_tables WHERE schemaname='public' AND tableowner=current_user) FROM pg_roles WHERE rolname=current_user", ).fetch_one(pool).await?; ensure!( @@ -22,7 +23,7 @@ pub async fn verify_application_role(pool: &PgPool) -> Result<()> { } pub async fn migrate(url: &str) -> Result<()> { let pool = PgPool::connect(url).await?; - sqlx::migrate!("../../migrations").run(&pool).await?; + sqlx_macros::migrate!("../../migrations").run(&pool).await?; pool.close().await; Ok(()) } diff --git a/crates/privileged-helper/Cargo.toml b/crates/privileged-helper/Cargo.toml index 2640d06..d443c35 100644 --- a/crates/privileged-helper/Cargo.toml +++ b/crates/privileged-helper/Cargo.toml @@ -15,7 +15,7 @@ serde_json.workspace = true uuid.workspace = true libc = "0.2.177" fs2 = "0.4.3" -rusqlite = { version = "0.32.1", features = ["bundled"] } +rusqlite.workspace = true rmm-domain = { path = "../domain" } rmm-agent-linux = { path = "../agent-linux" } diff --git a/crates/update-history/Cargo.toml b/crates/update-history/Cargo.toml index a9db270..a7c9a99 100644 --- a/crates/update-history/Cargo.toml +++ b/crates/update-history/Cargo.toml @@ -11,7 +11,7 @@ hex.workspace = true serde_json.workspace = true uuid.workspace = true ed25519-dalek = "=2.2.0" -rusqlite = { version = "0.32.1", features = ["bundled"] } +rusqlite.workspace = true rmm-domain = { path = "../domain" } rmm-agent-core = { path = "../agent-core" } rmm-updater = { path = "../updater" } diff --git a/crates/updater-worker/Cargo.toml b/crates/updater-worker/Cargo.toml index 356ab9e..1912b52 100644 --- a/crates/updater-worker/Cargo.toml +++ b/crates/updater-worker/Cargo.toml @@ -16,7 +16,7 @@ serde.workspace = true serde_json.workspace = true uuid.workspace = true libc = "0.2.177" -rusqlite = { version = "0.32.1", features = ["bundled"] } +rusqlite.workspace = true ed25519-dalek = "=2.2.0" base64 = "=0.22.1" rmm-domain = { path = "../domain" } diff --git a/docs/development.md b/docs/development.md index f96e380..1057eb7 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,7 +4,7 @@ Der [Lieferplan](mvp-completion-plan.md) nennt den nächsten Schritt. Für eine ## Buildziel wählen -Die Befehle werden im Repository ausgeführt. Der [Buildrunner](../tools/build.py) verwendet den gewählten Git-Commit, die gepinnte Rust-Toolchain und die vorhandenen Docker-Builder. `--plan` zeigt Revision, Zielbinaries und Kommando an. `--ref` akzeptiert einen Commit oder Branch und verwendet standardmäßig `HEAD`. Bei absichtlich geänderten Quellen einen eigenen abgegrenzten Git-Stand wählen, damit fremder WIP im Arbeitsbaum bleiben kann. Der Cargo-Cache wird je Ziel wiederverwendet. +Die Befehle werden im Repository ausgeführt. Der [Buildrunner](../tools/build.py) verwendet den gewählten Git-Commit, die gepinnte Rust-Toolchain und die vorhandenen Docker-Builder. `--plan` zeigt Revision, Zielbinaries und Kommando an. `--ref` akzeptiert einen Commit oder Branch und verwendet standardmäßig `HEAD`. Bei absichtlich geänderten Quellen einen eigenen abgegrenzten Git-Stand wählen, damit fremder WIP im Arbeitsbaum bleiben kann. Der Cargo-Cache wird je Ziel wiederverwendet. Fehlen dem gepinnten Builder Pakete aus dem gewählten Lockfile, lädt der Runner sie einmalig mit `cargo fetch --locked` in den zielbezogenen Cache. Kompilierung und Buildskripte laufen weiterhin ohne Netzwerk. | Ziel | Befehl | Ergebnis | | --- | --- | --- | diff --git a/tools/build.py b/tools/build.py index eeb9c64..505ecce 100644 --- a/tools/build.py +++ b/tools/build.py @@ -61,15 +61,41 @@ def prepare(target, jobs, windows_updater=False): return platform, triple, image, binaries, cargo +def crate_registry(work, docker, image, triple): + # Old revisions can build entirely from the registry embedded in the image. + # New lockfiles get a persistent package cache. Only Cargo fetch needs the + # network; build scripts and the compiler always run without it. + registry = work / "cargo-registry" + mount = ["-v", f"{registry}:/packages", "-e", "CARGO_HOME=/packages"] + fetch = ["--entrypoint", "cargo", image, "fetch", "--locked", "--target", triple] + offline = [*docker, "--network=none"] + cache = mount if registry.exists() else [] + probe = subprocess.run([*offline, *cache, *fetch, "--offline"], capture_output=True) + if probe.returncode == 0: + return cache + + if not registry.exists(): + with tempfile.TemporaryDirectory(dir=work) as temporary: + staging = Path(temporary) / "packages" + staging.mkdir() + subprocess.run([ + *offline, "-v", f"{staging}:/packages", "--entrypoint", "sh", image, "-c", + "mkdir /packages/registry && cp -R /usr/local/cargo/registry/cache " + "/usr/local/cargo/registry/index /packages/registry/", + ], check=True) + staging.rename(registry) + print("Fetching locked Cargo dependencies for " + triple, flush=True) + subprocess.run([*docker, *mount, *fetch], check=True) + return mount + + def build_archive(work, archive, revision, plan): platform, triple, image, binaries, cargo = plan source, cache = work / "src", work / "cargo" - command = ["docker", "run", "--rm", "--pull=never", "--platform", platform, "--network=none", + docker = ["docker", "run", "--rm", "--pull=never", "--platform", platform, "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--tmpfs", "/tmp:rw,exec", "--user", f"{os.getuid()}:{os.getgid()}", - "-v", f"{source}:/src:ro", "-v", f"{cache}:/target", "-w", "/src", - "-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]] - print(shlex.join(command), flush=True) + "-v", f"{source}:/src:ro", "-w", "/src"] os.umask(0o077) subprocess.run(["docker", "image", "inspect", image], check=True, stdout=subprocess.DEVNULL) work.mkdir(parents=True, exist_ok=True) @@ -85,6 +111,10 @@ def build_archive(work, archive, revision, plan): # Preserve unchanged inputs for Cargo. Changed inputs receive a fresh mtime # even when switching back to an older commit with an older Git timestamp. subprocess.run(["rsync", "-rlcp", "--delete", str(unpacked) + "/", source], check=True) + registry = crate_registry(work, docker, image, triple) + command = [*docker, "--network=none", *registry, "-v", f"{cache}:/target", + "-e", "CARGO_TARGET_DIR=/target", "--entrypoint", cargo[0], image, *cargo[1:]] + print(shlex.join(command), flush=True) subprocess.run(command, check=True) artifacts = staging / "artifacts" artifacts.mkdir()