diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index ceae344..fe7930c 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -1,31 +1,27 @@ -name: Native build (cross-platform spark_frost) +name: Native build (spark_frost from pinned Lightspark source) -# Builds libspark_frost / spark_frost.dll for every supported RID directly -# on the native CI runners (macos-latest, ubuntu-latest, windows-latest) -# and emits a consolidated runtimes/ tree + SHA256SUMS.txt as a single -# artifact. Run manually before a release to refresh the binaries -# committed under src/NSpark/runtimes/. +# Rebuilds libspark_frost from buildonspark/spark at a PINNED commit, on native runners, and +# checks it against the committed C# bindings. The output is the runtimes/ tree to commit under +# src/NSpark/runtimes/, plus SHA256SUMS.txt and build-provenance attestations. # -# Code-signing (Authenticode / macOS notarization) is intentionally NOT -# performed here for v0.x; the binaries ship unsigned with SHA-256 -# integrity check via runtimes/SHA256SUMS.txt and SLSA provenance on the -# overall NuGet package. Add signing in a separate workflow once OrkLabs -# acquires the certs. +# - Linux RIDs build inside Amazon Linux 2023 (glibc 2.34) so the library loads on the AWS +# Lambda dotnet runtime and on every newer distro. +# - The build logic lives in scripts/build-native.sh so it can be reproduced locally. +# - Bump SPARK_SHA (and the protos re-vendored from the same commit) in one PR. +# +# Supported RIDs: linux-x64, linux-arm64, osx-arm64. Other RIDs are not shipped. on: workflow_dispatch: - inputs: - spark_ref: - description: buildonspark/spark git ref to build (branch, tag, or SHA) - required: false - default: main permissions: contents: read env: SPARK_REPO: buildonspark/spark - SPARK_REF: ${{ github.event.inputs.spark_ref || 'main' }} + SPARK_SHA: 0b3a32a05c9ac06cc411683551dd1f1bde9d0caa + AL2023_IMAGE: amazonlinux:2023@sha256:5b29412077a463b4a3a8fbc99a8cdf4b929f38a3ecc8dac10328d8f36b0099b8 + UNIFFI_BINDGEN_CS_TAG: v0.9.2+v0.28.3 jobs: build: @@ -35,71 +31,75 @@ jobs: fail-fast: false matrix: include: - - { rid: osx-arm64, os: macos-latest, target: aarch64-apple-darwin, artifact: libspark_frost.dylib } - - { rid: osx-x64, os: macos-13, target: x86_64-apple-darwin, artifact: libspark_frost.dylib } - - { rid: linux-x64, os: ubuntu-latest, target: x86_64-unknown-linux-gnu, artifact: libspark_frost.so } - - { rid: linux-arm64, os: ubuntu-24.04-arm,target: aarch64-unknown-linux-gnu, artifact: libspark_frost.so } - - { rid: win-x64, os: windows-latest, target: x86_64-pc-windows-msvc, artifact: spark_frost.dll } - - { rid: win-arm64, os: windows-latest, target: aarch64-pc-windows-msvc, artifact: spark_frost.dll } + - { rid: linux-x64, os: ubuntu-24.04 } + - { rid: linux-arm64, os: ubuntu-24.04-arm } + - { rid: osx-arm64, os: macos-14 } steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@v4 + with: + path: nspark + + - uses: actions/checkout@v4 with: repository: ${{ env.SPARK_REPO }} - ref: ${{ env.SPARK_REF }} + ref: ${{ env.SPARK_SHA }} path: spark - - name: Setup Rust toolchain (pinned by spark/signer/rust-toolchain.toml) + - name: Build (Amazon Linux 2023 container) + if: runner.os == 'Linux' run: | - rustup show - rustup target add ${{ matrix.target }} - working-directory: spark/signer - shell: bash + docker run --rm \ + -v "$PWD/spark:/spark:ro" -v "$PWD/nspark/scripts:/scripts:ro" -v "$PWD/out:/out" \ + "$AL2023_IMAGE" bash /scripts/build-native.sh ${{ matrix.rid }} /spark /out - - name: Build - run: cargo build --release -p spark-frost-uniffi --target ${{ matrix.target }} - working-directory: spark/signer + - name: Build (native macOS) + if: runner.os == 'macOS' + run: nspark/scripts/build-native.sh ${{ matrix.rid }} spark out - - name: Strip (Linux) - if: runner.os == 'Linux' - run: strip --strip-unneeded spark/signer/target/${{ matrix.target }}/release/${{ matrix.artifact }} - - - name: Hash artifact - shell: bash + - name: Bindings match the committed SparkFrostBindings.cs + if: matrix.rid == 'linux-x64' run: | - mkdir -p out/${{ matrix.rid }}/native - cp "spark/signer/target/${{ matrix.target }}/release/${{ matrix.artifact }}" \ - "out/${{ matrix.rid }}/native/${{ matrix.artifact }}" - ( cd out && shasum -a 256 "${{ matrix.rid }}/native/${{ matrix.artifact }}" ) | tee out/${{ matrix.rid }}.sha256 + cargo install uniffi-bindgen-cs --git https://github.com/NordSecurity/uniffi-bindgen-cs \ + --tag "$UNIFFI_BINDGEN_CS_TAG" --locked --root "$RUNNER_TEMP/bindgen" + "$RUNNER_TEMP/bindgen/bin/uniffi-bindgen-cs" spark/signer/spark-frost-uniffi/src/spark_frost.udl \ + --lib-file out/runtimes/linux-x64/native/libspark_frost.so --out-dir "$RUNNER_TEMP/gen" + diff -u nspark/src/NSpark/Signer/SparkFrostBindings.cs "$RUNNER_TEMP/gen/spark_frost.cs" - - name: Upload artifact - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@v4 with: name: native-${{ matrix.rid }} - path: out/ + path: out/runtimes/ if-no-files-found: error consolidate: - name: Consolidate + SHA256SUMS.txt + name: SHA256SUMS + provenance runs-on: ubuntu-latest needs: build + permissions: + contents: read + id-token: write + attestations: write steps: - - uses: actions/download-artifact@v8 + - uses: actions/download-artifact@v4 with: pattern: native-* - path: runtimes-staging + path: runtimes merge-multiple: true - - name: Generate consolidated SHA256SUMS.txt + - name: SHA256SUMS.txt run: | - cd runtimes-staging - find . -type f \( -name "*.dll" -o -name "*.so" -o -name "*.dylib" \) \ - | sort | xargs shasum -a 256 \ - | sed 's| \./| |' > SHA256SUMS.txt + cd runtimes + find . -type f \( -name "*.so" -o -name "*.dylib" \) | sort | xargs sha256sum | sed 's| \./| |' > SHA256SUMS.txt cat SHA256SUMS.txt - - name: Upload consolidated runtimes/ tree - uses: actions/upload-artifact@v7 + - uses: actions/attest-build-provenance@v2 + with: + subject-path: | + runtimes/**/*.so + runtimes/**/*.dylib + + - uses: actions/upload-artifact@v4 with: name: runtimes - path: runtimes-staging/ + path: runtimes/ if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c00624..b9ba9b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,8 @@ env: jobs: release: name: Build, attest, publish + # Publishing to nuget.org is upstream's job; forks (e.g. Elenpay/nspark) never publish. + if: github.repository == 'orklabs/nspark' runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 diff --git a/docs/native-build.md b/docs/native-build.md index cfcaf27..cd2d06a 100644 --- a/docs/native-build.md +++ b/docs/native-build.md @@ -14,31 +14,53 @@ yourself or add support for a new RID. ## Supported runtime identifiers -| RID | Status today | Target for v1 | +This fork ships three RIDs, all rebuilt from Lightspark's source at a pinned +commit by `scripts/build-native.sh` (locally) and +`.github/workflows/native-build.yml` (CI): + +| RID | Built on | Notes | |---|---|---| -| `osx-arm64` | ✅ committed (native build on Apple Silicon) | unsigned for v0.x, notarized once Apple Developer cert is acquired | -| `osx-x64` | ✅ committed (cross-compiled from arm64 via `rustup target add x86_64-apple-darwin`) | unsigned for v0.x, notarized once Apple Developer cert is acquired | -| `linux-x64` | ✅ committed | stripped | -| `linux-arm64` | ✅ committed (cross-compiled via `cargo-zigbuild`) | stripped | -| `win-x64` | ✅ committed (cross-compiled via `cargo-xwin`) | unsigned for v0.x, Authenticode-signed once cert is acquired | -| `win-arm64` | ✅ committed (cross-compiled via `cargo-xwin`) | unsigned for v0.x, Authenticode-signed once cert is acquired | - -SHA-256 hashes of every shipped binary live in -`src/NSpark/runtimes/SHA256SUMS.txt` and are also packed into the NuGet at -the same path. Generated by `find … | shasum -a 256` against the binaries -themselves; verify with: +| `linux-x64` | Amazon Linux 2023 container | glibc ≤ 2.34 (AWS Lambda floor), stripped | +| `linux-arm64` | Amazon Linux 2023 container | glibc ≤ 2.34, stripped | +| `osx-arm64` | macOS, `MACOSX_DEPLOYMENT_TARGET=11.0` | stripped, ad-hoc signed | + +`osx-x64`, `win-x64` and `win-arm64` are not shipped: their upstream binaries +were built outside CI from a different Spark revision, and loading them with +the current bindings is not verified. + +Every build pins `SPARK_SHA` (the same commit the protos are vendored from), +uses `cargo build --locked`, a pinned `protoc`, and the Rust toolchain from +`signer/rust-toolchain.toml`, and checks: + +- the 34 `uniffi_spark_frost_checksum_func_*` exports the bindings verify at load; +- on Linux, that no symbol needs a glibc newer than 2.34; +- that `uniffi-bindgen-cs` regenerates `SparkFrostBindings.cs` byte-for-byte. + +The CI workflow also writes `SHA256SUMS.txt` and attaches build-provenance +attestations to each library. Verify a committed tree with: ```bash ( cd src/NSpark/runtimes && shasum -a 256 -c SHA256SUMS.txt ) ``` -The release CI workflow `.github/workflows/native-build.yml` rebuilds and -signs every binary on the matching native runner whenever the appropriate -secrets are configured. +### Rebuilding locally + +```bash +SPARK_SRC=~/spark # buildonspark/spark checked out at SPARK_SHA (see native-build.yml) +# Per-platform digests of the pinned amazonlinux:2023 index (Docker can't pull a +# second platform of an index digest it already holds). +AL2023_ARM64=amazonlinux@sha256:442a915d07801a5805b765f4f676f455a0007af70628aa34cd73a735db5740b3 +AL2023_AMD64=amazonlinux@sha256:2851878b108218cccbe7af6ab7dfb87a5883a725623384e19e6256f4072027a3 +docker run --rm --platform linux/arm64 -v "$SPARK_SRC:/spark:ro" -v "$PWD/scripts:/scripts:ro" \ + -v "$PWD/out:/out" "$AL2023_ARM64" bash /scripts/build-native.sh linux-arm64 /spark /out +docker run --rm --platform linux/amd64 -v "$SPARK_SRC:/spark:ro" -v "$PWD/scripts:/scripts:ro" \ + -v "$PWD/out:/out" "$AL2023_AMD64" bash /scripts/build-native.sh linux-x64 /spark /out +scripts/build-native.sh osx-arm64 "$SPARK_SRC" out +``` -The `NSpark.csproj` `` rules pack -whatever binaries are present. Missing RIDs do not break the local build; -they break consumers on that platform. +Copy +`out/runtimes//native/*` into `src/NSpark/runtimes/` and regenerate +`SHA256SUMS.txt`. The sections below are upstream's original manual notes. ## Prerequisites diff --git a/scripts/build-native.sh b/scripts/build-native.sh new file mode 100755 index 0000000..9fece8a --- /dev/null +++ b/scripts/build-native.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Builds libspark_frost (buildonspark/spark signer/spark-frost-uniffi) for one RID, reproducibly, +# and checks it against the committed C# bindings. Used by .github/workflows/native-build.yml and +# runnable locally: +# +# Linux RIDs build inside Amazon Linux 2023 (glibc 2.34, the AWS Lambda floor): +# docker run --rm --platform linux/arm64 -v "$SPARK_SRC:/spark:ro" -v "$PWD:/nspark" \ +# "$AL2023_IMAGE" bash /nspark/scripts/build-native.sh linux-arm64 /spark /nspark/out +# macOS arm64 builds natively: +# scripts/build-native.sh osx-arm64 "$SPARK_SRC" out +# +# The Spark checkout must be at SPARK_SHA (see native-build.yml). The script never touches it: +# the cargo target dir lives under the output dir. +set -euo pipefail + +RID="${1:?usage: build-native.sh }" +SRC="$(cd "${2:?spark source dir}" && pwd)" +OUT="${3:?output dir}" +mkdir -p "$OUT" && OUT="$(cd "$OUT" && pwd)" + +PROTOC_VERSION=29.3 +GLIBC_MAX=2.34 +EXPECTED_CHECKSUMS=34 + +case "$RID" in + linux-x64) TARGET=x86_64-unknown-linux-gnu; LIB=libspark_frost.so; PROTOC_ARCH=linux-x86_64; PROTOC_SHA=3e866620c5be27664f3d2fa2d656b5f3e09b5152b42f1bedbf427b333e90021a ;; + linux-arm64) TARGET=aarch64-unknown-linux-gnu; LIB=libspark_frost.so; PROTOC_ARCH=linux-aarch_64; PROTOC_SHA=6427349140e01f06e049e707a58709a4f221ae73ab9a0425bc4a00c8d0e1ab32 ;; + osx-arm64) TARGET=aarch64-apple-darwin; LIB=libspark_frost.dylib; PROTOC_ARCH=osx-aarch_64; PROTOC_SHA=2b8a3403cd097f95f3ba656e14b76c732b6b26d7f183330b11e36ef2bc028765 ;; + *) echo "unsupported RID: $RID" >&2; exit 2 ;; +esac + +TOOLS="$OUT/.tools/$RID" +mkdir -p "$TOOLS" + +if [[ "$RID" == linux-* ]] && command -v dnf > /dev/null; then + # Amazon Linux 2023 build container: compiler, binutils and the tools below. + dnf install -y -q gcc make tar gzip unzip findutils binutils git > /dev/null +fi + +# Pinned protoc (spark-frost's build.rs runs prost-build). +if [ ! -x "$TOOLS/protoc/bin/protoc" ]; then + curl -fsSL -o "$TOOLS/protoc.zip" \ + "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" + echo "$PROTOC_SHA $TOOLS/protoc.zip" | (command -v sha256sum > /dev/null && sha256sum -c - || shasum -a 256 -c -) + rm -rf "$TOOLS/protoc" && mkdir -p "$TOOLS/protoc" && unzip -q "$TOOLS/protoc.zip" -d "$TOOLS/protoc" +fi +export PROTOC="$TOOLS/protoc/bin/protoc" + +# Rust toolchain pinned by spark/signer/rust-toolchain.toml. +if ! command -v rustup > /dev/null; then + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none > /dev/null + # shellcheck disable=SC1091 + source "${CARGO_HOME:-$HOME/.cargo}/env" +fi + +cd "$SRC/signer" +rustup toolchain install > /dev/null +rustup target add "$TARGET" > /dev/null + +# Native host builds: override signer/.cargo/config.toml's cross linkers. +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=gcc +export CARGO_TARGET_DIR="$OUT/.target/$RID" +export RUSTFLAGS="--remap-path-prefix=$SRC=/spark --remap-path-prefix=${CARGO_HOME:-$HOME/.cargo}=/cargo" +[[ "$RID" == osx-* ]] && export MACOSX_DEPLOYMENT_TARGET=11.0 + +cargo build --locked --release -p spark-frost-uniffi --target "$TARGET" + +DEST="$OUT/runtimes/$RID/native" +mkdir -p "$DEST" +cp "$CARGO_TARGET_DIR/$TARGET/release/$LIB" "$DEST/$LIB" + +if [[ "$RID" == linux-* ]]; then + strip --strip-unneeded "$DEST/$LIB" + glibc="$(objdump -T "$DEST/$LIB" | grep -oE 'GLIBC_[0-9.]+' | sed 's/GLIBC_//' | sort -Vu | tail -1)" + if [ "$(printf '%s\n%s\n' "$glibc" "$GLIBC_MAX" | sort -V | tail -1)" != "$GLIBC_MAX" ]; then + echo "$LIB needs GLIBC_$glibc, above the $GLIBC_MAX floor" >&2; exit 1 + fi + echo "glibc floor: $glibc (max allowed $GLIBC_MAX)" + checksums="$(nm -D --defined-only "$DEST/$LIB" | grep -c uniffi_spark_frost_checksum_func_)" +else + strip -x "$DEST/$LIB" + codesign -f -s - "$DEST/$LIB" + checksums="$(nm -gU "$DEST/$LIB" | grep -c uniffi_spark_frost_checksum_func_)" +fi + +if [ "$checksums" -ne "$EXPECTED_CHECKSUMS" ]; then + echo "expected $EXPECTED_CHECKSUMS uniffi checksum functions, found $checksums" >&2; exit 1 +fi +echo "uniffi checksum functions: $checksums" +(cd "$OUT" && (command -v sha256sum > /dev/null && sha256sum "runtimes/$RID/native/$LIB" || shasum -a 256 "runtimes/$RID/native/$LIB")) diff --git a/src/NSpark/NSpark.csproj b/src/NSpark/NSpark.csproj index fdb117e..2f3f6ef 100644 --- a/src/NSpark/NSpark.csproj +++ b/src/NSpark/NSpark.csproj @@ -81,26 +81,11 @@ CopyToOutputDirectory="PreserveNewest" Link="runtimes/linux-x64/native/libspark_frost.so" Condition="Exists('runtimes/linux-x64/native/libspark_frost.so')" /> - - - diff --git a/src/NSpark/runtimes/SHA256SUMS.txt b/src/NSpark/runtimes/SHA256SUMS.txt index e815fac..8fff955 100644 --- a/src/NSpark/runtimes/SHA256SUMS.txt +++ b/src/NSpark/runtimes/SHA256SUMS.txt @@ -1,6 +1,3 @@ -7e5aed606a8aee2fb41a69a8ac57011747c03282262413d555ff933ddcef197c linux-arm64/native/libspark_frost.so -62fff70176a36a972ede575bfcc3dc5e2951c5d24fbbcce08179a014e1447496 linux-x64/native/libspark_frost.so -0e935fa9c2de1f28b7b77681f8860ea081eed0252d986b058295cb59c7a67abd osx-arm64/native/libspark_frost.dylib -82a1e33519ca3dcdb5e5350bfb6c5dd559c45d207d88f4a9308c8a3b9b6dcc04 osx-x64/native/libspark_frost.dylib -491c0c5c328a45cad06a9b3da37db0c3303644c56f4edaea57d16d078b1f5e5d win-arm64/native/spark_frost.dll -a044adacf57a52f45dd5a64a044db2b2d1255bdfa80368943490cf9c3b70fd43 win-x64/native/spark_frost.dll +7afe8a4c273db9957684d30ec7cb2dd414f7019669430ac417e8332b5acec808 linux-arm64/native/libspark_frost.so +9031ecc6d769d739bb3899f8ed00a79c943609dd9524b73db639d074f5eea264 linux-x64/native/libspark_frost.so +aaee55f4d3b492420b9a4d41feb5c1f06f5d68a4dc41865370b52c0d76761bdf osx-arm64/native/libspark_frost.dylib diff --git a/src/NSpark/runtimes/linux-arm64/native/libspark_frost.so b/src/NSpark/runtimes/linux-arm64/native/libspark_frost.so index 9dbef9f..297c8db 100755 Binary files a/src/NSpark/runtimes/linux-arm64/native/libspark_frost.so and b/src/NSpark/runtimes/linux-arm64/native/libspark_frost.so differ diff --git a/src/NSpark/runtimes/linux-x64/native/libspark_frost.so b/src/NSpark/runtimes/linux-x64/native/libspark_frost.so index f893d42..3942bd6 100755 Binary files a/src/NSpark/runtimes/linux-x64/native/libspark_frost.so and b/src/NSpark/runtimes/linux-x64/native/libspark_frost.so differ diff --git a/src/NSpark/runtimes/osx-arm64/native/libspark_frost.dylib b/src/NSpark/runtimes/osx-arm64/native/libspark_frost.dylib index 838946e..9cee43c 100755 Binary files a/src/NSpark/runtimes/osx-arm64/native/libspark_frost.dylib and b/src/NSpark/runtimes/osx-arm64/native/libspark_frost.dylib differ diff --git a/src/NSpark/runtimes/osx-x64/native/libspark_frost.dylib b/src/NSpark/runtimes/osx-x64/native/libspark_frost.dylib deleted file mode 100755 index 2695b91..0000000 Binary files a/src/NSpark/runtimes/osx-x64/native/libspark_frost.dylib and /dev/null differ diff --git a/src/NSpark/runtimes/win-arm64/native/spark_frost.dll b/src/NSpark/runtimes/win-arm64/native/spark_frost.dll deleted file mode 100755 index b9ffaf5..0000000 Binary files a/src/NSpark/runtimes/win-arm64/native/spark_frost.dll and /dev/null differ diff --git a/src/NSpark/runtimes/win-x64/native/spark_frost.dll b/src/NSpark/runtimes/win-x64/native/spark_frost.dll deleted file mode 100755 index c1f155e..0000000 Binary files a/src/NSpark/runtimes/win-x64/native/spark_frost.dll and /dev/null differ