From 5bae14c11c095f9eb08a0f744c78d1dc445f441a Mon Sep 17 00:00:00 2001 From: darshankabariya Date: Fri, 29 May 2026 18:37:49 +0530 Subject: [PATCH 1/4] feat: build librln from prebuilt zerokit release assets --- flake.lock | 44 +------------------------ flake.nix | 94 +++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 72 insertions(+), 66 deletions(-) diff --git a/flake.lock b/flake.lock index 8d0db92691..411bf24303 100644 --- a/flake.lock +++ b/flake.lock @@ -19,8 +19,7 @@ "root": { "inputs": { "nixpkgs": "nixpkgs", - "rust-overlay": "rust-overlay", - "zerokit": "zerokit" + "rust-overlay": "rust-overlay" } }, "rust-overlay": { @@ -42,47 +41,6 @@ "repo": "rust-overlay", "type": "github" } - }, - "rust-overlay_2": { - "inputs": { - "nixpkgs": [ - "zerokit", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1771211437, - "narHash": "sha256-lcNK438i4DGtyA+bPXXyVLHVmJjYpVKmpux9WASa3ro=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "c62195b3d6e1bb11e0c2fb2a494117d3b55d410f", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } - }, - "zerokit": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ], - "rust-overlay": "rust-overlay_2" - }, - "locked": { - "owner": "vacp2p", - "repo": "zerokit", - "rev": "5e64cb8822bee65eed6cf459f95ae72b80c6ba63", - "type": "github" - }, - "original": { - "owner": "vacp2p", - "repo": "zerokit", - "rev": "5e64cb8822bee65eed6cf459f95ae72b80c6ba63", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 6c283780de..7ef5155435 100644 --- a/flake.nix +++ b/flake.nix @@ -17,19 +17,9 @@ url = "github:oxalica/rust-overlay"; inputs.nixpkgs.follows = "nixpkgs"; }; - - # External flake input: Zerokit pinned to a specific commit. - # Update the rev here when a new zerokit version is needed. - zerokit = { - # Pinned to v2.0.2 (5e64cb8822bee65eed6cf459f95ae72b80c6ba63) to match - # the vendor/zerokit submodule. Keep these two in sync: the nix build - # links librln from this input, the Makefile build from the submodule. - url = "github:vacp2p/zerokit/5e64cb8822bee65eed6cf459f95ae72b80c6ba63"; - inputs.nixpkgs.follows = "nixpkgs"; - }; }; - outputs = { self, nixpkgs, rust-overlay, zerokit }: + outputs = { self, nixpkgs, rust-overlay }: let systems = [ "x86_64-linux" "aarch64-linux" @@ -69,19 +59,78 @@ inherit system; overlays = [ (import rust-overlay) nimbleOverlay ]; }; + + # Prebuilt zerokit librln, fetched from the upstream GitHub release + # rather than compiled from source. Building zerokit from source makes + # Nix's fetch-cargo-vendor-util pull ~100 crates from crates.io in one + # parallel burst, which crates.io's CDN intermittently answers with 403 + # (per-IP rate limiting on the self-hosted runners), breaking nix CI. + # The release ships the exact `stateless` flavor this project links + # (see scripts/build_rln.sh), so we consume it directly — no Rust + # toolchain, no crates.io, no cargoHash to keep in sync. + # + # Keep `rlnVersion` aligned with `LIBRLN_VERSION` in the Makefile and + # the vendor/zerokit submodule. The hashes are the SRI sha256 of each + # release tarball; refresh all four when bumping the version. + rlnVersion = "v2.0.2"; + rlnAssets = { + "x86_64-linux" = { triple = "x86_64-unknown-linux-gnu"; hash = "sha256-qbrUdaetYKFhjzxUP/QcwD3JHWJ8qk/tCMK3yXceIAk="; }; + "aarch64-linux" = { triple = "aarch64-unknown-linux-gnu"; hash = "sha256-s4bWrmCcNTWHNyJwV73ilWNp58ZdAVG+TAgtWN1cTQs="; }; + "x86_64-darwin" = { triple = "x86_64-apple-darwin"; hash = "sha256-ZaHP5CApN66FYY7jxwOmGcF9kJR78Fng3k1qE2W08Mk="; }; + "aarch64-darwin" = { triple = "aarch64-apple-darwin"; hash = "sha256-f2YppkPsKFdN00j+IY8fpvsebWTIb9lW/V1/vOTiVKU="; }; + }; + + mkZerokitRln = system: pkgs: + let + asset = rlnAssets.${system} or + (throw "zerokit ${rlnVersion} has no prebuilt rln asset for system '${system}'"); + in pkgs.stdenv.mkDerivation { + pname = "librln"; + version = lib.removePrefix "v" rlnVersion; + + src = pkgs.fetchurl { + url = "https://github.com/vacp2p/zerokit/releases/download/" + + "${rlnVersion}/${asset.triple}-stateless-rln.tar.gz"; + hash = asset.hash; + }; + + # The tarball lays its files out under release/. + sourceRoot = "release"; + dontConfigure = true; + dontBuild = true; + + # The release .so was linked on a non-Nix toolchain; rewire its + # NEEDED libs (libgcc_s, libstdc++, glibc) onto the Nix closure so + # it loads inside the Nix-built consumer. autoPatchelfHook is a + # no-op for the static .a, and the whole step is skipped on Darwin + # (dylib install names are fixed downstream in nix/default.nix). + nativeBuildInputs = + pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.autoPatchelfHook ]; + buildInputs = + pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.stdenv.cc.cc.lib ]; + + installPhase = '' + runHook preInstall + mkdir -p $out/lib + cp librln.a $out/lib/ 2>/dev/null || true + cp librln.so $out/lib/ 2>/dev/null || true + cp librln.dylib $out/lib/ 2>/dev/null || true + runHook postInstall + ''; + + meta = with pkgs.lib; { + description = "Prebuilt zerokit RLN library (stateless flavor)"; + homepage = "https://github.com/vacp2p/zerokit"; + license = with licenses; [ mit asl20 ]; + platforms = builtins.attrNames rlnAssets; + }; + }; in { packages = forAllSystems (system: let pkgs = pkgsFor system; - # HACK: Fix for stale cargoHash in 2.0.2 release. - zerokitRln = zerokit.packages.${system}.rln.overrideAttrs (old: { - cargoDeps = old.cargoDeps.overrideAttrs (oldCargoDeps: { - vendorStaging = oldCargoDeps.vendorStaging.overrideAttrs (_: { - outputHash = "sha256-PNwEdZLgGQPqQDrEK2hsQtSybVfBbD6xn4K47fPFJUU="; - }); - }); - }); + zerokitRln = mkZerokitRln system pkgs; liblogosdelivery = pkgs.callPackage ./nix/default.nix { inherit pkgs; @@ -94,14 +143,13 @@ inherit pkgs; src = ./.; targets = ["wakucanary"]; - zerokitRln = zerokit.packages.${system}.rln; + inherit zerokitRln; }; in { inherit liblogosdelivery wakucanary; - # Expose the cargoHash-corrected librln so downstream consumers + # Expose the prebuilt librln so downstream consumers # (e.g. logos-delivery-module) bundle the exact same librln this - # build links, instead of pulling zerokit's rln directly — whose - # committed cargoHash is stale for v2.0.2 (see zerokitRln above). + # build links against. rln = zerokitRln; default = liblogosdelivery; } From 5c1ec9f025e16e4cbe6dbf6f9910e5aa9eabf19b Mon Sep 17 00:00:00 2001 From: darshankabariya Date: Fri, 29 May 2026 19:23:06 +0530 Subject: [PATCH 2/4] feat: fetch prebuilt zerokit rln, fall back to source build --- scripts/build_rln.sh | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/scripts/build_rln.sh b/scripts/build_rln.sh index 35b5b89536..8fa35a4268 100755 --- a/scripts/build_rln.sh +++ b/scripts/build_rln.sh @@ -1,8 +1,16 @@ #!/usr/bin/env bash -# This script is used to build the rln library for the current platform. -# Previously downloaded prebuilt binaries, but due to compatibility issues -# we now always build from source. +# Provides the rln static library for the current platform. +# +# Strategy: if zerokit publishes a prebuilt `stateless` asset for this host's +# target triple, download it — that's faster than compiling and avoids pulling +# zerokit's ~100 crates from crates.io. Otherwise fall back to building from +# the vendored zerokit submodule. +# +# (Prebuilt was dropped in #3712 and is restored here. The earlier "compatibility +# issues" note was inaccurate: the release ships the same `stateless` librln.a +# this script builds, and its glibc floor is GLIBC_2.18 — below every platform +# we target. A missing asset just falls through to the source build.) set -e @@ -15,8 +23,26 @@ output_filename=$3 [[ -z "${rln_version}" ]] && { echo "No rln version specified"; exit 1; } [[ -z "${output_filename}" ]] && { echo "No output filename specified"; exit 1; } -echo "Building RLN library from source (version ${rln_version})..." +# --- Prefer the prebuilt release asset -------------------------------------- +# Host target triple, e.g. x86_64-unknown-linux-gnu / aarch64-apple-darwin. +host_triplet=$(rustc --version --verbose | awk '/host:/{print $2}') +tarball="${host_triplet}-stateless-rln.tar.gz" +url="https://github.com/vacp2p/zerokit/releases/download/${rln_version}/${tarball}" + +echo "Looking for prebuilt RLN: ${url}" +if curl --silent --fail-with-body -L "${url}" -o "${tarball}"; then + echo "Downloaded prebuilt ${tarball}" + tar -xzf "${tarball}" + mv "release/librln.a" "${output_filename}" + rm -rf "${tarball}" release + echo "Using prebuilt ${output_filename}" + exit 0 +fi +# curl --fail-with-body writes the error body to the file on HTTP failure. +rm -f "${tarball}" +echo "No prebuilt asset for ${host_triplet} at ${rln_version}; building from source." +# --- Fall back to building from the vendored submodule ---------------------- # Check if submodule version = version in Makefile cargo metadata --format-version=1 --no-deps --manifest-path "${build_dir}/rln/Cargo.toml" @@ -33,7 +59,6 @@ if [[ "v${submodule_version}" != "${rln_version}" ]]; then exit 1 fi -# Build rln from source. # `stateless` feature: logos-delivery does not maintain a local Merkle tree # (post-PR #3312); the contract is the source of truth and the path is fetched # via getMerkleProof(index). The stateless build compiles out tree code. From d49b3a68e34e4255c2f63f2d84ca6dd9dc7db097 Mon Sep 17 00:00:00 2001 From: darshankabariya Date: Fri, 29 May 2026 19:43:41 +0530 Subject: [PATCH 3/4] ci: run build/test when scripts and flake files change --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f924d0f8b3..84a7f0b8de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: - 'nimble.lock' - 'waku.nimble' - 'Makefile' + - 'scripts/**' + - 'flake.nix' + - 'flake.lock' - 'library/**' - 'liblogosdelivery/**' v2: From 319f9537bc9e5b19a717d8b3599c11699abd76c1 Mon Sep 17 00:00:00 2001 From: darshankabariya Date: Fri, 29 May 2026 23:51:40 +0530 Subject: [PATCH 4/4] docs: clarify comments header per review --- flake.nix | 30 +++++++++++++++--------------- scripts/build_rln.sh | 15 +++++++-------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/flake.nix b/flake.nix index 7ef5155435..b32a534550 100644 --- a/flake.nix +++ b/flake.nix @@ -61,17 +61,17 @@ }; # Prebuilt zerokit librln, fetched from the upstream GitHub release - # rather than compiled from source. Building zerokit from source makes - # Nix's fetch-cargo-vendor-util pull ~100 crates from crates.io in one - # parallel burst, which crates.io's CDN intermittently answers with 403 - # (per-IP rate limiting on the self-hosted runners), breaking nix CI. - # The release ships the exact `stateless` flavor this project links - # (see scripts/build_rln.sh), so we consume it directly — no Rust - # toolchain, no crates.io, no cargoHash to keep in sync. + # rather than compiled from source. Compiling zerokit makes Nix download + # its many crate dependencies from crates.io in one parallel burst, which + # crates.io intermittently rejects with HTTP 403 (rate limiting from the + # self-hosted runners' shared IP), breaking the nix build. The release + # ships the exact `stateless` library this project links (see + # scripts/build_rln.sh), so we use it directly — no Rust toolchain and + # no crates.io access needed. # - # Keep `rlnVersion` aligned with `LIBRLN_VERSION` in the Makefile and - # the vendor/zerokit submodule. The hashes are the SRI sha256 of each - # release tarball; refresh all four when bumping the version. + # Keep `rlnVersion` aligned with `LIBRLN_VERSION` in the Makefile and the + # vendor/zerokit submodule. Each hash is the sha256 of the release tarball + # for that platform; refresh all four when bumping the version. rlnVersion = "v2.0.2"; rlnAssets = { "x86_64-linux" = { triple = "x86_64-unknown-linux-gnu"; hash = "sha256-qbrUdaetYKFhjzxUP/QcwD3JHWJ8qk/tCMK3yXceIAk="; }; @@ -99,11 +99,11 @@ dontConfigure = true; dontBuild = true; - # The release .so was linked on a non-Nix toolchain; rewire its - # NEEDED libs (libgcc_s, libstdc++, glibc) onto the Nix closure so - # it loads inside the Nix-built consumer. autoPatchelfHook is a - # no-op for the static .a, and the whole step is skipped on Darwin - # (dylib install names are fixed downstream in nix/default.nix). + # The release .so was linked outside Nix, so it references system + # libraries (libgcc_s, libstdc++, glibc) by bare name. autoPatchelfHook + # points those at the Nix versions so the library loads correctly when + # used by the Nix build. It does nothing for the static .a, and the + # step is skipped on macOS (dylib paths are fixed in nix/default.nix). nativeBuildInputs = pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.autoPatchelfHook ]; buildInputs = diff --git a/scripts/build_rln.sh b/scripts/build_rln.sh index 8fa35a4268..b028885e24 100755 --- a/scripts/build_rln.sh +++ b/scripts/build_rln.sh @@ -2,15 +2,14 @@ # Provides the rln static library for the current platform. # -# Strategy: if zerokit publishes a prebuilt `stateless` asset for this host's -# target triple, download it — that's faster than compiling and avoids pulling -# zerokit's ~100 crates from crates.io. Otherwise fall back to building from -# the vendored zerokit submodule. +# If zerokit publishes a prebuilt `stateless` release asset for this platform, +# download and use it: that is faster than compiling and avoids fetching +# zerokit's many crate dependencies from crates.io. The asset is selected by +# the Rust host target triple (the platform identifier reported by rustc, +# e.g. x86_64-unknown-linux-gnu or aarch64-apple-darwin). # -# (Prebuilt was dropped in #3712 and is restored here. The earlier "compatibility -# issues" note was inaccurate: the release ships the same `stateless` librln.a -# this script builds, and its glibc floor is GLIBC_2.18 — below every platform -# we target. A missing asset just falls through to the source build.) +# When no matching asset exists (e.g. Windows), build from the vendored +# zerokit submodule instead. set -e