Upgrade zcash_script to zcashd v5.5.0

.gitignore

@@ -1,4 +1,5 @@
 .idea
 target
 *.iml
-Cargo.lock
+Cargo.lock
+.\#*

CHANGELOG.md

@@ -8,6 +8,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased] - ReleaseDate
 
+## [0.1.12] - 2023-05-03
+
+### Changed
+- Update `depend/zcash` to version 5.5.0 which includes updated dependencies
+    - This includes additional `zcashd` C++ and Rust code, and its dependencies
+    - Fix code generation C++ header paths to avoid conflicts
+- Update other dependencies to match Zebra
+
+### Fixed
+- Improve error reporting in `build.rs`
+
 ## [0.1.11] - 2023-02-24
 
 ### Changed

Cargo.toml

@@ -31,10 +31,10 @@ include = [
     "/depend/zcash/src/script/interpreter.cpp",
     "/depend/zcash/src/script/script.cpp",
     "/depend/zcash/src/script/script_error.cpp",
-    "/depend/zcash/src",
+    "/depend/zcash/src/",
     "/depend/zcash/src/rust/include/",
-    "/depend/zcash/src/secp256k1/include",
-    "/depend/zcash/src/secp256k1",
+    "/depend/zcash/src/secp256k1/include/",
+    "/depend/zcash/src/secp256k1/",
     "/depend/zcash/src/support/cleanse.cpp",
     "/depend/zcash/src/support/cleanse.h",
     "/depend/zcash/src/rust/gen/",
@@ -50,16 +50,28 @@ external-secp = []
 [dependencies]
 # All these dependencies must match the versions in:
 # https://github.com/zcash/zcash/blob/<git subtree version>/Cargo.toml
+bellman = "0.14"
 blake2b_simd = "1"
-cxx = { version = "=1.0.83", features = ["c++17"] }
+bls12_381 = "0.8"
+byteorder = "1"
+crossbeam-channel = "0.5"
+cxx = { version = "=1.0.94", features = ["c++17"] }
+group = "0.13"
+incrementalmerkletree = "0.3"
+jubjub = "0.10"
 libc = "0.2"
 memuse = "0.2"
-orchard = "0.3"
+metrics = "0.20"
+orchard = "0.4"
 rand_core = "0.6"
+rayon = "1.5"
+subtle = "2.2"
 tracing = "0.1"
+zcash_address = "0.2"
 zcash_encoding = "0.2"
-zcash_note_encryption = "0.2"
-zcash_primitives = { version = "0.9.1", features = ["transparent-inputs"] }
+zcash_note_encryption = "0.3"
+zcash_primitives = { version = "0.11", features = ["temporary-zcashd", "transparent-inputs"] }
+zcash_proofs = "0.11"
 
 [build-dependencies]
 # The `bindgen` dependency should automatically upgrade to match the version used by zebra-state's `rocksdb` dependency in:
@@ -71,10 +83,10 @@ bindgen = ">= 0.64.0"
 # These dependencies are shared with a lot of other Zebra dependencies,
 # so they are configured to automatically upgrade to match Zebra.
 # But we try to use the latest versions here, to catch any bugs in `zcash_script`'s CI.
-cc = { version = "1.0.73", features = ["parallel"] }
+cc = { version = "1.0.79", features = ["parallel"] }
 # Treat minor versions with a zero major version as compatible (cargo doesn't by default).
-cxx-gen = ">= 0.7.74"
-syn = { version = "1.0.104", features = ["full", "printing"] }
+cxx-gen = ">= 0.7.93"
+syn = { version = "1.0.109", features = ["full", "printing"] }
 
 [dev-dependencies]
 # These dependencies are shared with a lot of other Zebra dependencies.

README.md

@@ -22,16 +22,17 @@ to include a lot of other stuff e.g. the orchard library.
 
 ### Updating this crate
 
-1. Update `depend/zcash` with the latest tagged version of `zcashd`
+1. Update `depend/zcash` with the latest tagged version of `zcashd`, using the instructions below
 2. Update `Cargo.toml` versions to match the versions used by the latest tagged version of `zcashd`, and its dependencies
 3. For dependencies that are shared with Zebra (but not `zcashd`), match the latest version in Zebra's [Cargo.lock](https://github.com/ZcashFoundation/zebra/blob/main/Cargo.lock):
     - use `cargo tree --invert <crate>` to see if the crate is from `zcash_script` or another dependency
     - see the list in [Cargo.toml](https://github.com/ZcashFoundation/zcash_script/blob/master/Cargo.toml#L69)
 4. For new dependencies with a leading zero in their version (`0.x.y`), use a `>=` dependency [to make them automatically upgrade to match Zebra's dependencies](https://doc.rust-lang.org/cargo/reference/resolver.html#semver-compatibility)
 5. Check all open PRs to see if they can be merged before the release
-6. Run `cargo-release` with `--no-publish` to commit any automatic changes
+6. Run `cargo release patch` to commit the release version bump (but not actually publish)
 7. Open a `zcash_script` PR with the changes, get it reviewed, and wait for CI to pass
-8. Publish a new release
+8. Publish a new release using `cargo release --execute patch`
+9. Check the release tag was pushed to https://github.com/ZcashFoundation/zcash_script/tags
 
 ### Updating `depend/zcash`
 

build.rs

@@ -1,3 +1,5 @@
+//! Build script for zcash_script.
+
 use std::{env, fmt, fs, io::Read, path::PathBuf};
 
 use syn::__private::ToTokens;
@@ -62,23 +64,19 @@ fn bindgen_headers() -> Result<()> {
 /// (This is in contrast to zcash which generates in `depend/zcash/src/rust/gen/`)
 fn gen_cxxbridge() -> Result<()> {
     let out_path = env::var("OUT_DIR").map_err(Error::Env)?;
-    let out_path = PathBuf::from(out_path).join("include");
+    let out_path = PathBuf::from(out_path).join("gen");
+    let src_out_path = PathBuf::from(&out_path).join("src");
+    let header_out_path = PathBuf::from(&out_path).join("include").join("rust");
 
     // These must match `CXXBRIDGE_RS` in depend/zcash/src/Makefile.am
-    let filenames = [
-        "blake2b",
-        "bundlecache",
-        "equihash",
-        "orchard_bundle",
-        "sapling",
-        "wallet_scanner",
-    ];
+    let filenames = ["blake2b", "ed25519", "equihash", "streams", "bridge"];
 
     // The output folder must exist
-    fs::create_dir_all(out_path.join("rust")).unwrap();
+    fs::create_dir_all(&src_out_path).unwrap();
+    fs::create_dir_all(&header_out_path).unwrap();
 
     // Generate the generic header file
-    fs::write(out_path.join("rust/cxx.h"), cxx_gen::HEADER).unwrap();
+    fs::write(header_out_path.join("cxx.h"), cxx_gen::HEADER).unwrap();
 
     // Generate the source and header for each bridge file
     for filename in filenames {
@@ -99,11 +97,19 @@ fn gen_cxxbridge() -> Result<()> {
             path: "rust/cxx.h".to_string(),
             kind: cxx_gen::IncludeKind::Quoted,
         });
-        let output = cxx_gen::generate_header_and_cc(token_stream, &opt).unwrap();
+        let output = cxx_gen::generate_header_and_cc(token_stream, &opt).unwrap_or_else(|err| {
+            panic!(
+                "invalid bridge file {filename}: {err}. Try updating `filenames` to match zcashd"
+            )
+        });
 
-        fs::write(out_path.join(format!("rust/{}.h", filename)), output.header).unwrap();
         fs::write(
-            out_path.join(format!("rust/{}.c", filename)),
+            header_out_path.join(format!("{}.h", filename)),
+            output.header,
+        )
+        .unwrap();
+        fs::write(
+            src_out_path.join(format!("{}.c", filename)),
             output.implementation,
         )
         .unwrap();
@@ -115,19 +121,19 @@ fn main() -> Result<()> {
     bindgen_headers()?;
     gen_cxxbridge()?;
 
-    let include_path = env::var("OUT_DIR").map_err(Error::Env)?;
-    let include_path = PathBuf::from(include_path).join("include");
+    let gen_path = env::var("OUT_DIR").map_err(Error::Env)?;
+    let gen_path = PathBuf::from(gen_path).join("gen");
 
     let target = env::var("TARGET").expect("TARGET was not set");
     let mut base_config = cc::Build::new();
 
     language_std(&mut base_config, "c++17");
 
     base_config
-        .include("depend/zcash/src")
+        .include("depend/zcash/src/")
         .include("depend/zcash/src/rust/include/")
-        .include("depend/zcash/src/secp256k1/include")
-        .include(&include_path)
+        .include("depend/zcash/src/secp256k1/include/")
+        .include(&gen_path.join("include"))
         .flag_if_supported("-Wno-implicit-fallthrough")
         .flag_if_supported("-Wno-catch-value")
         .flag_if_supported("-Wno-reorder")
@@ -156,6 +162,7 @@ fn main() -> Result<()> {
         .file("depend/zcash/src/uint256.cpp")
         .file("depend/zcash/src/pubkey.cpp")
         .file("depend/zcash/src/hash.cpp")
+        .file("depend/zcash/src/streams_rust.cpp")
         .file("depend/zcash/src/primitives/transaction.cpp")
         .file("depend/zcash/src/crypto/ripemd160.cpp")
         .file("depend/zcash/src/crypto/sha1.cpp")
@@ -166,7 +173,11 @@ fn main() -> Result<()> {
         .file("depend/zcash/src/script/script.cpp")
         .file("depend/zcash/src/script/script_error.cpp")
         .file("depend/zcash/src/support/cleanse.cpp")
-        .file(include_path.join("rust/blake2b.c"))
+        // A subset of the files generated by gen_cxxbridge
+        // which are required by zcash_script.
+        .file(gen_path.join("src/blake2b.c"))
+        .file(gen_path.join("src/bridge.c"))
+        .file(gen_path.join("src/streams.c"))
         .compile("libzcash_script.a");
 
     Ok(())
@@ -181,7 +192,8 @@ fn build_secp256k1() {
 
     // Define configuration constants
     build
-        .define("SECP256K1_BUILD", "1")
+        // This matches the #define in depend/zcash/src/secp256k1/src/secp256k1.c
+        .define("SECP256K1_BUILD", "")
         .define("USE_NUM_NONE", "1")
         .define("USE_FIELD_INV_BUILTIN", "1")
         .define("USE_SCALAR_INV_BUILTIN", "1")

depend/zcash/.editorconfig

@@ -12,3 +12,6 @@ indent_size = 4
 indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
+
+[{Makefile*, *.mk}]
+indent_style = tab

depend/zcash/.github/workflows/audits.yml

@@ -11,20 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-
-      - name: Install cargo-vet
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: cargo-vet
-
-      - name: Run cargo vet --locked
-        uses: actions-rs/cargo@v1
-        with:
-          command: vet
-          args: --locked
+      - uses: dtolnay/rust-toolchain@stable
+        id: toolchain
+      - run: rustup override set ${{steps.toolchain.outputs.name}}
+      - run: cargo install cargo-vet --version ~0.6
+      - run: cargo vet --locked

depend/zcash/.github/workflows/book.yml

@@ -21,10 +21,7 @@ jobs:
           mdbook-version: 'latest'
 
       - name: Install mdbook-katex
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: mdbook-katex
+        run: cargo install mdbook-katex
 
       - name: Build zcashd book
         run: mdbook build doc/book/

depend/zcash/.github/workflows/build.yml

@@ -0,0 +1,112 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    name: Tier ${{ matrix.tier }} platform ${{ matrix.platform }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.tier == 3 }}
+    strategy:
+      matrix:
+        include:
+          - name: ubuntu-20.04
+            tier: 1
+            platform: Ubuntu 20.04
+            os: ubuntu-20.04
+
+          - name: macos-11
+            tier: 3
+            platform: macOS Big Sur 11
+            os: macos-11
+            brew_deps: >
+              autoconf
+              automake
+              coreutils
+              libtool
+              pkgconfig
+
+          - name: mingw32
+            tier: 3
+            platform: Windows (64-bit MinGW)
+            os: ubuntu-latest
+            cross_deps: >
+              mingw-w64
+            host: HOST=x86_64-w64-mingw32
+
+          - name: aarch64-linux
+            tier: 3
+            platform: ARM64 Linux
+            os: ubuntu-latest
+            cross_deps: >
+              g++-aarch64-linux-gnu
+            host: HOST=aarch64-linux-gnu
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Homebrew build dependencies
+        if: matrix.brew_deps != ''
+        run: brew install ${{ matrix.brew_deps }}
+
+      - name: Install cross-compilation build dependencies
+        if: matrix.cross_deps != ''
+        run: sudo apt install ${{ matrix.cross_deps }}
+
+      - name: Configure MinGW to use POSIX variant
+        if: matrix.name == 'mingw32'
+        run: |
+          sudo update-alternatives --set x86_64-w64-mingw32-gcc $(update-alternatives --query x86_64-w64-mingw32-gcc | grep Alternative | grep posix | cut -d' ' -f2)
+          sudo update-alternatives --set x86_64-w64-mingw32-g++ $(update-alternatives --query x86_64-w64-mingw32-g++ | grep Alternative | grep posix | cut -d' ' -f2)
+
+      - name: Cache built dependencies
+        uses: actions/cache@v3
+        with:
+          path: depends/built
+          key: ${{ matrix.name }}-built-${{ hashFiles('depends/Makefile', 'depends/funcs.mk') }}-${{ hashFiles('depends/packages/*.mk', 'depends/patches/**/*') }}
+          restore-keys: |
+            ${{ matrix.name }}-built-${{ hashFiles('depends/Makefile', 'depends/funcs.mk') }}-
+
+      - name: Prepare ccache timestamp
+        id: ccache_cache_timestamp
+        shell: bash
+        run: echo "timestamp=$(date +'%Y-%m-%d-%H;%M;%S')" >> "$GITHUB_OUTPUT"
+
+      - name: Cache ccache files
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/ccache
+          key: ${{ matrix.name }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
+          restore-keys: |
+            ${{ matrix.name }}-ccache-
+
+      - name: Build zcashd
+        id: build
+        run: >
+          ${{ matrix.host }}
+          ./zcutil/build.sh
+          -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)"
+
+      - name: Build zcashd with libraries enabled
+        if: ${{ always() && steps.build.outcome == 'success' }}
+        run: >
+          CONFIGURE_FLAGS="--with-libs"
+          ${{ matrix.host }}
+          ./zcutil/build.sh
+          -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)"
+
+      - name: Build zcashd with wallet disabled
+        if: ${{ always() && steps.build.outcome == 'success' }}
+        run: >
+          CONFIGURE_FLAGS="--disable-wallet"
+          ${{ matrix.host }}
+          ./zcutil/build.sh
+          -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)"
+
+      - name: Build zcashd with mining disabled
+        if: ${{ always() && steps.build.outcome == 'success' }}
+        run: >
+          CONFIGURE_FLAGS="--disable-mining"
+          ${{ matrix.host }}
+          ./zcutil/build.sh
+          -j"$(nproc 2> /dev/null || sysctl -n hw.ncpu)"