perf: VID ADVZ verify_share use parallelism over multiplicity

vid/.gitignore

@@ -0,0 +1 @@
+/target

vid/Cargo.toml

@@ -46,6 +46,11 @@ name = "advz"
 harness = false
 required-features = ["test-srs"]
 
+[[bench]]
+name = "advz_multiplicity"
+harness = false
+required-features = ["test-srs"]
+
 [features]
 default = ["parallel"]
 std = [

vid/benches/advz.rs

@@ -27,16 +27,15 @@ where
 {
     // play with these items
     //
-    // CODE_RATE is merely a convenient way to automatically choose polynomial
-    // degree as a function of storage node count.
-    // If desired, you could set polynomial degrees independent of storage node
-    // count.
-    const CODE_RATE: u32 = 4; // ratio of num_storage_nodes : polynomial_degree
+    // INVERSE_CODE_RATE is merely a convenient way to automatically choose
+    // polynomial degree as a function of storage node count. If desired, you
+    // could set polynomial degrees independent of storage node count.
+    const INVERSE_CODE_RATE: u32 = 4; // ratio of num_storage_nodes : polynomial_degree
     let storage_node_counts = [512, 1024];
     let payload_byte_lens = [1 * MB];
 
     // more items as a function of the above
-    let poly_degrees_iter = storage_node_counts.iter().map(|c| c / CODE_RATE);
+    let poly_degrees_iter = storage_node_counts.iter().map(|c| c / INVERSE_CODE_RATE);
     let supported_degree = poly_degrees_iter.clone().max().unwrap();
     let vid_sizes_iter = poly_degrees_iter.zip(storage_node_counts);
     let mut rng = jf_utils::test_rng();

vid/benches/advz_multiplicity.rs

@@ -0,0 +1,105 @@
+// Copyright (c) 2024 Espresso Systems (espressosys.com)
+// This file is part of the Jellyfish library.
+
+// You should have received a copy of the MIT License
+// along with the Jellyfish library. If not, see <https://mit-license.org/>.
+
+//! Benchmarks demonstrating performance improvement in [`Advz::verify_share`]
+//! from use of parallelism over `multiplicity`.
+//!
+//! Run
+//! ```
+//! cargo bench --bench=advz_multiplicity --features="test-srs"
+//! ```
+//!
+//! By
+//! [default](https://github.com/rayon-rs/rayon/blob/main/FAQ.md#how-many-threads-will-rayon-spawn)
+//! the number of threads = number of available CPU cores. You can override this
+//! choice by prevising the above command with `RAYON_NUM_THREADS=N `. Example:
+//! set `N=1` to eliminate parallelism.
+
+use ark_bn254::Bn254;
+use ark_ec::pairing::Pairing;
+use ark_serialize::Write;
+use ark_std::rand::RngCore;
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use digest::{crypto_common::generic_array::ArrayLength, Digest, DynDigest, OutputSizeUser};
+use jf_pcs::{checked_fft_size, prelude::UnivariateKzgPCS, PolynomialCommitmentScheme};
+use jf_utils::field_byte_len;
+use jf_vid::{advz::Advz, VidScheme};
+use sha2::Sha256;
+
+const KB: usize = 1 << 10;
+// const MB: usize = KB << 10;
+
+fn advz<E, H>(c: &mut Criterion)
+where
+    E: Pairing,
+    // TODO(Gus) clean up nasty trait bounds upstream
+    H: Digest + DynDigest + Default + Clone + Write + Send + Sync,
+    <<H as OutputSizeUser>::OutputSize as ArrayLength<u8>>::ArrayType: Copy,
+{
+    // play with these items
+    //
+    // INVERSE_CODE_RATE is merely a convenient way to automatically choose
+    // recovery threshold as a function of storage node count. If desired, you
+    // could set recovery thresholds independent of storage node counts.
+    let multiplicities = [1, 256];
+    let num_storage_nodes = 128;
+    const INVERSE_CODE_RATE: usize = 4; // ratio of num_storage_nodes : recovery_threshold
+
+    // more items as a function of the above
+    let recovery_threshold = num_storage_nodes / INVERSE_CODE_RATE;
+    let max_multiplicity = multiplicities.iter().max().unwrap();
+    let max_degree = recovery_threshold * max_multiplicity;
+    let coeff_byte_len = field_byte_len::<E::ScalarField>();
+    let payload_byte_len = {
+        // ensure payload is large enough to fill at least 1 polynomial at
+        // maximum multiplicity.
+        max_degree * coeff_byte_len
+    };
+    let mut rng = jf_utils::test_rng();
+    let payload_bytes = {
+        // random payload data
+        let mut payload_bytes = vec![0u8; payload_byte_len];
+        rng.fill_bytes(&mut payload_bytes);
+        payload_bytes
+    };
+    let srs =
+        UnivariateKzgPCS::<E>::gen_srs_for_testing(&mut rng, checked_fft_size(max_degree).unwrap())
+            .unwrap();
+
+    let benchmark_group_name = format!(
+        "advz_verify_payload_{}KB_multiplicity",
+        payload_byte_len / KB
+    );
+    let mut grp = c.benchmark_group(benchmark_group_name);
+    for multiplicity in multiplicities {
+        let mut advz = Advz::<E, H>::with_multiplicity(
+            num_storage_nodes.try_into().unwrap(),
+            recovery_threshold.try_into().unwrap(),
+            multiplicity.try_into().unwrap(),
+            &srs,
+        )
+        .unwrap();
+        let disperse = advz.disperse(&payload_bytes).unwrap();
+        let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit);
+        grp.bench_function(BenchmarkId::from_parameter(multiplicity), |b| {
+            // verify only the 0th share
+            b.iter(|| {
+                advz.verify_share(&shares[0], &common, &commit)
+                    .unwrap()
+                    .unwrap()
+            });
+        });
+    }
+    grp.finish();
+}
+
+fn advz_main(c: &mut Criterion) {
+    advz::<Bn254, Sha256>(c);
+}
+
+criterion_group!(name = benches; config = Criterion::default().sample_size(10); targets = advz_main);
+
+criterion_main!(benches);

vid/src/advz.rs

@@ -78,6 +78,7 @@ pub type AdvzGPU<'srs, E, H> = AdvzInternal<
 pub struct AdvzInternal<E, H, T>
 where
     E: Pairing,
+    T: Sync,
 {
     recovery_threshold: u32,
     num_storage_nodes: u32,
@@ -120,6 +121,7 @@ type KzgEvalsMerkleTreeProof<E, H> =
 impl<E, H, T> AdvzInternal<E, H, T>
 where
     E: Pairing,
+    T: Sync,
 {
     pub(crate) fn new_internal(
         num_storage_nodes: u32,  // n (code rate: r = k/n)
@@ -139,8 +141,6 @@ where
         multiplicity: u32,       // batch m chunks, keep the rate r = (m*k)/(m*n)
         srs: impl Borrow<KzgSrs<E>>,
     ) -> VidResult<Self> {
-        // TODO support any degree, give multiple shares to nodes if needed
-        // https://github.com/EspressoSystems/jellyfish/issues/393
         if num_storage_nodes < recovery_threshold {
             return Err(VidError::Argument(format!(
                 "recovery_threshold {} exceeds num_storage_nodes {}",
@@ -392,6 +392,7 @@ impl<E, H, T> VidScheme for AdvzInternal<E, H, T>
 where
     E: Pairing,
     H: HasherDigest,
+    T: Sync,
     AdvzInternal<E, H, T>: MaybeGPU<E>,
 {
     // use HasherNode<H> instead of Output<H> to easily meet trait bounds
@@ -562,28 +563,43 @@ where
         );
 
         // verify aggregate proof
-        (0..self.multiplicity as usize)
-            .map(|i| {
-                let aggregate_eval = polynomial_eval(
-                    share.evals[i * polys_len..(i + 1) * polys_len]
-                        .iter()
-                        .map(FieldMultiplier),
-                    pseudorandom_scalar,
-                );
-                Ok(UnivariateKzgPCS::verify(
-                    &self.vk,
-                    &aggregate_poly_commit,
-                    &self
-                        .multi_open_domain
-                        .element((share.index as usize * multiplicity) + i),
-                    &aggregate_eval,
-                    &share.aggregate_proofs[i],
-                )
-                .map_err(vid)?
-                .then_some(())
-                .ok_or(()))
-            })
-            .collect()
+        //
+        // some boilerplate needed to accommodate builds without `parallel`
+        // feature.
+        let multiplicities = Vec::from_iter((0..self.multiplicity as usize));
+        let verification_iter = parallelizable_slice_iter(&multiplicities).map(|i| {
+            let aggregate_eval = polynomial_eval(
+                share.evals[i * polys_len..(i + 1) * polys_len]
+                    .iter()
+                    .map(FieldMultiplier),
+                pseudorandom_scalar,
+            );
+            Ok(UnivariateKzgPCS::verify(
+                &self.vk,
+                &aggregate_poly_commit,
+                &self
+                    .multi_open_domain
+                    .element((share.index as usize * multiplicity) + i),
+                &aggregate_eval,
+                &share.aggregate_proofs[*i],
+            )
+            .map_err(vid)?
+            .then_some(())
+            .ok_or(()))
+        });
+        let abort = |result: &VidResult<Result<(), ()>>| match result {
+            Ok(success) => success.is_err(),
+            Err(_) => true,
+        };
+
+        // abort immediately on any failure of verification
+        #[cfg(feature = "parallel")]
+        let result = verification_iter.find_any(abort);
+
+        #[cfg(not(feature = "parallel"))]
+        let result = verification_iter.clone().find(abort); // `clone` because we need mutable
+
+        result.unwrap_or(Ok(Ok(())))
     }
 
     fn recover_payload(&self, shares: &[Self::Share], common: &Self::Common) -> VidResult<Vec<u8>> {
@@ -697,6 +713,7 @@ impl<E, H, SrsRef> AdvzInternal<E, H, SrsRef>
 where
     E: Pairing,
     H: HasherDigest,
+    SrsRef: Sync,
     AdvzInternal<E, H, SrsRef>: MaybeGPU<E>,
 {
     fn evaluate_polys(

vid/src/advz/payload_prover.rs

@@ -66,6 +66,7 @@ impl<E, H, T> PayloadProver<SmallRangeProof<KzgProof<E>>> for AdvzInternal<E, H,
 where
     E: Pairing,
     H: HasherDigest,
+    T: Sync,
     AdvzInternal<E, H, T>: MaybeGPU<E>,
 {
     fn payload_proof<B>(
@@ -202,6 +203,7 @@ impl<E, H, T> PayloadProver<LargeRangeProof<KzgEval<E>>> for AdvzInternal<E, H,
 where
     E: Pairing,
     H: HasherDigest,
+    T: Sync,
     AdvzInternal<E, H, T>: MaybeGPU<E>,
 {
     fn payload_proof<B>(
@@ -279,6 +281,7 @@ impl<E, H, T> AdvzInternal<E, H, T>
 where
     E: Pairing,
     H: HasherDigest,
+    T: Sync,
     AdvzInternal<E, H, T>: MaybeGPU<E>,
 {
     // lots of index manipulation

vid/src/advz/precomputable.rs

@@ -26,6 +26,7 @@ impl<E, H, T> Precomputable for AdvzInternal<E, H, T>
 where
     E: Pairing,
     H: HasherDigest,
+    T: Sync,
     AdvzInternal<E, H, T>: MaybeGPU<E>,
 {
     type PrecomputeData = PrecomputeData<E>;