Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: replace payload_chunk_size with erasure_code_rate #533

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: replace payload_chunk_size with erasure_code_rate #533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 40 additions & 29 deletions primitives/src/vid/advz.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,20 +128,22 @@ where
/// Return [`VidError::Argument`] if `num_storage_nodes <
/// payload_chunk_size`.
pub(crate) fn new_internal(
payload_chunk_size: usize, // k
num_storage_nodes: usize, // n (code rate: r = k/n)
multiplicity: usize, // batch m chunks, keep the rate r = (m*k)/(m*n)
erasure_code_rate: usize, // inverse of r
num_storage_nodes: usize, // n
multiplicity: usize, // m
srs: impl Borrow<KzgSrs<E>>,
) -> VidResult<Self> {
// TODO support any degree, give multiple shares to nodes if needed
// https://github.com/EspressoSystems/jellyfish/issues/393
if num_storage_nodes < payload_chunk_size {
// return error if rate r > 1 (implied by erasure_code_rate < 1)
if erasure_code_rate < 1 {
return Err(VidError::Argument(format!(
"payload_chunk_size {} exceeds num_storage_nodes {}",
payload_chunk_size, num_storage_nodes
"erasure_code_rate {} should be greater than 1",
erasure_code_rate
)));
}

// payload_chunk_size: k = r*n
let payload_chunk_size = num_storage_nodes.div_ceil(erasure_code_rate);

if !(1..=16).contains(&multiplicity) || !multiplicity.is_power_of_two() {
return Err(VidError::Argument(format!(
"multiplicity {} not allowed",
Expand Down Expand Up @@ -211,20 +213,20 @@ where
{
/// Construct a new VID instance
///
/// - `payload_chunk_size`: k
/// - `num_storage_nodes`: n (code rate: r = k/n)
/// - `erasure_code_rate`: r
/// - `num_storage_nodes`: n (payload_chunk_size: k = r*n)
/// - `multiplicity`: batch m chunks, keep the rate r = (m*k)/(m*n)
///
/// # Errors
/// Return [`VidError::Argument`] if `num_storage_nodes <
/// payload_chunk_size`.
pub fn new(
payload_chunk_size: usize,
erasure_code_rate: usize,
num_storage_nodes: usize,
multiplicity: usize,
srs: impl Borrow<KzgSrs<E>>,
) -> VidResult<Self> {
Self::new_internal(payload_chunk_size, num_storage_nodes, multiplicity, srs)
Self::new_internal(erasure_code_rate, num_storage_nodes, multiplicity, srs)
}
}

Expand All @@ -236,21 +238,20 @@ where
{
/// construct a new VID instance with SRS loaded to GPU
///
/// - `payload_chunk_size`: k
/// - `num_storage_nodes`: n (code rate: r = k/n)
/// - `erasure_code_rate`: r
/// - `num_storage_nodes`: n (payload_chunk_size: k = r*n)
/// - `multiplicity`: batch m chunks, keep the rate r = (m*k)/(m*n)
///
/// # Errors
/// Return [`VidError::Argument`] if `num_storage_nodes <
/// payload_chunk_size`.
pub fn new(
payload_chunk_size: usize,
erasure_code_rate: usize,
num_storage_nodes: usize,
multiplicity: usize,
srs: impl Borrow<KzgSrs<E>>,
) -> VidResult<Self> {
let mut advz =
Self::new_internal(payload_chunk_size, num_storage_nodes, multiplicity, srs)?;
let mut advz = Self::new_internal(erasure_code_rate, num_storage_nodes, multiplicity, srs)?;
let srs_on_gpu = <UnivariateKzgPCS<E> as GPUCommittable<E>>::load_prover_param_to_gpu(
&advz.ck,
advz.ck.powers_of_g.len() - 1,
Expand Down Expand Up @@ -969,15 +970,21 @@ mod tests {
#[test]
fn disperse_timer() {
// run with 'print-trace' feature to see timer output
let (payload_chunk_size, num_storage_nodes) = (256, 512);
let (erasure_code_rate, num_storage_nodes, multiplicity) = (4usize, 512usize, 1usize);
let payload_chunk_size = num_storage_nodes.div_ceil(erasure_code_rate);
let mut rng = jf_utils::test_rng();
let srs = init_srs(payload_chunk_size, &mut rng);
let srs = init_srs(payload_chunk_size * multiplicity, &mut rng);
let mut advz =
Advz::<Bn254, Sha256>::new(payload_chunk_size, num_storage_nodes, 1, srs).unwrap();
#[cfg(feature = "gpu-vid")]
let mut advz_gpu =
AdvzGPU::<'_, Bn254, Sha256>::new(payload_chunk_size, num_storage_nodes, 1, &srs)
Advz::<Bn254, Sha256>::new(erasure_code_rate, num_storage_nodes, multiplicity, srs)
.unwrap();
#[cfg(feature = "gpu-vid")]
let mut advz_gpu = AdvzGPU::<'_, Bn254, Sha256>::new(
erasure_code_rate,
num_storage_nodes,
multiplicity,
&srs,
)
.unwrap();

let payload_random = init_random_payload(1 << 25, &mut rng);

Expand All @@ -990,14 +997,16 @@ mod tests {
#[test]
fn commit_only_timer() {
// run with 'print-trace' feature to see timer output
let (payload_chunk_size, num_storage_nodes) = (256, 512);
let (erasure_code_rate, num_storage_nodes, multiplicity) = (2usize, 512usize, 1usize);
let payload_chunk_size = num_storage_nodes.div_ceil(erasure_code_rate);
let mut rng = jf_utils::test_rng();
let srs = init_srs(payload_chunk_size, &mut rng);
let srs = init_srs(payload_chunk_size * multiplicity, &mut rng);
let mut advz =
Advz::<Bn254, Sha256>::new(payload_chunk_size, num_storage_nodes, 1, srs).unwrap();
Advz::<Bn254, Sha256>::new(erasure_code_rate, num_storage_nodes, multiplicity, srs)
.unwrap();
#[cfg(feature = "gpu-vid")]
let mut advz_gpu =
AdvzGPU::<'_, Bn254, Sha256>::new(payload_chunk_size, num_storage_nodes, 1, &srs)
AdvzGPU::<'_, Bn254, Sha256>::new(erasure_code_rate, num_storage_nodes, 1, &srs)
.unwrap();

let payload_random = init_random_payload(1 << 25, &mut rng);
Expand Down Expand Up @@ -1207,10 +1216,12 @@ mod tests {
/// 1. An initialized [`Advz`] instance.
/// 2. A `Vec<u8>` filled with random bytes.
pub(super) fn advz_init() -> (Advz<Bls12_381, Sha256>, Vec<u8>) {
let (payload_chunk_size, num_storage_nodes) = (4, 6);
// run with 'print-trace' feature to see timer output
let (erasure_code_rate, num_storage_nodes) = (3usize, 6usize);
let payload_chunk_size = num_storage_nodes.div_ceil(erasure_code_rate);
let mut rng = jf_utils::test_rng();
let srs = init_srs(payload_chunk_size, &mut rng);
let advz = Advz::new(payload_chunk_size, num_storage_nodes, 1, srs).unwrap();
let advz = Advz::new(erasure_code_rate, num_storage_nodes, 1, srs).unwrap();
let bytes_random = init_random_payload(4000, &mut rng);
(advz, bytes_random)
}
Expand Down
13 changes: 4 additions & 9 deletions primitives/tests/advz.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ mod vid;
#[test]
fn round_trip() {
// play with these items
let vid_sizes = [(2, 3), (8, 11)];
let vid_sizes = [(2, 3), (3, 11)];
let payload_byte_lens = [0, 1, 2, 16, 32, 47, 48, 49, 64, 100, 400];
let mut multiplicities = [1, 2, 4, 8, 16];

Expand All @@ -34,14 +34,9 @@ fn round_trip() {
);

vid::round_trip(
|payload_chunk_size, num_storage_nodes, multiplicity| {
Advz::<Bls12_381, Sha256>::new(
payload_chunk_size,
num_storage_nodes,
multiplicity,
&srs,
)
.unwrap()
|erasure_code_rate, num_storage_nodes, multiplicity| {
Advz::<Bls12_381, Sha256>::new(erasure_code_rate, num_storage_nodes, multiplicity, &srs)
.unwrap()
},
&vid_sizes,
&multiplicities,
Expand Down
9 changes: 5 additions & 4 deletions primitives/tests/vid/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,17 @@ pub fn round_trip<V, R>(
V: VidScheme,
R: RngCore + CryptoRng,
{
for (&mult, &(payload_chunk_size, num_storage_nodes)) in
for (&mult, &(erasure_code_rate, num_storage_nodes)) in
zip(multiplicities.iter().cycle(), vid_sizes)
{
let mut vid = vid_factory(payload_chunk_size, num_storage_nodes, mult);
let mut vid = vid_factory(erasure_code_rate, num_storage_nodes, mult);

for &len in payload_byte_lens {
println!(
"m: {} n: {} mult: {} byte_len: {}",
payload_chunk_size, num_storage_nodes, mult, len
"r: {} n: {} mult: {} byte_len: {}",
erasure_code_rate, num_storage_nodes, mult, len
);
let payload_chunk_size = num_storage_nodes.div_ceil(erasure_code_rate);

let bytes_random = {
let mut bytes_random = vec![0u8; len];
Expand Down
Loading