More docs

README.md

@@ -2,7 +2,7 @@
 
 [![Actions Status](https://github.com/n0-computer/bao-tree/workflows/tests/badge.svg)](https://github.com/n0-computer/bao-tree/actions) [![docs.rs](https://docs.rs/bao-tree/badge.svg)](https://docs.rs/bao-tree) [![crates.io](https://img.shields.io/crates/v/bao-tree.svg)](https://crates.io/crates/bao-tree)
 
-The merkle tree used for blake3 verified streaming.
+The merkle tree used for BLAKE3 verified streaming.
 
 This is a slightly different take on blake3 verified streaming than the 
 [bao](https://github.com/oconnor663/bao) crate.
@@ -11,21 +11,14 @@ The network wire format for encoded data and slices is compatible with the bao
 crate, except that this crate has builtin support for *runtime* configurable chunk
 groups.
 
-The intention is also to support both sync and async en/decoding out of the box
-with maximum code sharing.
-
 It also allows encoding not just single ranges but sets of non-overlapping ranges.
 E.g. you can ask for bytes `[0..1000,5000..6000]` in a single query.
 
-It allows to define both pre- and post order outboard formats. Post order outboard
-formats have advantages for synchronizing append only files.
-
-# Example
-
-Run
+The intention is also to support both sync and async en/decoding out of the box
+with maximum code sharing.
 
-```
-cargo run --example cli
-```
+It allows to define both pre- and post order outboard formats as well as custom
+outboard formats. Post order outboard formats have advantages for synchronizing
+append only files.
 
-to see a cli that supports encoding and decoding
+For more detailed info, see the [docs](https://docs.rs/bao-tree/latest/bao_tree/)

examples/cli.rs

@@ -344,7 +344,7 @@ mod sync {
 mod fsm {
     use bao_tree::{
         io::{
-            fsm::{encode_ranges_validated, Outboard, ResponseDecoder, ResponseDecoderNext},
+            tokio::{encode_ranges_validated, Outboard, ResponseDecoder, ResponseDecoderNext},
             BaoContentItem,
         },
         BaoTree,

examples/encode_decode_async.rs

@@ -1,8 +1,8 @@
 use bao_tree::{
     io::{
-        fsm::{decode_ranges, encode_ranges_validated, valid_ranges, CreateOutboard},
         outboard::PreOrderOutboard,
         round_up_to_chunks,
+        tokio::{decode_ranges, encode_ranges_validated, valid_ranges, CreateOutboard},
     },
     BlockSize, ByteRanges, ChunkRanges,
 };

src/io/mod.rs

@@ -9,10 +9,10 @@ pub use error::*;
 use range_collections::{range_set::RangeSetRange, RangeSetRef};
 use std::future::Future;
 
-#[cfg(feature = "tokio_fsm")]
-pub mod fsm;
 pub mod outboard;
 pub mod sync;
+#[cfg(feature = "tokio_fsm")]
+pub mod tokio;
 
 /// A parent hash pair.
 #[derive(Debug)]

src/io/outboard.rs

@@ -35,7 +35,7 @@ impl crate::io::sync::Outboard for EmptyOutboard {
 }
 
 #[cfg(feature = "tokio_fsm")]
-impl crate::io::fsm::Outboard for EmptyOutboard {
+impl crate::io::tokio::Outboard for EmptyOutboard {
     fn root(&self) -> blake3::Hash {
         self.root
     }
@@ -70,7 +70,7 @@ impl crate::io::sync::OutboardMut for EmptyOutboard {
 }
 
 #[cfg(feature = "tokio_fsm")]
-impl crate::io::fsm::OutboardMut for EmptyOutboard {
+impl crate::io::tokio::OutboardMut for EmptyOutboard {
     async fn save(
         &mut self,
         node: TreeNode,
@@ -242,7 +242,7 @@ impl<T: AsRef<[u8]>> crate::io::sync::Outboard for PostOrderMemOutboard<T> {
 }
 
 #[cfg(feature = "tokio_fsm")]
-impl<T: AsRef<[u8]>> crate::io::fsm::Outboard for PostOrderMemOutboard<T> {
+impl<T: AsRef<[u8]>> crate::io::tokio::Outboard for PostOrderMemOutboard<T> {
     fn root(&self) -> blake3::Hash {
         self.root
     }
@@ -277,7 +277,7 @@ impl<T: AsMut<[u8]>> crate::io::sync::OutboardMut for PostOrderMemOutboard<T> {
 }
 
 #[cfg(feature = "tokio_fsm")]
-impl<T: AsMut<[u8]>> crate::io::fsm::OutboardMut for PostOrderMemOutboard<T> {
+impl<T: AsMut<[u8]>> crate::io::tokio::OutboardMut for PostOrderMemOutboard<T> {
     async fn save(
         &mut self,
         node: TreeNode,
@@ -432,7 +432,7 @@ impl<T: AsMut<[u8]>> crate::io::sync::OutboardMut for PreOrderMemOutboard<T> {
 }
 
 #[cfg(feature = "tokio_fsm")]
-impl<T: AsRef<[u8]>> crate::io::fsm::Outboard for PreOrderMemOutboard<T> {
+impl<T: AsRef<[u8]>> crate::io::tokio::Outboard for PreOrderMemOutboard<T> {
     fn root(&self) -> blake3::Hash {
         self.root
     }
@@ -445,7 +445,7 @@ impl<T: AsRef<[u8]>> crate::io::fsm::Outboard for PreOrderMemOutboard<T> {
 }
 
 #[cfg(feature = "tokio_fsm")]
-impl<T: AsMut<[u8]>> crate::io::fsm::OutboardMut for PreOrderMemOutboard<T> {
+impl<T: AsMut<[u8]>> crate::io::tokio::OutboardMut for PreOrderMemOutboard<T> {
     async fn save(
         &mut self,
         node: TreeNode,

src/io/sync.rs

@@ -97,9 +97,9 @@ pub trait CreateOutboard {
     /// tree and only init the data and set the root hash.
     ///
     /// So this can be used to initialize an outboard that does not have a default,
-    /// such as a file based one. It also does not require [Seek] on the data.
+    /// such as a file based one.
     ///
-    /// It will however only include data up the the current tree size.
+    /// It will only include data up the the current tree size.
     fn init_from(&mut self, data: impl ReadAt) -> io::Result<()>;
 }
 

src/io/fsm.rs → src/io/tokio.rs

@@ -127,9 +127,9 @@ pub trait CreateOutboard {
     /// tree and only init the data and set the root hash.
     ///
     /// So this can be used to initialize an outboard that does not have a default,
-    /// such as a file based one. It also does not require [AsyncSeek] on the data.
+    /// such as a file based one.
     ///
-    /// It will however only include data up the the current tree size.
+    /// It will only include data up the the current tree size.
     fn init_from(&mut self, data: impl AsyncSliceReader) -> impl Future<Output = io::Result<()>>;
 }
 
@@ -338,11 +338,11 @@ impl<R> ResponseDecoderInner<R> {
     }
 }
 
-/// Response decoder state machine, after reading the size
+/// Response decoder
 #[derive(Debug)]
 pub struct ResponseDecoder<R>(Box<ResponseDecoderInner<R>>);
 
-/// Next type for ResponseDecoderReading.
+/// Next type for ResponseDecoder.
 #[derive(Debug)]
 pub enum ResponseDecoderNext<R> {
     /// One more item, and you get back the state machine in the next state

src/lib.rs

@@ -17,7 +17,7 @@
 //! and [BlockSize] is the log base 2 of the chunk group size.
 //!
 //! All this is then used in the [io] module to implement the actual io, both
-//! [synchronous](io::sync) and [asynchronous](io::fsm).
+//! [synchronous](io::sync) and [asynchronous](io::tokio).
 //!
 //! # Basic usage
 //!
@@ -125,6 +125,72 @@
 //! # }
 //! ```
 //!
+//! # Async end to end example
+//!
+//! The async version is very similar to the sync version, except that it needs
+//! an async context. All functions that do IO are async. The file has to be
+//! an [iroh_io::File], which is just a wrapper for [std::fs::File] that implements
+//! async random access via the [AsyncSliceReader](iroh_io::AsyncSliceReader) trait.
+//!
+//! We use [futures_lite] crate, but using the normal futures crate will also work.
+//!
+//! ```no_run
+//! use bao_tree::{
+//!     io::{
+//!         outboard::PreOrderOutboard,
+//!         round_up_to_chunks,
+//!         tokio::{decode_ranges, encode_ranges_validated, valid_ranges, CreateOutboard},
+//!     },
+//!     BlockSize, ByteRanges, ChunkRanges,
+//! };
+//! use bytes::BytesMut;
+//! use futures_lite::StreamExt;
+//! use std::io;
+//!
+//! /// Use a block size of 16 KiB, a good default for most cases
+//! const BLOCK_SIZE: BlockSize = BlockSize::from_chunk_log(4);
+//!
+//!# #[tokio::main]
+//!# async fn main() -> io::Result<()> {
+//! // The file we want to serve
+//! let mut file = iroh_io::File::open("video.mp4".into()).await?;
+//! // Create an outboard for the file, using the current size
+//! let mut ob = PreOrderOutboard::<BytesMut>::create(&mut file, BLOCK_SIZE).await?;
+//! // Encode the first 100000 bytes of the file
+//! let ranges = ByteRanges::from(0..100000);
+//! let ranges = round_up_to_chunks(&ranges);
+//! // Stream of data to client. Needs to implement `io::Write`. We just use a vec here.
+//! let mut to_client = BytesMut::new();
+//! encode_ranges_validated(file, &mut ob, &ranges, &mut to_client).await?;
+//!
+//! // Stream of data from client. Needs to implement `io::Read`. We just wrap the vec in a cursor.
+//! let from_server = io::Cursor::new(to_client);
+//! let root = ob.root;
+//! let tree = ob.tree;
+//!
+//! // Decode the encoded data into a file
+//! let mut decoded = iroh_io::File::open("copy.mp4".into()).await?;
+//! let mut ob = PreOrderOutboard {
+//!     tree,
+//!     root,
+//!     data: BytesMut::new(),
+//! };
+//! decode_ranges(from_server, ranges, &mut decoded, &mut ob).await?;
+//!
+//! // the first 100000 bytes of the file should now be in `decoded`
+//! // in addition, the required part of the tree to validate that the data is
+//! // correct are in `ob.data`
+//!
+//! // Print the valid ranges of the file
+//! let ranges = ChunkRanges::all();
+//! let mut stream = valid_ranges(&mut ob, &mut decoded, &ranges);
+//! while let Some(range) = stream.next().await {
+//!     println!("{:?}", range);
+//! }
+//!# Ok(())
+//!# }
+//! ```
+//!
 //! # Compatibility with the [bao crate](https://crates.io/crates/bao)
 //!
 //! This crate will be compatible with the bao crate, provided you do the

src/tests.rs

@@ -129,7 +129,7 @@ fn bao_tree_decode_slice_iter_impl(data: Vec<u8>, range: Range<u64>) {
 mod fsm_tests {
 
     use super::*;
-    use crate::{io::fsm::*, rec::make_test_data};
+    use crate::{io::tokio::*, rec::make_test_data};
 
     /// range is a range of chunks. Just using u64 for convenience in tests
     async fn bao_tree_decode_slice_fsm_impl(data: Vec<u8>, range: Range<u64>) {
@@ -983,7 +983,7 @@ proptest! {
         let mut expected_encoded = Vec::new();
         let outboard = PostOrderMemOutboard::create(&data, block_size);
         let data: Bytes = data.into();
-        tokio::runtime::Runtime::new().unwrap().block_on(crate::io::fsm::encode_ranges_validated(
+        tokio::runtime::Runtime::new().unwrap().block_on(crate::io::tokio::encode_ranges_validated(
             data,
             outboard,
             &ranges,

src/tests2.rs

@@ -22,7 +22,9 @@ use crate::rec::{
 use crate::{assert_tuple_eq, prop_assert_tuple_eq, ChunkRanges, ChunkRangesRef};
 use crate::{
     blake3, hash_subtree,
-    io::{fsm::ResponseDecoderNext, outboard::PostOrderMemOutboard, sync::Outboard, Leaf, Parent},
+    io::{
+        outboard::PostOrderMemOutboard, sync::Outboard, tokio::ResponseDecoderNext, Leaf, Parent,
+    },
     iter::{BaoChunk, PreOrderPartialChunkIterRef, ResponseIterRef},
     rec::{encode_selected_rec, select_nodes_rec},
     BaoTree, BlockSize, ChunkNum, TreeNode,
@@ -159,7 +161,7 @@ fn outboard_test_sync(data: &[u8], outboard: impl crate::io::sync::Outboard) {
 }
 
 /// Brute force test for an outboard that just computes the expected hash for each pair
-async fn outboard_test_fsm(data: &[u8], mut outboard: impl crate::io::fsm::Outboard) {
+async fn outboard_test_fsm(data: &[u8], mut outboard: impl crate::io::tokio::Outboard) {
     let tree = outboard.tree();
     let nodes = tree
         .pre_order_nodes_iter()
@@ -250,10 +252,10 @@ mod validate {
     }
 
     /// range is a range of chunks. Just using u64 for convenience in tests
-    fn valid_ranges_fsm(outboard: impl crate::io::fsm::Outboard, data: Bytes) -> ChunkRanges {
+    fn valid_ranges_fsm(outboard: impl crate::io::tokio::Outboard, data: Bytes) -> ChunkRanges {
         run_blocking(async move {
             let ranges = ChunkRanges::all();
-            let mut stream = crate::io::fsm::valid_ranges(outboard, data, &ranges);
+            let mut stream = crate::io::tokio::valid_ranges(outboard, data, &ranges);
             let mut res = ChunkRanges::empty();
             while let Some(item) = stream.next().await {
                 let item = item?;
@@ -280,7 +282,7 @@ mod validate {
     fn valid_outboard_ranges_fsm(outboard: &mut PostOrderMemOutboard) -> ChunkRanges {
         run_blocking(async move {
             let ranges = ChunkRanges::all();
-            let mut stream = crate::io::fsm::valid_outboard_ranges(outboard, &ranges);
+            let mut stream = crate::io::tokio::valid_outboard_ranges(outboard, &ranges);
             let mut res = ChunkRanges::empty();
             while let Some(item) = stream.next().await {
                 let item = item?;
@@ -504,7 +506,7 @@ async fn encode_decode_full_fsm_impl(
     let mut outboard = outboard;
     let ranges = ChunkRanges::all();
     let mut encoded = Vec::new();
-    crate::io::fsm::encode_ranges_validated(
+    crate::io::tokio::encode_ranges_validated(
         Bytes::from(data.clone()),
         &mut outboard,
         &ranges,
@@ -526,7 +528,7 @@ async fn encode_decode_full_fsm_impl(
         }
     };
     let mut decoded = BytesMut::new();
-    crate::io::fsm::decode_ranges(read_encoded, ranges, &mut decoded, &mut ob_res)
+    crate::io::tokio::decode_ranges(read_encoded, ranges, &mut decoded, &mut ob_res)
         .await
         .unwrap();
     ((data, outboard), (decoded.to_vec(), ob_res))
@@ -580,7 +582,7 @@ async fn encode_decode_partial_fsm_impl(
     let size = outboard.tree.size;
     let mut encoded = Vec::new();
     let mut outboard = outboard;
-    crate::io::fsm::encode_ranges_validated(
+    crate::io::tokio::encode_ranges_validated(
         Bytes::from(data.to_vec()),
         &mut outboard,
         &ranges,
@@ -590,7 +592,7 @@ async fn encode_decode_partial_fsm_impl(
     .unwrap();
     let expected_data = data;
     let encoded_read = std::io::Cursor::new(encoded);
-    let mut reading = crate::io::fsm::ResponseDecoder::new(
+    let mut reading = crate::io::tokio::ResponseDecoder::new(
         outboard.root,
         ranges,
         BaoTree::new(size, outboard.tree.block_size),