rust: add sans-io linear reader (#1244)

### Changelog
- Rust: Added a `sans_io` module to allow users to read MCAP data while
remaining agnostic to I/O. Replace the internal implementation details
of both the memory-mapped I/O and tokio-async read modules with this
sans_io reader.

### Docs

See generated rust docs.

### Description

As we need more functionality from the async MCAP reader, I feel less
comfortable adding those and not also ensuring they're available for the
memory-mapped reader.

This PR strips out all of that code and moves it into a
[sans-io](https://sans-io.readthedocs.io/) reader module. The sans-io
reader is a state machine with no dependency on any particular I/O
framework.
<!-- In addition to unit tests, describe any manual testing you did to
validate this change. -->

<table><tr><th>Before</th><th>After</th></tr><tr><td>

<!--before content goes here-->

</td><td>

<!--after content goes here-->

</td></tr></table>

<!-- If necessary, link relevant Linear or Github issues. Use `Fixes:
foxglove/repo#1234` to auto-close the Github issue or Fixes: FG-### for
Linear isses. -->

---------

Co-authored-by: Jacob Bandes-Storch <[email protected]>

.github/workflows/ci.yml

@@ -504,6 +504,7 @@ jobs:
       - run: cargo test --all-features
       - run: cargo build --all-features --target wasm32-unknown-unknown
       - run: cargo check --all-features --target wasm32-unknown-unknown
+      - run: cargo publish --dry-run
       - name: "publish to crates.io"
         if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/releases/rust/v')
         run: cargo publish --token ${{ secrets.RUST_CRATES_IO_TOKEN }}

rust/Cargo.toml

@@ -7,7 +7,7 @@ categories = [ "science::robotics", "compression" ]
 repository = "https://github.com/foxglove/mcap"
 documentation = "https://docs.rs/mcap"
 readme = "README.md"
-version = "0.10.0"
+version = "0.11.0"
 edition = "2021"
 license = "MIT"
 
@@ -23,7 +23,6 @@ num_cpus = "1.13"
 paste = "1.0"
 thiserror = "1.0"
 lz4 = { version = "1.27", optional = true }
-async-compression = { version = "*", features = ["tokio"], optional = true }
 tokio = { version = "1", features = ["io-util"] , optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
@@ -34,9 +33,9 @@ zstd = { version = "0.11", features = ["zstdmt"], optional = true }
 
 [features]
 default = ["zstd", "lz4"]
-zstd = ["dep:zstd", "async-compression/zstd"]
+zstd = ["dep:zstd"]
 lz4 = ["dep:lz4"]
-tokio = ["dep:async-compression", "dep:tokio"]
+tokio = [ "dep:tokio"]
 
 [dev-dependencies]
 anyhow = "1"

rust/src/io_utils.rs

@@ -2,45 +2,6 @@ use std::io::{self, prelude::*};
 
 use crc32fast::Hasher;
 
-/// Counts how many bytes have been read and calculates a running CRC32
-pub struct CountingCrcReader<R> {
-    inner: R,
-    hasher: Hasher,
-    count: u64,
-}
-
-impl<R: Read> CountingCrcReader<R> {
-    /// Creates a new `CountingCrcReader` with the given reader.
-    ///
-    /// This is not used when both `lz4` and `zstd` features are disabled.
-    #[allow(dead_code)]
-    pub fn new(inner: R) -> Self {
-        Self {
-            inner,
-            hasher: Hasher::new(),
-            count: 0,
-        }
-    }
-
-    pub fn position(&self) -> u64 {
-        self.count
-    }
-
-    /// Consumes the reader and returns the checksum
-    pub fn finalize(self) -> u32 {
-        self.hasher.finalize()
-    }
-}
-
-impl<R: Read> Read for CountingCrcReader<R> {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        let res = self.inner.read(buf)?;
-        self.count += res as u64;
-        self.hasher.update(&buf[..res]);
-        Ok(res)
-    }
-}
-
 pub struct CountingCrcWriter<W> {
     inner: W,
     hasher: Hasher,

rust/src/lib.rs

@@ -80,6 +80,7 @@ pub mod tokio;
 pub mod write;
 
 mod io_utils;
+pub mod sans_io;
 
 use std::{borrow::Cow, collections::BTreeMap, fmt, sync::Arc};
 
@@ -131,6 +132,10 @@ pub enum McapError {
     UnexpectedChunkRecord(u8),
     #[error("Unsupported compression format `{0}`")]
     UnsupportedCompression(String),
+    #[error("Error during decompression: `{0}`")]
+    DecompressionError(String),
+    #[error("length exceeds usize max: `{0}`")]
+    TooLong(u64),
 }
 
 pub type McapResult<T> = Result<T, McapError>;