Combine small files; use stitched basis

Cargo.toml

@@ -25,6 +25,7 @@ crossterm = "0.17.5"
 derive_more = "0.99.7"
 globset = "0.4.5"
 hex = "0.4.2"
+itertools = "0.9.0"
 lazy_static = "1.4.0"
 rayon = "1.3.0"
 regex = "1.3.9"

NEWS.md

@@ -1,5 +1,23 @@
 # Conserve release history
 
+## v0.6.9 NOT RELEASED YET
+
+### Features
+
+- New option `conserve size --bytes` gives output in bytes, rather than
+  megabytes.
+
+### Performance
+
+- Reading a subtree of a local source is faster, because it no longer reads and
+  discards the whole tree.
+
+- Small files (under 100kB) are combined into blocks, to allow better cross-file
+  compression and to produce fewer small file reads and writes.
+
+- Backups use a stitched index as the basis. As a result, Conserve is better
+  able to recognize unchanged files after an interrupted backup.
+
 ## v0.6.8 2020-10-16
 
 ### Features

src/archive.rs

@@ -21,7 +21,6 @@ use std::sync::Mutex;
 use rayon::prelude::*;
 use serde::{Deserialize, Serialize};
 
-use crate::backup::BackupOptions;
 use crate::blockhash::BlockHash;
 use crate::copy_tree::CopyOptions;
 use crate::errors::Error;
@@ -115,27 +114,9 @@ impl Archive {
         })
     }
 
-    /// Backup a source directory into a new band in the archive.
-    ///
-    /// Returns statistics about what was copied.
-    pub fn backup(&self, source_path: &Path, options: &BackupOptions) -> Result<CopyStats> {
-        let live_tree = LiveTree::open(source_path)?.with_excludes(options.excludes.clone());
-        let writer = BackupWriter::begin(self)?;
-        copy_tree(
-            &live_tree,
-            writer,
-            &CopyOptions {
-                print_filenames: options.print_filenames,
-                measure_first: false,
-                ..CopyOptions::default()
-            },
-        )
-    }
-
     /// Restore a selected version, or by default the latest, to a destination directory.
     pub fn restore(&self, destination_path: &Path, options: &RestoreOptions) -> Result<CopyStats> {
         let st = self.open_stored_tree(options.band_selection.clone())?;
-        let st = st.with_excludes(options.excludes.clone());
         let rt = if options.overwrite {
             RestoreTree::create_overwrite(destination_path)
         } else {
@@ -144,6 +125,7 @@ impl Archive {
         let opts = CopyOptions {
             print_filenames: options.print_filenames,
             only_subtree: options.only_subtree.clone(),
+            excludes: options.excludes.clone(),
             ..CopyOptions::default()
         };
         copy_tree(&st, rt, &opts)
@@ -255,7 +237,7 @@ impl Archive {
             .enumerate()
             .inspect(move |(i, _)| progress_bar.set_fraction(*i, num_bands))
             .map(move |(_i, band_id)| Band::open(&archive, &band_id).expect("Failed to open band"))
-            .flat_map(|band| band.iter_entries().expect("Failed to iter entries"))
+            .flat_map(|band| band.iter_entries())
             .flat_map(|entry| entry.addrs)
             .map(|addr| addr.hash))
     }
@@ -385,7 +367,7 @@ impl Archive {
                 }
                 stats
             })
-            .reduce(|| ValidateStats::default(), |a, b| a + b);
+            .reduce(ValidateStats::default, |a, b| a + b);
 
         Ok(stats)
     }