added partial

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "quicksync"
-version = "0.1.16"
+version = "0.2.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -13,11 +13,13 @@ duration-string = "0.4.0"
 md5 = "0.7.0"
 regex = "1.11.0"
 reqwest = { version = "0.12.8", features = ["json", "stream", "blocking"] }
-rusqlite = { version = "0.32.1", features = ["bundled"] }
+rusqlite = { version = "0.32.1", features = ["bundled", "backup"] }
 serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "1.0.128"
 url = "2.5.0"
 zstd = "0.13.0"
+hex = "0.4"
+parse-display = "0.10.0"
 
 [dev-dependencies]
 mockito = "1.5.0"

README.md

@@ -4,7 +4,7 @@ When a new node joins the Spacemesh network, it must first get up to speed with
 
 With Quicksync, instead of performing all of the syncing actions as stated above and calculating the network state from genesis, one just needs to download the current state from a trusted peer like the Spacemesh dev team or some other node. While this runs contrary to the web3 philosophy of "Don't trust, verify", we believe that this could be a choice some smeshers may be interested in given the high rate of trouble with syncing. Moreover, nothing precludes a smesher from verifying this state in the background once it is downloaded.
 
-The state (also called an archive) that is downloaded is in the form of a state.sql file and can either be downloaded automatically using Smapp, or manually by using the `quicksync-rs` utility. 
+The state (also called an archive) that is downloaded is in the form of a state.sql file and can either be downloaded automatically using Smapp, or manually by using the `quicksync-rs` utility.
 
 Instructions for using `quicksync-rs` to download the latest state are given below. Note that if you use the latest version of Smapp, it will automatically offer to use quicksync to fetch the latest state.
 
@@ -56,12 +56,23 @@ Listed below are the exit codes and what they mean:
 - `7` - Invalid checksum of archive.
 - `8` - Cannot validate archive checksum.
 
+
+# Partial quicksync
+
+It is also possible to download and apply delta-based quicksync. Assuming that the `state.sql` is already present, it's worth considering applying only deltas on top of that.
+Please note that syncing large portions will be faster with full quicksync, but if you are already synced and just need to catch up with the latest state, partial quicksync is the way to go.
+
+Partial quicksync works by checking the latest verified layer in the database and then downloading small files (usually about 50MB but up to 200MB) and applying them on top of the existing `state.sql`. Each batch can be interrupted.
+
+Restoring the same batch twice is considered a no-op and will not affect the database.
+
 ## Commands
 
 The list of available commands for the `quicksync` utility is presented below. Note that these commands are for Linux. Simply, Change `./quicksync` to `.\quicksync.exe` For the Windows commands.
 
 - `./quicksync download`: Downloads the latest `state.sql` file.
 - `./quicksync check`: Checks if the current `state.sql` is up to date.
 - `./quicksync help`: Displays all operations that `quicksync` can perform.
+- `./quicksync partial`: Allows to work with delta based quicksync.
 - `./quicksync --version`: Displays the quicksync version.
 - `cargo run -- help`: Displays helpful commands for running the package. Relevant for developers.

src/main.rs

@@ -12,18 +12,20 @@ mod download;
 mod eta;
 mod go_spacemesh;
 mod parsers;
+mod partial_quicksync;
 mod read_error_response;
 mod reader_with_bytes;
 mod sql;
 mod unpack;
 mod user_agent;
 mod utils;
 
-use anyhow::Context;
+use anyhow::{anyhow, Context};
 use checksum::*;
 use download::download_with_retries;
 use go_spacemesh::get_version;
 use parsers::*;
+use partial_quicksync::partial_restore;
 use sql::get_last_layer_from_db;
 use utils::*;
 
@@ -79,6 +81,22 @@ enum Commands {
     #[clap(short = 'r', long, default_value = "10")]
     max_retries: u32,
   },
+  /// Uses partial recovery quicksync method
+  Partial {
+    /// Path to the node state.sql
+    #[clap(short = 's', long)]
+    state_sql: PathBuf,
+    /// Number of layers present in the DB that are not trusted to be fully synced.
+    /// These layers will also be synced.
+    #[clap(long, default_value_t = 10)]
+    untrusted_layers: u32,
+    /// Jump-back to recover earlier than latest layer. It will jump back one row in recovery metadata
+    #[clap(short = 'j', long, default_value_t = 0)]
+    jump_back: usize,
+    /// URL to download parts from
+    #[clap(short = 'u', long, default_value = partial_quicksync::DEFAULT_BASE_URL)]
+    base_url: String,
+  },
 }
 
 fn go_spacemesh_default_path() -> &'static str {
@@ -311,5 +329,27 @@ fn main() -> anyhow::Result<()> {
 
       Ok(())
     }
+    Commands::Partial {
+      state_sql,
+      untrusted_layers,
+      jump_back,
+      base_url,
+    } => {
+      let state_sql_path = resolve_path(&state_sql).context("resolving state.sql path")?;
+      if !state_sql_path
+        .try_exists()
+        .context("checking if state file exists")?
+      {
+        return Err(anyhow!("state file not found: {:?}", state_sql_path));
+      }
+      let download_path = resolve_path(Path::new(".")).unwrap();
+      partial_restore(
+        &base_url,
+        &state_sql_path,
+        &download_path,
+        untrusted_layers,
+        jump_back,
+      )
+    }
   }
 }