From 905e65d278da78a184e2702da25b55f27ad78f23 Mon Sep 17 00:00:00 2001 From: Jay Date: Tue, 8 Jun 2021 17:56:15 +0800 Subject: [PATCH] update docs (#442) - mention async ready - fix doc warnings Signed-off-by: Jay Lee --- src/confchange/changer.rs | 2 +- src/lib.rs | 27 ++++++++++++++++++++++----- src/log_unstable.rs | 4 ++-- src/quorum/majority.rs | 4 ++-- src/raw_node.rs | 25 +++++++++++++------------ src/tracker.rs | 4 ++-- 6 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/confchange/changer.rs b/src/confchange/changer.rs index e584bf032..32e231708 100644 --- a/src/confchange/changer.rs +++ b/src/confchange/changer.rs @@ -129,7 +129,7 @@ impl Changer<'_> { } /// Carries out a series of configuration changes that (in aggregate) mutates the - /// incoming majority config Voters[0] by at most one. This method will return an + /// incoming majority config `Voters[0]` by at most one. This method will return an /// error if that is not the case, if the resulting quorum is zero, or if the /// configuration is in a joint state (i.e. if there is an outgoing configuration). pub fn simple(&mut self, ccs: &[ConfChangeSingle]) -> Result<(Configuration, MapChange)> { diff --git a/src/lib.rs b/src/lib.rs index 5d8ed9789..bd8b49fbe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,10 +18,8 @@ ## Creating a Raft node -You can use [`RawNode::new`](raw_node/struct.RawNode.html#method.new) to create the Raft node. To -create the Raft node, you need to provide a [`Storage`](storage/trait.Storage.html) component, and -a [`Config`](struct.Config.html) to the [`RawNode::new`](raw_node/struct.RawNode.html#method.new) -function. +You can use [`RawNode::new`] to create the Raft node. To create the Raft node, you need to +provide a [`Storage`] component, and a [`Config`] to the [`RawNode::new`] function. ```rust use raft::{ @@ -51,7 +49,7 @@ let mut node = RawNode::new(&config, storage, &logger).unwrap(); Use a timer to tick the Raft node at regular intervals. See the following example using Rust channel `recv_timeout` to drive the Raft node at least every 100ms, calling -[`tick()`](raw_node/struct.RawNode.html#method.tick) each time. +[`tick()`](RawNode::tick) each time. ```rust # use slog::{Drain, o}; @@ -303,6 +301,14 @@ need to update the applied index and resume `apply` later: } ``` + Note, although Raft guarentees only persisted committed entries will be applied, + but it doesn't guarentee commit index is persisted before being applied. For example, + if application is restarted after applying committed entries before persisting + commit index, apply index can be larger than commit index and cause panic. To + solve the problem, persisting commit index with or before applying entries. + You can also always assign commit index to the `max(commit_index, applied_index)` + after restarting, *it may work but potential log loss may also be ignored silently*. + 4. Check whether `entries` is empty or not. If not empty, it means that there are newly added entries but have not been committed yet, we must append the entries to the Raft log: @@ -412,6 +418,17 @@ to advance the applied index inside. For more information, check out an [example](examples/single_mem_node/main.rs#L113-L179). +Sometimes it's better not to block the raft machine in IO operation, so that latency of +read/write can be more predictable and the fsync frequencey can be controlled. The crate +supports async ready to offload the IO operation to other thread. The usage is the same as +above except: +1. All writes are not required to be persisted immediately, they can be written into memory caches; +2. Persisted messages should be sent after all coresponding writes are persisted; +3. [`advance_append_async`](RawNode::advance_append_async) is used when all writes are finished + instead of `advance/advance_append`. +4. Only persisted entries can be committed and applied, so to make progress, all writes should + be persisted at some point. + ## Arbitrary Membership Changes When building a resilient, scalable distributed system there is a strong need to be able to change diff --git a/src/log_unstable.rs b/src/log_unstable.rs index c4491a976..b13593035 100644 --- a/src/log_unstable.rs +++ b/src/log_unstable.rs @@ -19,8 +19,8 @@ use crate::eraftpb::{Entry, Snapshot}; use slog::Logger; -/// The unstable.entries[i] has raft log position i+unstable.offset. -/// Note that unstable.offset may be less than the highest log +/// The `unstable.entries[i]` has raft log position `i+unstable.offset`. +/// Note that `unstable.offset` may be less than the highest log /// position in storage; this means that the next write to storage /// might need to truncate the log before persisting unstable.entries. #[derive(Debug)] diff --git a/src/quorum/majority.rs b/src/quorum/majority.rs index 927e7e375..13bcc0ed4 100644 --- a/src/quorum/majority.rs +++ b/src/quorum/majority.rs @@ -65,8 +65,8 @@ impl Configuration { /// The bool flag indicates whether the index is computed by group commit algorithm /// successfully. /// - /// Eg. If the matched indexes are [2,2,2,4,5], it will return 2. - /// If the matched indexes and groups are `[(1, 1), (2, 2), (3, 2)]`, it will return 1. + /// Eg. If the matched indexes are `[2,2,2,4,5]`, it will return `2`. + /// If the matched indexes and groups are `[(1, 1), (2, 2), (3, 2)]`, it will return `1`. pub fn committed_index(&self, use_group_commit: bool, l: &impl AckedIndexer) -> (u64, bool) { if self.voters.is_empty() { // This plays well with joint quorums which, when one half is the zero diff --git a/src/raw_node.rs b/src/raw_node.rs index c7f77dfe7..fd35cb468 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -219,7 +219,7 @@ impl Ready { /// 1. no HardState or only its commit is different from before /// 2. no Entries and Snapshot /// If it's false, an asynchronous write of HardState is permissible before calling - /// `on_persist_ready` or `advance` or its families. + /// [`RawNode::on_persist_ready`] or [`RawNode::advance`] or its families. #[inline] pub fn must_sync(&self) -> bool { self.must_sync @@ -298,7 +298,7 @@ pub struct RawNode { impl RawNode { #[allow(clippy::new_ret_no_self)] - /// Create a new RawNode given some [`Config`](../struct.Config.html). + /// Create a new RawNode given some [`Config`]. pub fn new(config: &Config, store: T, logger: &Logger) -> Result { assert_ne!(config.id, 0, "config.id must not be zero"); let r = Raft::new(config, store, logger)?; @@ -320,7 +320,7 @@ impl RawNode { Ok(rn) } - /// Create a new RawNode given some [`Config`](../struct.Config.html) and the default logger. + /// Create a new RawNode given some [`Config`] and the default logger. /// /// The default logger is an `slog` to `log` adapter. #[cfg(feature = "default-logger")] @@ -440,7 +440,7 @@ impl RawNode { /// passed back via `advance` or its families. Before that, *DO NOT* call any function like /// `step`, `propose`, `campaign` to change internal state. /// - /// `has_ready` should be called first to check if it's necessary to handle the ready. + /// [`Self::has_ready`] should be called first to check if it's necessary to handle the ready. pub fn ready(&mut self) -> Ready { let raft = &mut self.raft; @@ -578,8 +578,8 @@ impl RawNode { /// Since Ready must be persisted in order, calling this function implicitly means /// all readies with numbers smaller than this one have been persisted. /// - /// `has_ready` and `ready` should be called later to handle further updates that become - /// valid after ready being persisted. + /// [`Self::has_ready`] and [`Self::ready`] should be called later to handle further + /// updates that become valid after ready being persisted. pub fn on_persist_ready(&mut self, number: u64) { let (mut index, mut term) = (0, 0); let mut snap_index = 0; @@ -613,9 +613,10 @@ impl RawNode { /// Fully processing a ready requires to persist snapshot, entries and hard states, apply all /// committed entries, send all messages. /// - /// Returns the LightReady that contains commit index, committed entries and messages. `LightReady` + /// Returns the LightReady that contains commit index, committed entries and messages. [`LightReady`] /// contains updates that only valid after persisting last ready. It should also be fully processed. - /// Then `advance_apply` or `advance_apply_to` should be used later to update applying progress. + /// Then [`Self::advance_apply`] or [`Self::advance_apply_to`] should be used later to update applying + /// progress. pub fn advance(&mut self, rd: Ready) -> LightReady { let applied = self.commit_since_index; let light_rd = self.advance_append(rd); @@ -623,8 +624,8 @@ impl RawNode { light_rd } - /// Advances the ready without applying committed entries. `advance_apply` or `advance_apply_to` - /// should be used later to update applying progress. + /// Advances the ready without applying committed entries. [`Self::advance_apply`] or + /// [`Self::advance_apply_to`] should be used later to update applying progress. /// /// Returns the LightReady that contains commit index, committed entries and messages. /// @@ -651,8 +652,8 @@ impl RawNode { light_rd } - /// Same as `advance_append` except that it allows to only store the updates in cache. `on_persist_ready` - /// should be used later to update the persisting progress. + /// Same as [`Self::advance_append`] except that it allows to only store the updates in cache. + /// [`Self::on_persist_ready`] should be used later to update the persisting progress. /// /// Raft works on an assumption persisted updates should not be lost, which usually requires expensive /// operations like `fsync`. `advance_append_async` allows you to control the rate of such operations and diff --git a/src/tracker.rs b/src/tracker.rs index c9ec30056..2d79c03e4 100644 --- a/src/tracker.rs +++ b/src/tracker.rs @@ -289,8 +289,8 @@ impl ProgressTracker { /// Returns the maximal committed index for the cluster. The bool flag indicates whether /// the index is computed by group commit algorithm successfully. /// - /// Eg. If the matched indexes are [2,2,2,4,5], it will return 2. - /// If the matched indexes and groups are `[(1, 1), (2, 2), (3, 2)]`, it will return 1. + /// Eg. If the matched indexes are `[2,2,2,4,5]`, it will return `2`. + /// If the matched indexes and groups are `[(1, 1), (2, 2), (3, 2)]`, it will return `1`. pub fn maximal_committed_index(&mut self) -> (u64, bool) { self.conf .voters