Skip to content

Commit

Permalink
indexer restorer: simplify commands (#19552)
Browse files Browse the repository at this point in the history
## Description 

- we made `archives` and `display` buckets public, thus restorer no
longer needs gcs cred
- remove un-necessary required args / env vars so that to restore from
mainnet, only start-epoch & local snapshot download dir needs to be
specified.

## Test plan 

local run of restorer

---

## Release notes

Check each box that your changes affect. If none of the boxes relate to
your changes, release notes aren't required.

For each box you select, include information after the relevant heading
that describes the impact of your changes that a user might notice and
any actions they must take to implement updates.

- [ ] Protocol: 
- [ ] Nodes (Validators and Full nodes): 
- [ ] Indexer: 
- [ ] JSON-RPC: 
- [ ] GraphQL: 
- [ ] CLI: 
- [ ] Rust SDK:
- [ ] REST API:
  • Loading branch information
gegaowp authored Sep 25, 2024
1 parent c22271c commit 897aab2
Show file tree
Hide file tree
Showing 4 changed files with 29 additions and 22 deletions.
35 changes: 25 additions & 10 deletions crates/sui-indexer/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -251,26 +251,41 @@ pub struct UploadOptions {

#[derive(Args, Debug, Clone)]
pub struct RestoreConfig {
#[arg(long, env = "START_EPOCH", required = true)]
pub start_epoch: u64,
#[arg(long, env = "SNAPSHOT_ENDPOINT")]
pub snapshot_endpoint: String,
#[arg(long, env = "SNAPSHOT_BUCKET")]
pub snapshot_bucket: String,
#[arg(long, env = "SNAPSHOT_DOWNLOAD_DIR", required = true)]
pub snapshot_download_dir: String,

#[arg(long, env = "GCS_ARCHIVE_BUCKET")]
pub gcs_archive_bucket: String,
#[arg(long, env = "GCS_CRED_PATH")]
pub gcs_cred_path: String,
#[arg(long, env = "GCS_DISPLAY_BUCKET")]
pub gcs_display_bucket: String,
#[arg(long, env = "GCS_SNAPSHOT_DIR")]
pub gcs_snapshot_dir: String,
#[arg(long, env = "GCS_SNAPSHOT_BUCKET")]
pub gcs_snapshot_bucket: String,
#[arg(long, env = "START_EPOCH")]
pub start_epoch: u64,
#[arg(long, env = "S3_ENDPOINT")]
pub s3_endpoint: String,

#[arg(env = "OBJECT_STORE_CONCURRENT_LIMIT")]
pub object_store_concurrent_limit: usize,
#[arg(env = "OBJECT_STORE_MAX_TIMEOUT_SECS")]
pub object_store_max_timeout_secs: u64,
}

impl Default for RestoreConfig {
fn default() -> Self {
Self {
start_epoch: 0, // not used b/c it's required
snapshot_endpoint: "https://formal-snapshot.mainnet.sui.io".to_string(),
snapshot_bucket: "mysten-mainnet-formal".to_string(),
snapshot_download_dir: "".to_string(), // not used b/c it's required
gcs_archive_bucket: "mysten-mainnet-archives".to_string(),
gcs_display_bucket: "mysten-mainnet-display-table".to_string(),
object_store_concurrent_limit: 50,
object_store_max_timeout_secs: 512,
}
}
}

#[cfg(test)]
mod test {
use super::*;
Expand Down
7 changes: 2 additions & 5 deletions crates/sui-indexer/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,8 @@ async fn main() -> anyhow::Result<()> {
.await;
}
Command::Restore(restore_config) => {
let upload_options = UploadOptions {
gcs_display_bucket: Some(restore_config.gcs_display_bucket.clone()),
gcs_cred_path: Some(restore_config.gcs_snapshot_bucket.clone()),
};
let store = PgIndexerStore::new(pool, upload_options, indexer_metrics.clone());
let store =
PgIndexerStore::new(pool, UploadOptions::default(), indexer_metrics.clone());
let mut formal_restorer =
IndexerFormalSnapshotRestorer::new(store, restore_config).await?;
formal_restorer.restore().await?;
Expand Down
2 changes: 0 additions & 2 deletions crates/sui-indexer/src/restorer/archives.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,12 @@ pub struct RestoreCheckpointInfo {
}

pub async fn read_restore_checkpoint_info(
cred_path: String,
archive_bucket: Option<String>,
epoch: u64,
) -> IndexerResult<RestoreCheckpointInfo> {
let archive_store_config = ObjectStoreConfig {
object_store: Some(ObjectStoreType::GCS),
bucket: archive_bucket,
google_service_account: Some(cred_path.clone()),
object_store_connection_limit: 50,
no_sign_request: false,
..Default::default()
Expand Down
7 changes: 2 additions & 5 deletions crates/sui-indexer/src/restorer/formal_snapshot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,14 @@ impl IndexerFormalSnapshotRestorer {
) -> Result<Self, IndexerError> {
let remote_store_config = ObjectStoreConfig {
object_store: Some(ObjectStoreType::S3),
aws_endpoint: Some(restore_config.s3_endpoint.clone()),
aws_endpoint: Some(restore_config.snapshot_endpoint.clone()),
aws_virtual_hosted_style_request: true,
object_store_connection_limit: restore_config.object_store_concurrent_limit,
no_sign_request: true,
..Default::default()
};

let base_path = PathBuf::from(restore_config.gcs_snapshot_dir.clone());
let base_path = PathBuf::from(restore_config.snapshot_download_dir.clone());
let snapshot_dir = base_path.join("snapshot");
if snapshot_dir.exists() {
fs::remove_dir_all(snapshot_dir.clone()).unwrap();
Expand Down Expand Up @@ -238,14 +238,12 @@ impl IndexerFormalSnapshotRestorer {
}

async fn restore_display_table(&self) -> std::result::Result<(), anyhow::Error> {
let cred_path = self.restore_config.gcs_cred_path.clone();
let bucket = self.restore_config.gcs_display_bucket.clone();
let start_epoch = self.restore_config.start_epoch;

let remote_store_config = ObjectStoreConfig {
object_store: Some(ObjectStoreType::GCS),
bucket: Some(bucket),
google_service_account: Some(cred_path),
object_store_connection_limit: 200,
no_sign_request: false,
..Default::default()
Expand All @@ -261,7 +259,6 @@ impl IndexerFormalSnapshotRestorer {

async fn restore_cp_watermark_and_chain_id(&self) -> Result<(), IndexerError> {
let restore_checkpoint_info = read_restore_checkpoint_info(
self.restore_config.gcs_cred_path.clone(),
Some(self.restore_config.gcs_archive_bucket.clone()),
self.restore_config.start_epoch,
)
Expand Down

0 comments on commit 897aab2

Please sign in to comment.