-
Notifications
You must be signed in to change notification settings - Fork 11.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
indexer-alt: kv_protocol_configs pipeline
## Description Add a pipeline for indexing protocol configs. ## Test plan Index the first ~1M checkpoints, which should pick up the first protocol version upgrade (from 4 to 5 for epoch 10) and then query `kv_protocol_configs` results after that: ``` sui$ cargo run -p sui-indexer-alt -- \ --database-url postgres://postgres:postgrespw@localhost:5432/sui_indexer_alt \ --remote-store-url https://checkpoints.mainnet.sui.io \ --last-checkpoint 500000 ``` ``` sui_indexer_alt=# SELECT * FROM kv_protocol_configs; ```
- Loading branch information
Showing
8 changed files
with
106 additions
and
1 deletion.
There are no files selected for viewing
1 change: 1 addition & 0 deletions
1
crates/sui-indexer-alt/migrations/2024-11-01-153859_kv_protocol_configs/down.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
DROP TABLE IF EXISTS kv_protocol_configs; |
7 changes: 7 additions & 0 deletions
7
crates/sui-indexer-alt/migrations/2024-11-01-153859_kv_protocol_configs/up.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
CREATE TABLE IF NOT EXISTS kv_protocol_configs | ||
( | ||
protocol_version BIGINT NOT NULL, | ||
config_name TEXT NOT NULL, | ||
config_value TEXT, | ||
PRIMARY KEY (protocol_version, config_name) | ||
); |
68 changes: 68 additions & 0 deletions
68
crates/sui-indexer-alt/src/handlers/kv_protocol_configs.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use std::sync::Arc; | ||
|
||
use anyhow::{Context, Result}; | ||
use diesel_async::RunQueryDsl; | ||
use sui_protocol_config::ProtocolConfig; | ||
use sui_types::full_checkpoint_content::CheckpointData; | ||
|
||
use crate::{ | ||
db, | ||
models::{checkpoints::StoredGenesis, epochs::StoredProtocolConfig}, | ||
pipeline::{concurrent::Handler, Processor}, | ||
schema::kv_protocol_configs, | ||
}; | ||
|
||
pub struct KvProtocolConfigs(pub StoredGenesis); | ||
|
||
impl Processor for KvProtocolConfigs { | ||
const NAME: &'static str = "kv_protocol_configs"; | ||
type Value = StoredProtocolConfig; | ||
|
||
fn process(&self, checkpoint: &Arc<CheckpointData>) -> Result<Vec<Self::Value>> { | ||
let CheckpointData { | ||
checkpoint_summary, .. | ||
} = checkpoint.as_ref(); | ||
|
||
let protocol_version = if checkpoint_summary.sequence_number == 0 { | ||
self.0.initial_protocol_version() | ||
} else if let Some(end_of_epoch) = checkpoint_summary.end_of_epoch_data.as_ref() { | ||
end_of_epoch.next_epoch_protocol_version | ||
} else { | ||
return Ok(vec![]); | ||
}; | ||
|
||
let protocol_config = ProtocolConfig::get_for_version( | ||
protocol_version, | ||
self.0.chain().context("Failed to identify chain")?, | ||
); | ||
|
||
let protocol_version = protocol_version.as_u64() as i64; | ||
Ok(protocol_config | ||
.attr_map() | ||
.into_iter() | ||
.map(|(config_name, value)| StoredProtocolConfig { | ||
protocol_version, | ||
config_name, | ||
config_value: value.map(|v| v.to_string()), | ||
}) | ||
.collect()) | ||
} | ||
} | ||
|
||
#[async_trait::async_trait] | ||
impl Handler for KvProtocolConfigs { | ||
const MIN_EAGER_ROWS: usize = 1; | ||
const MAX_CHUNK_ROWS: usize = i16::MAX as usize / 3; | ||
const MAX_PENDING_ROWS: usize = 10000; | ||
|
||
async fn commit(values: &[Self::Value], conn: &mut db::Connection<'_>) -> Result<usize> { | ||
Ok(diesel::insert_into(kv_protocol_configs::table) | ||
.values(values) | ||
.on_conflict_do_nothing() | ||
.execute(conn) | ||
.await?) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
// Copyright (c) Mysten Labs, Inc. | ||
// SPDX-License-Identifier: Apache-2.0 | ||
|
||
use diesel::prelude::*; | ||
use sui_field_count::FieldCount; | ||
|
||
use crate::schema::kv_protocol_configs; | ||
|
||
#[derive(Insertable, Debug, Clone, FieldCount)] | ||
#[diesel(table_name = kv_protocol_configs)] | ||
pub struct StoredProtocolConfig { | ||
pub protocol_version: i64, | ||
pub config_name: String, | ||
pub config_value: Option<String>, | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters