Skip to content

Commit

Permalink
indexer: trim unused columns from tx table (#13062)
Browse files Browse the repository at this point in the history 
## Description 

Trim columns off tx table to bend the curve of storage consumption
growth

## Test Plan 
- cargo run to make sure that DB can still be populated properly
- run local RPC server to make sure proper tx block can be returned

```
curl --location --request POST http://127.0.0.1:3030 --header 'Content-Type: application/json' --data-raw '{
    "jsonrpc": "2.0",
    "id": 1,
    "method": "sui_getTransactionBlock",
    "params": [
        "2JxYGfYyqvUCJLtkqF71oVFQVk7CasppV6BswFW8J2vs",
        {
            "showInput": true
        }
    ]
}'
{"jsonrpc":"2.0","result":{"digest":"2JxYGfYyqvUCJLtkqF71oVFQVk7CasppV6BswFW8J2vs","transaction":{"data":{"messageVersion":"v1","transaction":{"kind":"ConsensusCommitPrologue","epoch":"0","round":"2","commit_timestamp_ms":"1681392093366"},"sender":"0x0000000000000000000000000000000000000000000000000000000000000000","gasData":{"payment":[{"objectId":"0x0000000000000000000000000000000000000000000000000000000000000000","version":0,"digest":"11111111111111111111111111111111"}],"owner":"0x0000000000000000000000000000000000000000000000000000000000000000","price":"1","budget":"0"}},"txSignatures":["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="]},"timestampMs":"1681392093366","checkpoint":"1"},"id":1}%      

curl --location --request POST https://explorer-rpc.mainnet.sui.io/ --header 'Content-Type: application/json' --data-raw '{
    "jsonrpc": "2.0",
    "id": 1,
    "method": "sui_getTransactionBlock",
    "params": [
        "2JxYGfYyqvUCJLtkqF71oVFQVk7CasppV6BswFW8J2vs",
        {
            "showInput": true
        }
    ]
}'
{"jsonrpc":"2.0","result":{"digest":"2JxYGfYyqvUCJLtkqF71oVFQVk7CasppV6BswFW8J2vs","transaction":{"data":{"messageVersion":"v1","transaction":{"kind":"ConsensusCommitPrologue","epoch":"0","round":"2","commit_timestamp_ms":"1681392093366"},"sender":"0x0000000000000000000000000000000000000000000000000000000000000000","gasData":{"payment":[{"objectId":"0x0000000000000000000000000000000000000000000000000000000000000000","version":0,"digest":"11111111111111111111111111111111"}],"owner":"0x0000000000000000000000000000000000000000000000000000000000000000","price":"1","budget":"0"}},"txSignatures":["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="]},"timestampMs":"1681392093366","checkpoint":"1"},"id":1}%           
```

---
If your changes are not user-facing and not a breaking change, you can
skip the following section. Otherwise, please indicate what changed, and
then add to the Release Notes section as highlighted during the release
process.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes
  • Loading branch information
@gegaowp
gegaowp authored Jul 20, 2023
1 parent 2f1edb7 commit b1dc358
Show file tree
Hide file tree
Showing 5 changed files with 7 additions and 103 deletions.
8 changes: 0 additions & 8 deletions crates/sui-indexer/benches/indexer_benchmark.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,18 +116,11 @@ fn create_transaction(sequence_number: i64) -> Transaction {
id: None,
transaction_digest: TransactionDigest::random().base58_encode(),
sender: SuiAddress::random_for_testing_only().to_string(),
recipients: vec![],
checkpoint_sequence_number: Some(sequence_number),
timestamp_ms: Some(Utc::now().timestamp_millis()),
transaction_kind: "test".to_string(),
transaction_count: 0,
execution_success: true,
created: vec![],
mutated: vec![],
deleted: vec![],
unwrapped: vec![],
wrapped: vec![],
move_calls: vec![],
gas_object_id: ObjectID::random().to_string(),
gas_object_sequence: 0,
gas_object_digest: ObjectDigest::random().base58_encode(),
Expand All @@ -139,7 +132,6 @@ fn create_transaction(sequence_number: i64) -> Transaction {
non_refundable_storage_fee: 0,
gas_price: 0,
raw_transaction: bcs::to_bytes(&tx).unwrap(),
transaction_content: serde_json::to_string(&tx).unwrap(),
transaction_effects_content: "".to_string(),
confirmed_local_execution: None,
}
Expand Down
10 changes: 0 additions & 10 deletions crates/sui-indexer/migrations/2022-11-18-195259_transactions/up.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,11 @@ CREATE TABLE transactions (
id BIGSERIAL PRIMARY KEY,
transaction_digest base58digest NOT NULL,
sender VARCHAR(255) NOT NULL,
recipients TEXT[] NOT NULL,
checkpoint_sequence_number BIGINT,
timestamp_ms BIGINT,
transaction_kind TEXT NOT NULL,
transaction_count BIGINT NOT NULL,
execution_success BOOLEAN NOT NULL,
-- object related
created TEXT[] NOT NULL,
mutated TEXT[] NOT NULL,
deleted TEXT[] NOT NULL,
unwrapped TEXT[] NOT NULL,
wrapped TEXT[] NOT NULL,
-- each move call is <package>::<module>::<function>
move_calls TEXT[] NOT NULL,
-- gas object related
gas_object_id address NOT NULL,
gas_object_sequence BIGINT NOT NULL,
Expand All @@ -45,7 +36,6 @@ CREATE TABLE transactions (
gas_price BIGINT NOT NULL,
-- BCS serialized SenderSignedData
raw_transaction bytea NOT NULL,
transaction_content TEXT NOT NULL,
transaction_effects_content TEXT NOT NULL,
confirmed_local_execution BOOLEAN,
UNIQUE (transaction_digest)
Expand Down
74 changes: 1 addition & 73 deletions crates/sui-indexer/src/models/transactions.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@

use diesel::prelude::*;

use sui_json_rpc_types::{
OwnedObjectRef, SuiObjectRef, SuiTransactionBlockDataAPI, SuiTransactionBlockEffectsAPI,
};
use sui_json_rpc_types::{SuiTransactionBlockDataAPI, SuiTransactionBlockEffectsAPI};

use crate::errors::IndexerError;
use crate::schema::transactions;
Expand All @@ -18,18 +16,11 @@ pub struct Transaction {
pub id: Option<i64>,
pub transaction_digest: String,
pub sender: String,
pub recipients: Vec<Option<String>>,
pub checkpoint_sequence_number: Option<i64>,
pub timestamp_ms: Option<i64>,
pub transaction_kind: String,
pub transaction_count: i64,
pub execution_success: bool,
pub created: Vec<Option<String>>,
pub mutated: Vec<Option<String>>,
pub deleted: Vec<Option<String>>,
pub unwrapped: Vec<Option<String>>,
pub wrapped: Vec<Option<String>>,
pub move_calls: Vec<Option<String>>,
pub gas_object_id: String,
pub gas_object_sequence: i64,
pub gas_object_digest: String,
Expand All @@ -42,7 +33,6 @@ pub struct Transaction {
pub gas_price: i64,
// BCS bytes of SenderSignedData
pub raw_transaction: Vec<u8>,
pub transaction_content: String,
pub transaction_effects_content: String,
pub confirmed_local_execution: Option<bool>,
}
Expand All @@ -64,55 +54,13 @@ impl TryFrom<TemporaryTransactionBlockResponseStore> for Transaction {
checkpoint,
} = tx_resp;

let tx_json = serde_json::to_string(&transaction).map_err(|err| {
IndexerError::InsertableParsingError(format!(
"Failed converting transaction block {:?} to JSON with error: {:?}",
transaction, err
))
})?;
let tx_effect_json = serde_json::to_string(&effects).map_err(|err| {
IndexerError::InsertableParsingError(format!(
"Failed converting transaction block effects {:?} to JSON with error: {:?}",
effects.clone(),
err
))
})?;
let recipients: Vec<String> = effects
.mutated()
.iter()
.cloned()
.chain(effects.created().iter().cloned())
.chain(effects.unwrapped().iter().cloned())
.map(|owned_obj_ref| owned_obj_ref.owner.to_string())
.collect();
let created: Vec<String> = effects
.created()
.iter()
.map(owned_obj_ref_to_obj_id)
.collect();
let mutated: Vec<String> = effects
.mutated()
.iter()
.map(owned_obj_ref_to_obj_id)
.collect();
let unwrapped: Vec<String> = effects
.unwrapped()
.iter()
.map(owned_obj_ref_to_obj_id)
.collect();
let deleted: Vec<String> = effects.deleted().iter().map(obj_ref_to_obj_id).collect();
let wrapped: Vec<String> = effects.wrapped().iter().map(obj_ref_to_obj_id).collect();
let move_call_strs: Vec<String> = transaction
.data
.move_calls()
.into_iter()
.map(|move_call| {
let package = move_call.package.to_string();
let module = move_call.module.to_string();
let function = move_call.function.to_string();
format!("{}::{}::{}", package, module, function)
})
.collect();

let gas_summary = effects.gas_cost_summary();
let computation_cost = gas_summary.computation_cost;
Expand All @@ -123,18 +71,11 @@ impl TryFrom<TemporaryTransactionBlockResponseStore> for Transaction {
id: None,
transaction_digest: digest.base58_encode(),
sender: transaction.data.sender().to_string(),
recipients: vec_string_to_vec_opt(recipients),
checkpoint_sequence_number: checkpoint.map(|seq| seq as i64),
transaction_kind: transaction.data.transaction().name().to_string(),
transaction_count: transaction.data.transaction().transaction_count() as i64,
execution_success: effects.status().is_ok(),
timestamp_ms: timestamp_ms.map(|ts| ts as i64),
created: vec_string_to_vec_opt(created),
mutated: vec_string_to_vec_opt(mutated),
unwrapped: vec_string_to_vec_opt(unwrapped),
deleted: vec_string_to_vec_opt(deleted),
wrapped: vec_string_to_vec_opt(wrapped),
move_calls: vec_string_to_vec_opt(move_call_strs),
gas_object_id: effects.gas_object().reference.object_id.to_string(),
gas_object_sequence: effects.gas_object().reference.version.value() as i64,
gas_object_digest: effects.gas_object().reference.digest.base58_encode(),
Expand All @@ -148,21 +89,8 @@ impl TryFrom<TemporaryTransactionBlockResponseStore> for Transaction {
storage_rebate: storage_rebate as i64,
non_refundable_storage_fee: non_refundable_storage_fee as i64,
raw_transaction,
transaction_content: tx_json,
transaction_effects_content: tx_effect_json,
confirmed_local_execution,
})
}
}

fn owned_obj_ref_to_obj_id(owned_obj_ref: &OwnedObjectRef) -> String {
owned_obj_ref.reference.object_id.to_string()
}

fn obj_ref_to_obj_id(obj_ref: &SuiObjectRef) -> String {
obj_ref.object_id.to_string()
}

fn vec_string_to_vec_opt(v: Vec<String>) -> Vec<Option<String>> {
v.into_iter().map(Some).collect::<Vec<Option<String>>>()
}
8 changes: 0 additions & 8 deletions crates/sui-indexer/src/schema.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,18 +272,11 @@ diesel::table! {
id -> Int8,
transaction_digest -> Varchar,
sender -> Varchar,
recipients -> Array<Nullable<Text>>,
checkpoint_sequence_number -> Nullable<Int8>,
timestamp_ms -> Nullable<Int8>,
transaction_kind -> Text,
transaction_count -> Int8,
execution_success -> Bool,
created -> Array<Nullable<Text>>,
mutated -> Array<Nullable<Text>>,
deleted -> Array<Nullable<Text>>,
unwrapped -> Array<Nullable<Text>>,
wrapped -> Array<Nullable<Text>>,
move_calls -> Array<Nullable<Text>>,
gas_object_id -> Varchar,
gas_object_sequence -> Int8,
gas_object_digest -> Varchar,
Expand All @@ -295,7 +288,6 @@ diesel::table! {
non_refundable_storage_fee -> Int8,
gas_price -> Int8,
raw_transaction -> Bytea,
transaction_content -> Text,
transaction_effects_content -> Text,
confirmed_local_execution -> Nullable<Bool>,
}
Expand Down
10 changes: 6 additions & 4 deletions crates/sui-indexer/src/store/pg_indexer_store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ use sui_types::messages_checkpoint::{
CheckpointCommitment, CheckpointSequenceNumber, ECMHLiveObjectSetDigest, EndOfEpochData,
};
use sui_types::object::ObjectRead;
use sui_types::transaction::SenderSignedData;

use crate::errors::{Context, IndexerError};
use crate::metrics::IndexerMetrics;
Expand Down Expand Up @@ -485,13 +486,14 @@ impl PgIndexerStore {
tx: Transaction,
options: Option<&SuiTransactionBlockResponseOptions>,
) -> Result<SuiTransactionBlockResponse, IndexerError> {
let transaction: SuiTransactionBlock =
serde_json::from_str(&tx.transaction_content).map_err(|err| {
let sender_signed_data: SenderSignedData =
bcs::from_bytes(&tx.raw_transaction).map_err(|err| {
IndexerError::InsertableParsingError(format!(
"Failed converting transaction JSON {:?} to SuiTransactionBlock with error: {:?}",
tx.transaction_content, err
"Failed converting transaction BCS to SenderSignedData with error: {:?}",
err
))
})?;
let transaction = SuiTransactionBlock::try_from(sender_signed_data, &self.module_cache)?;
let effects: SuiTransactionBlockEffects = serde_json::from_str(&tx.transaction_effects_content).map_err(|err| {
IndexerError::InsertableParsingError(format!(
"Failed converting transaction effect JSON {:?} to SuiTransactionBlockEffects with error: {:?}",
Expand Down

1 comment on commit b1dc358

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

4 Validators 500/s Owned Transactions Benchmark Results

Benchmark Report:
+-------------+-----+-----+--------+---------------+---------------+---------------+-----------------------+----------------------------+
| duration(s) | tps | cps | error% | latency (min) | latency (p50) | latency (p99) | gas used (MIST total) | gas used/hr (MIST approx.) |
+=======================================================================================================================================+
| 60          | 595 | 595 | 0      | 35            | 8295          | 8527          | 451,947,264,000       | 27,116,835,840,000         |
Stress Performance Report:
+-----------+-----+-----+
| metric    | p50 | p99 |
+=======================+
| cpu usage | 13  | 100 |

4 Validators 500/s Shared Transactions Benchmark Results

Benchmark Report:
+-------------+-----+-----+--------+---------------+---------------+---------------+-----------------------+----------------------------+
| duration(s) | tps | cps | error% | latency (min) | latency (p50) | latency (p99) | gas used (MIST total) | gas used/hr (MIST approx.) |
+=======================================================================================================================================+
| 60          | 489 | 489 | 0      | 21            | 9127          | 13351         | 437,043,964,800       | 26,222,637,888,000         |
Stress Performance Report:
+-----------+-----+-----+
| metric    | p50 | p99 |
+=======================+
| cpu usage | 12  | 100 |

20 Validators 50/s Owned Transactions Benchmark Results

Benchmark Report:
+-------------+-----+-----+--------+---------------+---------------+---------------+-----------------------+----------------------------+
| duration(s) | tps | cps | error% | latency (min) | latency (p50) | latency (p99) | gas used (MIST total) | gas used/hr (MIST approx.) |
+=======================================================================================================================================+
| 60          | 199 | 199 | 0      | 21            | 65            | 127           | 149,552,928,000       | 8,973,175,680,000          |
Stress Performance Report:
+-----------+-----+-----+
| metric    | p50 | p99 |
+=======================+
| cpu usage | 26  | 51  |

20 Validators 50/s Shared Transactions Benchmark Results

Benchmark Report:
+-------------+-----+-----+--------+---------------+---------------+---------------+-----------------------+----------------------------+
| duration(s) | tps | cps | error% | latency (min) | latency (p50) | latency (p99) | gas used (MIST total) | gas used/hr (MIST approx.) |
+=======================================================================================================================================+
| 60          | 194 | 194 | 0      | 45            | 1409          | 2149          | 178,070,529,600       | 10,684,231,776,000         |
Stress Performance Report:
+-----------+-----+-----+
| metric    | p50 | p99 |
+=======================+
| cpu usage | 25  | 56  |

Narwhal Benchmark Results

 SUMMARY:
-----------------------------------------
 + CONFIG:
 Faults: 0 node(s)
 Committee size: 4 node(s)
 Worker(s) per node: 1 worker(s)
 Collocate primary and workers: True
 Input rate: 50,000 tx/s
 Transaction size: 512 B
 Execution time: 0 s

 Header number of batches threshold: 32 digests
 Header maximum number of batches: 1,000 digests
 Max header delay: 2,000 ms
 GC depth: 50 round(s)
 Sync retry delay: 10,000 ms
 Sync retry nodes: 3 node(s)
 batch size: 500,000 B
 Max batch delay: 200 ms
 Max concurrent requests: 500,000 

 + RESULTS:
 Batch creation avg latency: 201 ms
 Header creation avg latency: -1 ms
 	Batch to header avg latency: -1 ms
 Header to certificate avg latency: 1 ms
 	Request vote outbound avg latency: 0 ms
 Certificate commit avg latency: 718 ms

 Consensus TPS: 0 tx/s
 Consensus BPS: 0 B/s
 Consensus latency: 0 ms

 End-to-end TPS: 0 tx/s
 End-to-end BPS: 0 B/s
 End-to-end latency: 0 ms
-----------------------------------------

Please sign in to comment.