Skip to content

Commit

Permalink
wip: hardcode known stale accounts at Eden
Browse files Browse the repository at this point in the history
  • Loading branch information
ptrus committed Jan 26, 2024
1 parent f6af89d commit 5dc52bc
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 22 deletions.
34 changes: 20 additions & 14 deletions analyzer/evmtokenbalances/evm_token_balances.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,20 @@ func NewAnalyzer(
}

type StaleTokenBalance struct {
TokenAddr string
AccountAddr string
Type common.TokenType
Balance *big.Int
TokenAddrContextIdentifier string
TokenAddrContextVersion int
TokenAddrData []byte
AccountAddrContextIdentifier string
AccountAddrContextVersion int
TokenAddr string
AccountAddr string
Type common.TokenType
Balance *big.Int
TokenAddrContextIdentifier string
TokenAddrContextVersion int
TokenAddrData []byte

// Not necessary for native tokens.
AccountAddrContextIdentifier *string
AccountAddrContextVersion *int
AccountAddrData []byte
DownloadRound uint64

DownloadRound uint64
}

func (p *processor) GetItems(ctx context.Context, limit uint64) ([]*StaleTokenBalance, error) {
Expand Down Expand Up @@ -160,10 +163,6 @@ func (p *processor) GetItems(ctx context.Context, limit uint64) ([]*StaleTokenBa
}

func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch, staleTokenBalance *StaleTokenBalance) error {
accountEthAddr, err := client.EVMEthAddrFromPreimage(staleTokenBalance.AccountAddrContextIdentifier, staleTokenBalance.AccountAddrContextVersion, staleTokenBalance.AccountAddrData)
if err != nil {
return fmt.Errorf("account address: %w", err)
}
switch staleTokenBalance.Type {
case common.TokenTypeUnsupported:
// Do nothing; we'll just mark this token as processed so we remove it from the queue.
Expand Down Expand Up @@ -204,6 +203,13 @@ func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch,
if err != nil {
return fmt.Errorf("token address: %w", err)
}
if staleTokenBalance.AccountAddrContextIdentifier == nil || staleTokenBalance.AccountAddrContextVersion == nil || staleTokenBalance.AccountAddrData == nil {
return fmt.Errorf("account address: missing preimage for: '%s' (token address: '%s')", staleTokenBalance.AccountAddr, staleTokenBalance.TokenAddr)
}
accountEthAddr, err := client.EVMEthAddrFromPreimage(*staleTokenBalance.AccountAddrContextIdentifier, *staleTokenBalance.AccountAddrContextVersion, staleTokenBalance.AccountAddrData)
if err != nil {
return fmt.Errorf("account address: %w", err)
}
balanceData, err := evm.EVMDownloadTokenBalance(
ctx,
p.logger,
Expand Down
8 changes: 4 additions & 4 deletions analyzer/queries/queries.go
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,7 @@ var (
($1, $2, $3, $4)
ON CONFLICT (runtime, token_address, account_address) DO UPDATE
SET
last_mutate_round = excluded.last_mutate_round`
last_mutate_round = GREATEST(excluded.last_mutate_round, analysis.evm_token_balances.last_mutate_round)`

RuntimeFastSyncEVMTokenBalanceAnalysisMutateRoundInsert = `
INSERT INTO todo_updates.evm_token_balances
Expand Down Expand Up @@ -900,21 +900,21 @@ var (

RuntimeEvmVerifiedContractTxs = `
WITH abi_contracts AS (
SELECT
SELECT
runtime,
contract_address AS addr,
abi
FROM chain.evm_contracts
WHERE
runtime = $1 AND abi IS NOT NULL
)
SELECT
SELECT
abi_contracts.addr,
abi_contracts.abi,
txs.tx_hash,
txs.body->'data',
txs.error_message_raw
FROM abi_contracts
FROM abi_contracts
JOIN chain.runtime_transactions as txs ON
txs.runtime = abi_contracts.runtime AND
txs.to = abi_contracts.addr AND
Expand Down
10 changes: 10 additions & 0 deletions analyzer/runtime/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/oasisprotocol/nexus/analyzer/block"
"github.com/oasisprotocol/nexus/analyzer/queries"
evm "github.com/oasisprotocol/nexus/analyzer/runtime/evm"
"github.com/oasisprotocol/nexus/analyzer/runtime/static"
uncategorized "github.com/oasisprotocol/nexus/analyzer/uncategorized"
apiTypes "github.com/oasisprotocol/nexus/api/v1/types"
"github.com/oasisprotocol/nexus/common"
Expand All @@ -25,6 +26,7 @@ import (

// processor is the block processor for runtimes.
type processor struct {
chain common.ChainName
runtime common.Runtime
runtimeMetadata *sdkConfig.ParaTime
mode analyzer.BlockAnalysisMode
Expand All @@ -38,6 +40,7 @@ var _ block.BlockProcessor = (*processor)(nil)

// NewRuntimeAnalyzer returns a new runtime analyzer for a runtime.
func NewRuntimeAnalyzer(
chain common.ChainName,
runtime common.Runtime,
runtimeMetadata *sdkConfig.ParaTime,
blockRange config.BlockRange,
Expand All @@ -49,6 +52,7 @@ func NewRuntimeAnalyzer(
) (analyzer.Analyzer, error) {
// Initialize runtime block processor.
processor := &processor{
chain: chain,
runtime: runtime,
runtimeMetadata: runtimeMetadata,
mode: mode,
Expand Down Expand Up @@ -144,6 +148,12 @@ func (m *processor) FinalizeFastSync(ctx context.Context, lastFastSyncHeight int
return err
}

// Queue to refetch the balances of known-to-be-stale accounts.
m.logger.Info("queueing known-to-be-stale (as of roughly Eden genesis heights) EVM accounts for re-query")
if err := static.QueueEVMKnownStaleAccounts(batch, m.chain, m.runtime); err != nil {
return fmt.Errorf("queue eden accounts: %w", err)
}

// Update tables where fast-sync was not updating their columns in-place, and was instead writing
// a log of updates to a temporary table.
m.logger.Info("updating last_mutate_round for EVM token balances")
Expand Down
81 changes: 81 additions & 0 deletions analyzer/runtime/static/stale_accounts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package static

import (
"github.com/oasisprotocol/nexus/analyzer/queries"
"github.com/oasisprotocol/nexus/analyzer/runtime/evm"
apiTypes "github.com/oasisprotocol/nexus/api/v1/types"
"github.com/oasisprotocol/nexus/common"
"github.com/oasisprotocol/nexus/storage"
)

func staleAccountsList(chainName common.ChainName, runtime common.Runtime) (uint64, []string) {
// TODO: probably use embed.FS to keep these long list in files, but
// also include them in the binary.
// Also name the files something like: {chainName}_{runtime}_{height}_stale_accounts.txt
// So that the switch below is not needed.
switch chainName {
case common.ChainNameMainnet:
switch runtime {
case common.RuntimeEmerald:
return 7875129, []string{}
case common.RuntimeSapphire:
// For Sapphire the number of all account at Eden genesis is ~2k, so we could include all.
return 1357486, []string{}
default:
// No known accounts for this runtime.
return 0, nil
}
case common.ChainNameTestnet:
switch runtime {
case common.RuntimeEmerald:
return 2627790, []string{}
case common.RuntimeSapphire:
// For Sapphire the number of all account at Eden genesis is ~2k, so we could include all.
return 2995927, []string{}
default:
// No known accounts for this runtime.
return 0, nil
}
default:
// No known accounts for this chain.
return 0, nil
}
}

// QueueEVMKnownStaleAccounts queues (known-to-be stale) account lists at specific heights for native token balance update.
//
// At the moment, these lists were manually obtained at Eden genesis heights, to mitigate the following issues:
// - runtimes used to not emit Transfer events for FEE payments
// - therefore we are unable to dead-reckon these balances, as fee payment was missed
//
// - oasis-sdk runtimes used to not emit Transfer events for Reward and Fee disbursements
// - therefore there exist accounts that have received native balance, but if they never submitted any transactions the Nexus is unaware of them
//
// These were fixed sometime during Damask, but for simplicity we take the list of accounts at Eden genesis, which at worst contains additional
// accounts that were not effected by the above issues.
//
// This list also helps partially mitigating the issue of accounts that have only ever interacted within internal EVM transactions (e.g. transfers within evm.Call).
// This is an ongoing issue where Nexus cannot known about the existence of these accounts as Nexus is unable to simulate EVM transactions.
// With these lists we at least know about all such accounts at Eden genesis time.
func QueueEVMKnownStaleAccounts(batch *storage.QueryBatch, chainName common.ChainName, rt common.Runtime) error {
round, accounts := staleAccountsList(chainName, rt)
if len(accounts) == 0 {
return nil
}

for _, account := range accounts {
// Depending on fast-sync configuration different scenarios can happen:
// - The analyzer has already indexed past the height of these accounts:
// - The account might have been mutated recently, therefore the below will be a no-op.
// - The account might have never been mutated (or not in a long time), below will queue it to be required.
// - The analyzer is behind the height of these accounts:
// - The account will have `last_mutated_round` set in future and it will periodically be required
// until the round is reached so that `download_round` >= `last_mutated_round`.
batch.Queue(
queries.RuntimeEVMTokenBalanceAnalysisMutateRoundUpsert,
rt, evm.NativeRuntimeTokenAddress, apiTypes.Address(account), round,
)
}

return nil
}
8 changes: 4 additions & 4 deletions cmd/analyzer/analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ func NewService(cfg *config.AnalysisConfig) (*Service, error) { //nolint:gocyclo
if err1 != nil {
return nil, err1
}
return runtime.NewRuntimeAnalyzer(runtimeName, sdkPT, *fastRange, config.BatchSize, analyzer.FastSyncMode, sourceClient, dbClient, logger)
return runtime.NewRuntimeAnalyzer(cfg.Source.ChainName, runtimeName, sdkPT, *fastRange, config.BatchSize, analyzer.FastSyncMode, sourceClient, dbClient, logger)
})
}
}
Expand Down Expand Up @@ -335,7 +335,7 @@ func NewService(cfg *config.AnalysisConfig) (*Service, error) { //nolint:gocyclo
if err1 != nil {
return nil, err1
}
return runtime.NewRuntimeAnalyzer(common.RuntimeEmerald, runtimeMetadata, cfg.Analyzers.Emerald.SlowSyncRange(), cfg.Analyzers.Emerald.BatchSize, analyzer.SlowSyncMode, sourceClient, dbClient, logger)
return runtime.NewRuntimeAnalyzer(cfg.Source.ChainName, common.RuntimeEmerald, runtimeMetadata, cfg.Analyzers.Emerald.SlowSyncRange(), cfg.Analyzers.Emerald.BatchSize, analyzer.SlowSyncMode, sourceClient, dbClient, logger)
})
}
if cfg.Analyzers.Sapphire != nil {
Expand All @@ -345,7 +345,7 @@ func NewService(cfg *config.AnalysisConfig) (*Service, error) { //nolint:gocyclo
if err1 != nil {
return nil, err1
}
return runtime.NewRuntimeAnalyzer(common.RuntimeSapphire, runtimeMetadata, cfg.Analyzers.Sapphire.SlowSyncRange(), cfg.Analyzers.Sapphire.BatchSize, analyzer.SlowSyncMode, sourceClient, dbClient, logger)
return runtime.NewRuntimeAnalyzer(cfg.Source.ChainName, common.RuntimeSapphire, runtimeMetadata, cfg.Analyzers.Sapphire.SlowSyncRange(), cfg.Analyzers.Sapphire.BatchSize, analyzer.SlowSyncMode, sourceClient, dbClient, logger)
})
}
if cfg.Analyzers.Cipher != nil {
Expand All @@ -355,7 +355,7 @@ func NewService(cfg *config.AnalysisConfig) (*Service, error) { //nolint:gocyclo
if err1 != nil {
return nil, err1
}
return runtime.NewRuntimeAnalyzer(common.RuntimeCipher, runtimeMetadata, cfg.Analyzers.Cipher.SlowSyncRange(), cfg.Analyzers.Cipher.BatchSize, analyzer.SlowSyncMode, sourceClient, dbClient, logger)
return runtime.NewRuntimeAnalyzer(cfg.Source.ChainName, common.RuntimeCipher, runtimeMetadata, cfg.Analyzers.Cipher.SlowSyncRange(), cfg.Analyzers.Cipher.BatchSize, analyzer.SlowSyncMode, sourceClient, dbClient, logger)
})
}
if cfg.Analyzers.EmeraldEvmTokens != nil {
Expand Down

0 comments on commit 5dc52bc

Please sign in to comment.