-
Notifications
You must be signed in to change notification settings - Fork 58
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Updates * updates * cleaned logs * updates * updated recycleNonce to check if isNan * comments updated * chore: expose NonceResyncQueue to bullboard * Convert nonce-recycled to set and add nonce-sent set helpers * fix: handle removing from sent nonces in fail handler * chore: Remove commented out code in setup.ts file * Update nonceResyncWorker to use stricter wildcard pattern for redis keys * Add isSentNonce helper function for checking if a nonce is in the sent nonces set * refactor and solve nonce resynchronization logic bugs and nits in nonceResyncWorker * chore: Refactor cancelRecycledNoncesWorker to use SMEMBERS instead of LRANGE for retrieving unused nonces * treat `ReplacementGasFeeTowLow` as `NonceAlreadyUsed` * updated logs & removeSentNonce --------- Co-authored-by: Prithvish Baidya <[email protected]>
- Loading branch information
Showing
9 changed files
with
256 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| import { Queue } from "bullmq"; | ||
| import { redis } from "../../utils/redis/redis"; | ||
| import { defaultJobOptions } from "./queues"; | ||
|
|
||
| export class NonceResyncQueue { | ||
| static q = new Queue<string>("nonce-resync-cron", { | ||
| connection: redis, | ||
| defaultJobOptions, | ||
| }); | ||
|
|
||
| constructor() { | ||
| NonceResyncQueue.q.setGlobalConcurrency(1); | ||
|
|
||
| // The cron job is defined in `initNonceResyncWorker` | ||
| // because it requires an async call to query configuration. | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| import { Job, Processor, Worker } from "bullmq"; | ||
| import { eth_getTransactionCount, getRpcClient } from "thirdweb"; | ||
| import { | ||
| inspectNonce, | ||
| isSentNonce, | ||
| recycleNonce, | ||
| splitSentNoncesKey, | ||
| } from "../../db/wallets/walletNonce"; | ||
| import { getConfig } from "../../utils/cache/getConfig"; | ||
| import { getChain } from "../../utils/chain"; | ||
| import { logger } from "../../utils/logger"; | ||
| import { redis } from "../../utils/redis/redis"; | ||
| import { thirdwebClient } from "../../utils/sdk"; | ||
| import { NonceResyncQueue } from "../queues/nonceResyncQueue"; | ||
| import { logWorkerExceptions } from "../queues/queues"; | ||
|
|
||
| // Must be explicitly called for the worker to run on this host. | ||
| export const initNonceResyncWorker = async () => { | ||
| const config = await getConfig(); | ||
| if (config.minedTxListenerCronSchedule) { | ||
| NonceResyncQueue.q.add("cron", "", { | ||
| repeat: { pattern: config.minedTxListenerCronSchedule }, | ||
| jobId: "nonce-resync-cron", | ||
| }); | ||
| } | ||
|
|
||
| const _worker = new Worker(NonceResyncQueue.q.name, handler, { | ||
| connection: redis, | ||
| concurrency: 1, | ||
| }); | ||
| logWorkerExceptions(_worker); | ||
| }; | ||
|
|
||
| /** | ||
| * Resyncs nonces for all wallets. | ||
| * This worker should be run periodically to ensure that nonces are not skipped. | ||
| * It checks the onchain nonce for each wallet and recycles any missing nonces. | ||
| * | ||
| * This is to unblock a wallet that has been stuck due to one or more skipped nonces. | ||
| */ | ||
| const handler: Processor<any, void, string> = async (job: Job<string>) => { | ||
| const sentNoncesKeys = await redis.keys("nonce-sent*"); | ||
| job.log(`Found ${sentNoncesKeys.length} nonce-sent* keys`); | ||
|
|
||
| for (const sentNonceKey of sentNoncesKeys) { | ||
| const { chainId, walletAddress } = splitSentNoncesKey(sentNonceKey); | ||
|
|
||
| const rpcRequest = getRpcClient({ | ||
| client: thirdwebClient, | ||
| chain: await getChain(chainId), | ||
| }); | ||
|
|
||
| const [transactionCount, lastUsedNonceDb] = await Promise.all([ | ||
| eth_getTransactionCount(rpcRequest, { | ||
| address: walletAddress, | ||
| }), | ||
| inspectNonce(chainId, walletAddress), | ||
| ]); | ||
|
|
||
| if (isNaN(transactionCount)) { | ||
| job.log( | ||
| `Received invalid onchain transaction count for ${walletAddress}: ${transactionCount}`, | ||
| ); | ||
|
|
||
| logger({ | ||
| level: "error", | ||
| message: `[nonceResyncWorker] Received invalid onchain transaction count for ${walletAddress}: ${transactionCount}`, | ||
| service: "worker", | ||
| }); | ||
|
|
||
| return; | ||
| } | ||
|
|
||
| const lastUsedNonceOnchain = transactionCount - 1; | ||
|
|
||
| job.log( | ||
| `${walletAddress} last used onchain nonce: ${lastUsedNonceOnchain} and last used db nonce: ${lastUsedNonceDb}`, | ||
| ); | ||
| logger({ | ||
| level: "debug", | ||
| message: `[nonceResyncWorker] last used onchain nonce: ${transactionCount} and last used db nonce: ${lastUsedNonceDb}`, | ||
| service: "worker", | ||
| }); | ||
|
|
||
| // If the last used nonce onchain is the same as or ahead of the last used nonce in the db, | ||
| // There is no need to resync the nonce. | ||
| if (lastUsedNonceOnchain >= lastUsedNonceDb) { | ||
| job.log(`No need to resync nonce for ${walletAddress}`); | ||
| logger({ | ||
| level: "debug", | ||
| message: `[nonceResyncWorker] No need to resync nonce for ${walletAddress}`, | ||
| service: "worker", | ||
| }); | ||
| return; | ||
| } | ||
|
|
||
| // for each nonce between last used db nonce and last used onchain nonce | ||
| // check if nonce exists in nonce-sent set | ||
| // if it does not exist, recycle it | ||
| for ( | ||
| let _nonce = lastUsedNonceOnchain + 1; | ||
| _nonce < lastUsedNonceDb; | ||
| _nonce++ | ||
| ) { | ||
| const exists = await isSentNonce(chainId, walletAddress, _nonce); | ||
| logger({ | ||
| level: "debug", | ||
| message: `[nonceResyncWorker] nonce ${_nonce} exists in nonce-sent set: ${exists}`, | ||
| service: "worker", | ||
| }); | ||
|
|
||
| // If nonce does not exist in nonce-sent set, recycle it | ||
| if (!exists) { | ||
| await recycleNonce(chainId, walletAddress, _nonce); | ||
| } | ||
| } | ||
| } | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.