Implement l2 transactions preprocessing

packages/backend/src/Application.ts

@@ -51,6 +51,7 @@ import { PerpetualHistoryPreprocessor } from './core/preprocessing/PerpetualHist
 import { Preprocessor } from './core/preprocessing/Preprocessor'
 import { SpotHistoryPreprocessor } from './core/preprocessing/SpotHistoryPreprocessor'
 import { StateDetailsPreprocessor } from './core/preprocessing/StateDetailsPreprocessor'
+import { UserL2TransactionsStatisticsPreprocessor } from './core/preprocessing/UserL2TransactionsPreprocessor'
 import { UserStatisticsPreprocessor } from './core/preprocessing/UserStatisticsPreprocessor'
 import { SpotValidiumSyncService } from './core/SpotValidiumSyncService'
 import { SpotValidiumUpdater } from './core/SpotValidiumUpdater'
@@ -71,6 +72,7 @@ import { PositionRepository } from './peripherals/database/PositionRepository'
 import { PreprocessedAssetHistoryRepository } from './peripherals/database/PreprocessedAssetHistoryRepository'
 import { PreprocessedStateDetailsRepository } from './peripherals/database/PreprocessedStateDetailsRepository'
 import { PreprocessedStateUpdateRepository } from './peripherals/database/PreprocessedStateUpdateRepository'
+import { PreprocessedUserL2TransactionsStatisticsRepository } from './peripherals/database/PreprocessedUserL2TransactionsStatisticsRepository'
 import { PreprocessedUserStatisticsRepository } from './peripherals/database/PreprocessedUserStatisticsRepository'
 import { Database } from './peripherals/database/shared/Database'
 import { StateTransitionRepository } from './peripherals/database/StateTransitionRepository'
@@ -87,7 +89,6 @@ import { AvailabilityGatewayClient } from './peripherals/starkware/AvailabilityG
 import { FeederGatewayClient } from './peripherals/starkware/FeederGatewayClient'
 import { FetchClient } from './peripherals/starkware/FetchClient'
 import { handleServerError, reportError } from './tools/ErrorReporter'
-import { shouldShowL2Transactions } from './utils/shouldShowL2Transactions'
 
 export class Application {
   start: () => Promise<void>
@@ -414,6 +415,16 @@ export class Application {
     const preprocessedUserStatisticsRepository =
       new PreprocessedUserStatisticsRepository(database, logger)
 
+    const preprocessedUserL2TransactionsStatisticsRepository =
+      new PreprocessedUserL2TransactionsStatisticsRepository(database, logger)
+
+    const userL2TransactionsPreprocessor =
+      new UserL2TransactionsStatisticsPreprocessor(
+        preprocessedUserL2TransactionsStatisticsRepository,
+        l2TransactionRepository,
+        logger
+      )
+
     let preprocessor: Preprocessor<AssetHash> | Preprocessor<AssetId>
     const isPreprocessorEnabled = config.enablePreprocessing
 
@@ -443,13 +454,13 @@ export class Application {
         preprocessedStateDetailsRepository,
         preprocessedAssetHistoryRepository,
         userTransactionRepository,
+        l2TransactionRepository,
         logger
       )
 
       const userStatisticsPreprocessor = new UserStatisticsPreprocessor(
         preprocessedUserStatisticsRepository,
         preprocessedAssetHistoryRepository,
-        preprocessedStateUpdateRepository,
         stateUpdateRepository,
         kvStore,
         logger
@@ -462,6 +473,8 @@ export class Application {
         perpetualHistoryPreprocessor,
         stateDetailsPreprocessor,
         userStatisticsPreprocessor,
+        userL2TransactionsPreprocessor,
+        l2TransactionRepository,
         logger,
         isPreprocessorEnabled
       )
@@ -479,13 +492,13 @@ export class Application {
         preprocessedStateDetailsRepository,
         preprocessedAssetHistoryRepository,
         userTransactionRepository,
+        l2TransactionRepository,
         logger
       )
 
       const userStatisticsPreprocessor = new UserStatisticsPreprocessor(
         preprocessedUserStatisticsRepository,
         preprocessedAssetHistoryRepository,
-        preprocessedStateUpdateRepository,
         stateUpdateRepository,
         kvStore,
         logger
@@ -498,6 +511,8 @@ export class Application {
         spotHistoryPreprocessor,
         stateDetailsPreprocessor,
         userStatisticsPreprocessor,
+        userL2TransactionsPreprocessor,
+        l2TransactionRepository,
         logger,
         isPreprocessorEnabled
       )
@@ -517,16 +532,14 @@ export class Application {
 
     // #endregion core
     // #region api
-    const showL2Transactions = shouldShowL2Transactions(config)
     const homeController = new HomeController(
       pageContextService,
       assetDetailsService,
       forcedTradeOfferViewService,
       userTransactionRepository,
       forcedTradeOfferRepository,
       l2TransactionRepository,
-      preprocessedStateDetailsRepository,
-      showL2Transactions
+      preprocessedStateDetailsRepository
     )
 
     const userController = new UserController(
@@ -541,8 +554,8 @@ export class Application {
       forcedTradeOfferViewService,
       withdrawableAssetRepository,
       preprocessedUserStatisticsRepository,
-      config.starkex.contracts.perpetual,
-      showL2Transactions
+      preprocessedUserL2TransactionsStatisticsRepository,
+      config.starkex.contracts.perpetual
     )
     const stateUpdateController = new StateUpdateController(
       pageContextService,
@@ -551,7 +564,7 @@ export class Application {
       userTransactionRepository,
       l2TransactionRepository,
       preprocessedAssetHistoryRepository,
-      showL2Transactions
+      preprocessedStateDetailsRepository
     )
     const transactionController = new TransactionController(
       pageContextService,
@@ -629,7 +642,6 @@ export class Application {
     this.start = async () => {
       logger.for(this).info('Starting')
 
-      await apiServer.listen()
       if (config.freshStart) await database.rollbackAll()
       await database.migrateToLatest()
       await preprocessor.catchUp()
@@ -641,6 +653,8 @@ export class Application {
       await stateUpdateWithBatchIdMigrator.migrate()
       await stateUpdater.initTree()
 
+      await apiServer.listen()
+
       if (config.enableSync) {
         transactionStatusService.start()
         await syncScheduler.start()

packages/backend/src/api/controllers/ForcedTradeOfferController.test.ts

@@ -54,6 +54,7 @@ describe(ForcedTradeOfferController.name, () => {
   const pageContext: PageContext = {
     user: undefined,
     tradingMode: 'perpetual',
+    showL2Transactions: true,
     chainId: 1,
     instanceName: 'dYdX',
     collateralAsset: fakeCollateralAsset,

packages/backend/src/api/controllers/HomeController.ts

@@ -12,6 +12,7 @@ import { PageContextService } from '../../core/PageContextService'
 import { PaginationOptions } from '../../model/PaginationOptions'
 import { ForcedTradeOfferRepository } from '../../peripherals/database/ForcedTradeOfferRepository'
 import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository'
+import { sumUpTransactionCount } from '../../peripherals/database/PreprocessedL2TransactionsStatistics'
 import { PreprocessedStateDetailsRepository } from '../../peripherals/database/PreprocessedStateDetailsRepository'
 import { UserTransactionData } from '../../peripherals/database/transactions/UserTransaction'
 import { UserTransactionRepository } from '../../peripherals/database/transactions/UserTransactionRepository'
@@ -33,8 +34,7 @@ export class HomeController {
     private readonly userTransactionRepository: UserTransactionRepository,
     private readonly forcedTradeOfferRepository: ForcedTradeOfferRepository,
     private readonly l2TransactionRepository: L2TransactionRepository,
-    private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository,
-    private readonly showL2Transactions: boolean
+    private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository
   ) {}
 
   async getHomePage(
@@ -44,7 +44,7 @@ export class HomeController {
     const paginationOpts = { offset: 0, limit: 6 }
     const [
       l2Transactions,
-      l2TransactionsCount,
+      lastStateDetailsWithL2TransactionsStatistics,
       stateUpdates,
       stateUpdatesCount,
       forcedUserTransactions,
@@ -53,7 +53,7 @@ export class HomeController {
       availableOffersCount,
     ] = await Promise.all([
       this.l2TransactionRepository.getPaginatedWithoutMulti(paginationOpts),
-      this.l2TransactionRepository.countAllDistinctTransactionIds(),
+      this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(),
       this.preprocessedStateDetailsRepository.getPaginated(paginationOpts),
       this.preprocessedStateDetailsRepository.countAll(),
       this.userTransactionRepository.getPaginated({
@@ -85,12 +85,10 @@ export class HomeController {
     const content = renderHomePage({
       context,
       tutorials: [], // explicitly no tutorials
-      l2Transactions: this.showL2Transactions
-        ? {
-            data: l2Transactions.map(l2TransactionToEntry),
-            total: l2TransactionsCount,
-          }
-        : undefined,
+      l2Transactions: l2Transactions.map(l2TransactionToEntry),
+      totalL2Transactions: sumUpTransactionCount(
+        lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics
+      ),
       stateUpdates: stateUpdateEntries,
       totalStateUpdates: stateUpdatesCount,
       forcedTransactions: forcedTransactionEntries,
@@ -111,16 +109,19 @@ export class HomeController {
   ): Promise<ControllerResult> {
     const context = await this.pageContextService.getPageContext(givenUser)
 
-    const [total, l2Transactions] = await Promise.all([
-      this.l2TransactionRepository.countAllDistinctTransactionIds(),
-      this.l2TransactionRepository.getPaginatedWithoutMulti(pagination),
-    ])
+    const [l2Transactions, lastStateDetailsWithL2TransactionsStatistics] =
+      await Promise.all([
+        this.l2TransactionRepository.getPaginatedWithoutMulti(pagination),
+        this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(),
+      ])
 
     const content = renderHomeL2TransactionsPage({
       context,
       l2Transactions: l2Transactions.map(l2TransactionToEntry),
+      total: sumUpTransactionCount(
+        lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics
+      ),
       ...pagination,
-      total,
     })
     return { type: 'success', content }
   }

packages/backend/src/api/controllers/L2TransactionController.ts

@@ -27,7 +27,9 @@ export class L2TransactionController {
       return { type: 'not found' }
     }
     const aggregatedL2Transaction =
-      await this.l2TransactionRepository.findByTransactionId(transactionId)
+      await this.l2TransactionRepository.findAggregatedByTransactionId(
+        transactionId
+      )
 
     if (!aggregatedL2Transaction) {
       return {

packages/backend/src/api/controllers/StateUpdateController.ts

@@ -16,6 +16,8 @@ import {
   PreprocessedAssetHistoryRecord,
   PreprocessedAssetHistoryRepository,
 } from '../../peripherals/database/PreprocessedAssetHistoryRepository'
+import { sumUpTransactionCount } from '../../peripherals/database/PreprocessedL2TransactionsStatistics'
+import { PreprocessedStateDetailsRepository } from '../../peripherals/database/PreprocessedStateDetailsRepository'
 import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository'
 import { UserTransactionData } from '../../peripherals/database/transactions/UserTransaction'
 import { UserTransactionRepository } from '../../peripherals/database/transactions/UserTransactionRepository'
@@ -38,7 +40,7 @@ export class StateUpdateController {
     private readonly userTransactionRepository: UserTransactionRepository,
     private readonly l2TransactionRepository: L2TransactionRepository,
     private readonly preprocessedAssetHistoryRepository: PreprocessedAssetHistoryRepository,
-    private readonly showL2Transactions: boolean
+    private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository
   ) {}
 
   async getStateUpdatePage(
@@ -56,7 +58,7 @@ export class StateUpdateController {
       forcedUserTransactions,
       totalForcedUserTransactions,
       l2Transactions,
-      totalL2Transactions,
+      preprocessedStateDetails,
     ] = await Promise.all([
       this.stateUpdateRepository.findById(stateUpdateId),
       this.preprocessedAssetHistoryRepository.getByStateUpdateIdPaginated(
@@ -82,7 +84,7 @@ export class StateUpdateController {
           limit: 6,
         }
       ),
-      this.l2TransactionRepository.countAllDistinctTransactionIdsByStateUpdateId(
+      this.preprocessedStateDetailsRepository.findByStateUpdateId(
         stateUpdateId
       ),
     ])
@@ -127,12 +129,12 @@ export class StateUpdateController {
       balanceChanges: balanceChangeEntries,
       totalBalanceChanges,
       priceChanges: priceEntries,
-      l2Transactions: this.showL2Transactions
-        ? {
-            data: l2Transactions.map(l2TransactionToEntry),
-            total: totalL2Transactions,
-          }
-        : undefined,
+      l2Transactions: l2Transactions.map(l2TransactionToEntry),
+      totalL2Transactions: preprocessedStateDetails?.l2TransactionsStatistics
+        ? sumUpTransactionCount(
+            preprocessedStateDetails.l2TransactionsStatistics
+          )
+        : 'processing',
       transactions,
       totalTransactions: totalForcedUserTransactions,
     })
@@ -147,12 +149,12 @@ export class StateUpdateController {
   ): Promise<ControllerResult> {
     const context = await this.pageContextService.getPageContext(givenUser)
 
-    const [l2Transactions, total] = await Promise.all([
+    const [l2Transactions, preprocessedStateDetails] = await Promise.all([
       this.l2TransactionRepository.getPaginatedWithoutMultiByStateUpdateId(
         stateUpdateId,
         pagination
       ),
-      this.l2TransactionRepository.countAllDistinctTransactionIdsByStateUpdateId(
+      this.preprocessedStateDetailsRepository.findByStateUpdateId(
         stateUpdateId
       ),
     ])
@@ -162,7 +164,11 @@ export class StateUpdateController {
       id: stateUpdateId.toString(),
       l2Transactions: l2Transactions.map(l2TransactionToEntry),
       ...pagination,
-      total,
+      total: preprocessedStateDetails?.l2TransactionsStatistics
+        ? sumUpTransactionCount(
+            preprocessedStateDetails.l2TransactionsStatistics
+          )
+        : 'processing',
     })
 
     return { type: 'success', content }