Store Epochs in IndexedDB and serve from extension (#699)

* Add epochs to IndexedDB

* Fix how epochs are stored in IndexedDB

* Store epochs in the database via the block processor

* Make an extension-local impl of the SCT Service

* Clean up

* Revert styling

* Use descriptive naming

* Fix mock

apps/extension/.env

@@ -1,6 +1,6 @@
 
 PRAX=lkpmkhpnhknhmibgnmmhdhgdilepfghe
-IDB_VERSION=25
+IDB_VERSION=26
 USDC_ASSET_ID="reum7wQmk/owgvGMWMZn/6RFPV24zIKq3W6In/WwZgg="
 MINIFRONT_URL=https://app.testnet.penumbra.zone/
 PENUMBRA_NODE_PD_URL=https://grpc.testnet.penumbra.zone/

apps/extension/src/impls.ts

@@ -17,6 +17,7 @@ import { TendermintProxyService } from '@buf/penumbra-zone_penumbra.connectrpc_e
 import { CustodyService } from '@buf/penumbra-zone_penumbra.connectrpc_es/penumbra/custody/v1/custody_connect';
 import { ViewService } from '@buf/penumbra-zone_penumbra.connectrpc_es/penumbra/view/v1/view_connect';
 import { custodyImpl } from '@penumbra-zone/router/src/grpc/custody';
+import { sctImpl } from '@penumbra-zone/router/src/grpc/sct';
 import { viewImpl } from '@penumbra-zone/router/src/grpc/view-protocol-server';
 
 import { localExtStorage } from '@penumbra-zone/storage';
@@ -30,7 +31,6 @@ const penumbraProxies = [
   DexService,
   DexSimulationService,
   GovernanceService,
-  SctService,
   ShieldedPoolService,
   StakeService,
   TendermintProxyService,
@@ -49,6 +49,7 @@ export const rpcImpls = [
   // rpc local implementations
   // @ts-expect-error TODO: accept partial impl
   [CustodyService, rethrowImplErrors(CustodyService, custodyImpl)],
+  [SctService, rethrowImplErrors(SctService, sctImpl)],
   [ViewService, rethrowImplErrors(ViewService, viewImpl)],
   // rpc remote proxies
   ...penumbraProxies,

packages/query/src/block-processor.ts

@@ -133,13 +133,19 @@ export class BlockProcessor implements BlockProcessorInterface {
     const startHeight = fullSyncHeight ? fullSyncHeight + 1n : 0n;
     const latestBlockHeight = await this.querier.tendermint.latestBlockHeight();
 
+    let nextEpochStartHeight: bigint | undefined = startHeight === 0n ? 0n : undefined;
+
     // this is an indefinite stream of the (compact) chain from the network
     // intended to run continuously
     for await (const compactBlock of this.querier.compactBlock.compactBlockRange({
       startHeight,
       keepAlive: true,
       abortSignal: this.abortController.signal,
     })) {
+      if (nextEpochStartHeight === compactBlock.height) {
+        await this.indexedDb.addEpoch(nextEpochStartHeight);
+        nextEpochStartHeight = undefined;
+      }
       if (compactBlock.appParametersUpdated) {
         await this.indexedDb.saveAppParams(await this.querier.app.appParams());
       }
@@ -229,6 +235,9 @@ export class BlockProcessor implements BlockProcessorInterface {
         // - saves to idb
         await this.saveTransactionInfos(compactBlock.height, relevantTx);
       }
+
+      const isLastBlockOfEpoch = !!compactBlock.epochRoot;
+      if (isLastBlockOfEpoch) nextEpochStartHeight = compactBlock.height + 1n;
     }
   }
 

packages/router/src/grpc/sct/epoch-by-height.test.ts

@@ -0,0 +1,53 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { epochByHeight } from './epoch-by-height';
+import { IndexedDbMock, MockServices } from '../test-utils';
+import { HandlerContext, createContextValues, createHandlerContext } from '@connectrpc/connect';
+import { QueryService as SctService } from '@buf/penumbra-zone_penumbra.connectrpc_es/penumbra/core/component/sct/v1/sct_connect';
+import { ServicesInterface } from '@penumbra-zone/types';
+import { servicesCtx } from '../../ctx';
+import {
+  Epoch,
+  EpochByHeightRequest,
+} from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/sct/v1/sct_pb';
+
+describe('EpochByHeight request handler', () => {
+  let mockServices: MockServices;
+  let mockIndexedDb: IndexedDbMock;
+  let mockCtx: HandlerContext;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+
+    mockIndexedDb = {
+      getEpochByHeight: vi.fn(),
+    };
+    mockServices = {
+      getWalletServices: vi.fn(() =>
+        Promise.resolve({ indexedDb: mockIndexedDb }),
+      ) as MockServices['getWalletServices'],
+    };
+    mockCtx = createHandlerContext({
+      service: SctService,
+      method: SctService.methods.epochByHeight,
+      protocolName: 'mock',
+      requestMethod: 'MOCK',
+      url: '/mock',
+      contextValues: createContextValues().set(
+        servicesCtx,
+        mockServices as unknown as ServicesInterface,
+      ),
+    });
+  });
+
+  it('returns an `EpochByHeightResponse` with the result of the database query', async () => {
+    const expected = new Epoch({ startHeight: 0n, index: 0n });
+
+    mockIndexedDb.getEpochByHeight?.mockResolvedValue(expected);
+    const req = new EpochByHeightRequest({ height: 0n });
+
+    const result = await epochByHeight(req, mockCtx);
+
+    expect(result.epoch).toBeInstanceOf(Epoch);
+    expect((result.epoch as Epoch).toJson()).toEqual(expected.toJson());
+  });
+});

packages/router/src/grpc/sct/epoch-by-height.ts

@@ -0,0 +1,14 @@
+import { EpochByHeightResponse } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/sct/v1/sct_pb';
+import { Impl } from '.';
+import { servicesCtx } from '../../ctx';
+
+export const epochByHeight: Impl['epochByHeight'] = async (req, ctx) => {
+  const { height } = req;
+
+  const services = ctx.values.get(servicesCtx);
+  const { indexedDb } = await services.getWalletServices();
+
+  const epoch = await indexedDb.getEpochByHeight(height);
+
+  return new EpochByHeightResponse({ epoch });
+};

packages/router/src/grpc/sct/index.ts

@@ -0,0 +1,9 @@
+import { QueryService as SctService } from '@buf/penumbra-zone_penumbra.connectrpc_es/penumbra/core/component/sct/v1/sct_connect';
+import { ServiceImpl } from '@connectrpc/connect';
+import { epochByHeight } from './epoch-by-height';
+
+export type Impl = ServiceImpl<typeof SctService>;
+
+export const sctImpl: Impl = {
+  epochByHeight,
+};

packages/router/src/grpc/test-utils.ts

@@ -20,6 +20,7 @@ export interface IndexedDbMock {
   iterateTransactionInfo?: () => Partial<AsyncIterable<Mock>>;
   subscribe?: (table: string) => Partial<AsyncIterable<Mock>>;
   getSwapByCommitment?: Mock;
+  getEpochByHeight?: Mock;
 }
 export interface TendermintMock {
   broadcastTx?: Mock;

packages/storage/src/indexed-db/index.ts

@@ -14,7 +14,10 @@ import {
 } from '@penumbra-zone/types';
 import { IbdUpdater, IbdUpdates } from './updater';
 import { FmdParameters } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/shielded_pool/v1/shielded_pool_pb';
-import { Nullifier } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/sct/v1/sct_pb';
+import {
+  Epoch,
+  Nullifier,
+} from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/sct/v1/sct_pb';
 import { TransactionId } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/txhash/v1/txhash_pb';
 import {
   NotesForVotingResponse,
@@ -90,6 +93,7 @@ export class IndexedDb implements IndexedDbInterface {
         }).createIndex('nullifier', 'nullifier.inner');
         db.createObjectStore('GAS_PRICES');
         db.createObjectStore('POSITIONS', { keyPath: 'id.inner' });
+        db.createObjectStore('EPOCHS', { autoIncrement: true });
       },
     });
     const constants = {
@@ -409,6 +413,62 @@ export class IndexedDb implements IndexedDbInterface {
     });
   }
 
+  async addEpoch(startHeight: bigint, index?: bigint): Promise<void> {
+    if (index === undefined) {
+      const cursor = await this.db.transaction('EPOCHS', 'readonly').store.openCursor(null, 'prev');
+      const previousEpoch = cursor?.value ? Epoch.fromJson(cursor.value) : undefined;
+      index = previousEpoch?.index !== undefined ? previousEpoch.index + 1n : 0n;
+    }
+
+    const newEpoch = {
+      startHeight: startHeight.toString(),
+      index: index.toString(),
+    };
+
+    await this.u.update({
+      table: 'EPOCHS',
+      value: newEpoch,
+    });
+  }
+
+  /**
+   * Get the epoch that contains the given block height.
+   */
+  async getEpochByHeight(
+    /**
+     * The block height to query by. Will return the epoch with the largest
+     * start height smaller than `height` -- that is, the epoch that contains
+     * this height.
+     */
+    height: bigint,
+  ): Promise<Epoch | undefined> {
+    let cursor = await this.db.transaction('EPOCHS', 'readonly').store.openCursor();
+
+    let epoch: Epoch | undefined;
+
+    /**
+     * Iterate over epochs and return the one with the largest start height
+     * smaller than `height`.
+     *
+     * Unfortunately, there doesn't appear to be a more efficient way of doing
+     * this. We tried using epochs' start heights as their key so that we could
+     * use a particular start height as a query bounds, but IndexedDB casts the
+     * `bigint` start height to a string, which messes up sorting (the string
+     * '11' is greater than the string '100', for example). For now, then, we
+     * have to just iterate over all epochs to find the correct starting height.
+     */
+    while (cursor) {
+      const currentEpoch = Epoch.fromJson(cursor.value);
+
+      if (currentEpoch.startHeight <= height) epoch = currentEpoch;
+      else if (currentEpoch.startHeight > height) break;
+
+      cursor = await cursor.continue();
+    }
+
+    return epoch;
+  }
+
   private addSctUpdates(txs: IbdUpdates, sctUpdates: ScanBlockResult['sctUpdates']): void {
     if (sctUpdates.set_position) {
       txs.add({

packages/storage/src/indexed-db/indexed-db.test-data.ts

@@ -10,6 +10,7 @@ import {
   PositionId,
   TradingPair,
 } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/dex/v1/dex_pb';
+import { Epoch } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/sct/v1/sct_pb';
 
 export const emptyScanResult: ScanBlockResult = {
   height: 1092n,
@@ -1463,3 +1464,18 @@ export const tradingPairGmGn = TradingPair.fromJson({
   asset1: { inner: 'HW2Eq3UZVSBttoUwUi/MUtE7rr2UU7/UH500byp7OAc=' },
   asset2: { inner: 'nwPDkQq3OvLnBwGTD+nmv1Ifb2GEmFCgNHrU++9BsRE=' },
 });
+
+export const epoch1 = new Epoch({
+  index: 0n,
+  startHeight: 0n,
+});
+
+export const epoch2 = new Epoch({
+  index: 1n,
+  startHeight: 100n,
+});
+
+export const epoch3 = new Epoch({
+  index: 2n,
+  startHeight: 200n,
+});

packages/storage/src/indexed-db/indexed-db.test.ts

@@ -5,12 +5,15 @@ import {
   TransactionInfo,
 } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/view/v1/view_pb';
 import { IdbUpdate, PenumbraDb } from '@penumbra-zone/types';
-import { describe, expect, it } from 'vitest';
+import { beforeEach, describe, expect, it } from 'vitest';
 import { IndexedDb } from '.';
 import {
   delegationMetadataA,
   delegationMetadataB,
   emptyScanResult,
+  epoch1,
+  epoch2,
+  epoch3,
   metadataA,
   metadataB,
   metadataC,
@@ -512,4 +515,43 @@ describe('IndexedDb', () => {
       expect(ownedPositions.length).toBe(1);
     });
   });
+
+  describe('epochs', () => {
+    let db: IndexedDb;
+
+    beforeEach(async () => {
+      db = await IndexedDb.initialize({ ...generateInitialProps() });
+      await db.addEpoch(epoch1.startHeight);
+      await db.addEpoch(epoch2.startHeight);
+      await db.addEpoch(epoch3.startHeight);
+    });
+
+    describe('addEpoch', () => {
+      it('auto-increments the epoch index if one is not provided', async () => {
+        const [result1, result2, result3] = await Promise.all([
+          db.getEpochByHeight(50n),
+          db.getEpochByHeight(150n),
+          db.getEpochByHeight(250n),
+        ]);
+
+        expect(result1?.index).toBe(0n);
+        expect(result2?.index).toBe(1n);
+        expect(result3?.index).toBe(2n);
+      });
+    });
+
+    describe('getEpochByHeight', () => {
+      it('returns the epoch containing the given block height', async () => {
+        const [result1, result2, result3] = await Promise.all([
+          db.getEpochByHeight(50n),
+          db.getEpochByHeight(150n),
+          db.getEpochByHeight(250n),
+        ]);
+
+        expect(result1?.toJson()).toEqual(epoch1.toJson());
+        expect(result2?.toJson()).toEqual(epoch2.toJson());
+        expect(result3?.toJson()).toEqual(epoch3.toJson());
+      });
+    });
+  });
 });

packages/types/src/indexed-db.ts

@@ -22,7 +22,10 @@ import {
   FmdParameters,
   Note,
 } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/shielded_pool/v1/shielded_pool_pb';
-import { Nullifier } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/sct/v1/sct_pb';
+import {
+  Epoch,
+  Nullifier,
+} from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/sct/v1/sct_pb';
 import { TransactionId } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/txhash/v1/txhash_pb';
 import { StateCommitment } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/crypto/tct/v1/tct_pb';
 import { GasPrices } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/component/fee/v1/fee_pb';
@@ -83,6 +86,8 @@ export interface IndexedDbInterface {
   ): AsyncGenerator<PositionId, void>;
   addPosition(positionId: PositionId, position: Position): Promise<void>;
   updatePosition(positionId: PositionId, newState: PositionState): Promise<void>;
+  addEpoch(startHeight: bigint, index?: bigint): Promise<void>;
+  getEpochByHeight(height: bigint): Promise<Epoch | undefined>;
 }
 
 export interface PenumbraDb extends DBSchema {
@@ -154,6 +159,10 @@ export interface PenumbraDb extends DBSchema {
     key: string; // base64 PositionRecord['id']['inner'];
     value: PositionRecord;
   };
+  EPOCHS: {
+    key: number; // auto-increment
+    value: Jsonified<Epoch>;
+  };
 }
 
 // need to store PositionId and Position in the same table
@@ -180,4 +189,5 @@ export const IDB_TABLES: Tables = {
   fmd_parameters: 'FMD_PARAMETERS',
   app_parameters: 'APP_PARAMETERS',
   gas_prices: 'GAS_PRICES',
+  epochs: 'EPOCHS',
 };