Moving window normalized fee

lib/bitcoin/BitcoinProcessor.ts

@@ -69,7 +69,7 @@ export default class BitcoinProcessor {
   private versionManager: VersionManager;
 
   /** Last seen block */
-  private lastProcessedBlock: IBlockInfo | undefined;
+  private lastProcessedBlock: BlockMetadata | undefined;
 
   /** Poll timeout identifier */
   private pollTimeoutId: number | undefined;
@@ -95,8 +95,8 @@ export default class BitcoinProcessor {
   /** at least 100 blocks per page unless reaching the last block */
   private static readonly pageSizeInBlocks = 100;
 
-  public constructor (private config: IBitcoinConfig, versionModels: BitcoinVersionModel[]) {
-    this.versionManager = new VersionManager(versionModels, config);
+  public constructor (private config: IBitcoinConfig) {
+    this.versionManager = new VersionManager();
 
     this.genesisBlockNumber = config.genesisBlockNumber;
 
@@ -149,8 +149,8 @@ export default class BitcoinProcessor {
   /**
    * Initializes the Bitcoin processor
    */
-  public async initialize () {
-    await this.versionManager.initialize(this.blockMetadataStore);
+  public async initialize (versionModels: BitcoinVersionModel[]) {
+    await this.versionManager.initialize(versionModels, this.config, this.blockMetadataStore);
     await this.serviceStateStore.initialize();
     await this.blockMetadataStore.initialize();
     await this.transactionStore.initialize();
@@ -405,7 +405,7 @@ export default class BitcoinProcessor {
    * Fetches Sidetree transactions in chronological order from since or genesis.
    * @param since A transaction number
    * @param hash The associated transaction time hash
-   * @returns Transactions in complete blocks since given transaction number, with normaliedFee.
+   * @returns Transactions in complete blocks since given transaction number, with normalizedFee.
    */
   public async transactions (since?: number, hash?: string): Promise<{
     moreTransactions: boolean,
@@ -441,17 +441,18 @@ export default class BitcoinProcessor {
       const inclusiveFirstBlockHeight = transactions[0].transactionTime;
       const exclusiveLastBlockHeight = transactions[transactions.length - 1].transactionTime + 1;
       const blockMetaData = await this.blockMetadataStore.get(inclusiveFirstBlockHeight, exclusiveLastBlockHeight);
-      let blockMetadataIndex = 0;
+      const blockMetaDataMap: Map<number, BlockMetadata> = new Map();
+      for (const block of blockMetaData) {
+        blockMetaDataMap.set(block.height, block);
+      }
+
       for (const transaction of transactions) {
-        while (transaction.transactionTime !== blockMetaData[blockMetadataIndex].height) {
-          blockMetadataIndex++;
-          // This means that it has reached the end of block metadata and still cannot find the data for the current transaction.
-          // This can happen when a block reorg happens, and some block metadata are getting deleted.
-          if (blockMetadataIndex === blockMetaData.length) {
-            throw new RequestError(ResponseStatus.ServerError, ErrorCode.BitcoinBlockMetadataNotFound);
-          }
-        };
-        transaction.normalizedTransactionFee = blockMetaData[blockMetadataIndex].normalizedFee;
+        const block = blockMetaDataMap.get(transaction.transactionTime);
+        if (block !== undefined) {
+          transaction.normalizedTransactionFee = block.normalizedFee;
+        } else {
+          throw new RequestError(ResponseStatus.ServerError, ErrorCode.BitcoinBlockMetadataNotFound);
+        }
       }
     }
 
@@ -548,23 +549,22 @@ export default class BitcoinProcessor {
    * @param blocks the ordered block metadata to set the normalized fee for.
    */
   private async writeBlocksToMetadataStoreWithFee (blocks: BlockMetadataWithoutNormalizedFee[]) {
+    const blocksToWrite = [];
     for (const block of blocks) {
       const normalizedFee = (await this.getNormalizedFee(block.height)).normalizedTransactionFee;
-      this.blockMetadataStore.add([{
+      const blockMetadata = {
         normalizedFee,
         height: block.height,
         hash: block.hash,
         previousHash: block.previousHash,
         transactionCount: block.transactionCount,
         totalFee: block.totalFee
-      }]);
-
-      this.lastProcessedBlock = {
-        hash: block.hash,
-        height: block.height,
-        previousHash: block.previousHash
       };
+
+      blocksToWrite.push(blockMetadata);
     }
+    this.blockMetadataStore.add(blocksToWrite);
+    this.lastProcessedBlock = blocksToWrite[blocksToWrite.length - 1];
   }
 
   /**
@@ -689,13 +689,7 @@ export default class BitcoinProcessor {
     while (blockHeight <= endBlockHeight) {
       const processedBlockMetadata = await this.processBlock(blockHeight, previousBlockHash);
 
-      await this.blockMetadataStore.add([processedBlockMetadata]);
-
-      this.lastProcessedBlock = {
-        hash: processedBlockMetadata.hash,
-        height: processedBlockMetadata.height,
-        previousHash: processedBlockMetadata.previousHash
-      };
+      this.lastProcessedBlock = processedBlockMetadata;
 
       blockHeight++;
       previousBlockHash = processedBlockMetadata.hash;
@@ -826,6 +820,8 @@ export default class BitcoinProcessor {
       transactionCount
     };
 
+    await this.blockMetadataStore.add([processedBlockMetadata]);
+
     return processedBlockMetadata;
   }
 

lib/bitcoin/VersionManager.ts

@@ -16,10 +16,8 @@ export default class VersionManager {
   private feeCalculators: Map<string, IFeeCalculator>;
   private protocolParameters: Map<string, ProtocolParameters>;
 
-  public constructor (versions: BitcoinVersionModel[], private config: IBitcoinConfig) {
-    // Reverse sort versions.
-    this.versionsReverseSorted = versions.sort((a, b) => b.startingBlockchainTime - a.startingBlockchainTime);
-
+  public constructor () {
+    this.versionsReverseSorted = [];
     this.feeCalculators = new Map();
     this.protocolParameters = new Map();
   }
@@ -28,26 +26,30 @@ export default class VersionManager {
    * Loads all the implementation versions.
    */
   public async initialize (
+    versions: BitcoinVersionModel[],
+    config: IBitcoinConfig,
     blockMetadataStore: IBlockMetadataStore
   ) {
+    // Reverse sort versions.
+    this.versionsReverseSorted = versions.sort((a, b) => b.startingBlockchainTime - a.startingBlockchainTime);
     // NOTE: In principal each version of the interface implementations can have different constructors,
     // but we currently keep the constructor signature the same as much as possible for simple instance construction,
     // but it is not inherently "bad" if we have to have conditional constructions for each if we have to.
     for (const versionModel of this.versionsReverseSorted) {
       const version = versionModel.version;
-      const initialNormalizedFee = versionModel.protocolParameters.initialNormalizedFee;
-      const lookBackWindowInterval = versionModel.protocolParameters.lookBackWindowInterval;
-      const fluctuationRate = versionModel.protocolParameters.fluctuationRate;
-
       this.protocolParameters.set(version, versionModel.protocolParameters);
 
+      const initialNormalizedFee = versionModel.protocolParameters.initialNormalizedFee;
+      const feeLookBackWindowInBlocks = versionModel.protocolParameters.feeLookBackWindowInBlocks;
+      const feeMaxFluctuationMultiplierPerBlock = versionModel.protocolParameters.feeMaxFluctuationMultiplierPerBlock;
+
       const FeeCalculator = await this.loadDefaultExportsForVersion(version, 'NormalizedFeeCalculator');
       const feeCalculator = new FeeCalculator(
         blockMetadataStore,
-        this.config.genesisBlockNumber,
+        config.genesisBlockNumber,
         initialNormalizedFee,
-        lookBackWindowInterval,
-        fluctuationRate
+        feeLookBackWindowInBlocks,
+        feeMaxFluctuationMultiplierPerBlock
       );
       this.feeCalculators.set(version, feeCalculator);
     }

lib/bitcoin/lock/LockMonitor.ts

@@ -355,7 +355,7 @@ export default class LockMonitor {
     const currentLockIdentifier = LockIdentifierSerializer.deserialize(currentValueTimeLock.identifier);
     const currentLockDuration = currentValueTimeLock.unlockTransactionTime - currentValueTimeLock.lockTransactionTime;
 
-    const newLockDuration = this.versionManager.getLockDurationInBlocks(currentValueTimeLock.unlockTransactionTime);
+    const newLockDuration = this.versionManager.getLockDurationInBlocks(await this.bitcoinClient.getCurrentBlockHeight());
     const relockTransaction =
       await this.bitcoinClient.createRelockTransaction(
         currentLockIdentifier.transactionId,

lib/bitcoin/lock/LockResolver.ts

@@ -87,11 +87,11 @@ export default class LockResolver {
 
     const lockDurationInBlocks = this.versionManager.getLockDurationInBlocks(lockStartBlock);
 
-    if (!this.isLockDurationValid(lockStartBlock, unlockAtBlock)) {
+    if (this.versionManager.getLockDurationInBlocks(lockStartBlock) !== scriptVerifyResult.lockDurationInBlocks!) {
       throw new SidetreeError(
         ErrorCode.LockResolverDurationIsInvalid,
         // eslint-disable-next-line max-len
-        `Lock start block: ${lockStartBlock}. Unlock block: ${unlockAtBlock}. Invalid duration: ${unlockAtBlock - lockStartBlock}. Allowed duration: ${lockDurationInBlocks}`
+        `Lock start block: ${lockStartBlock}. Unlock block: ${unlockAtBlock}. Invalid duration: ${scriptVerifyResult.lockDurationInBlocks!}. Allowed duration: ${lockDurationInBlocks}`
       );
     }
 
@@ -183,17 +183,4 @@ export default class LockResolver {
 
     return blockInfo.height;
   }
-
-  private isLockDurationValid (startBlock: number, unlockBlock: number): boolean {
-    // Example:
-    //  startBlock:  10
-    //  unlockBlock: 20 - no lock at this block
-    //
-    //  lock-duration: unlockBlock - startBlock = 10 blocks
-
-    const lockDuration = unlockBlock - startBlock;
-    const lockDurationInBlocks = this.versionManager.getLockDurationInBlocks(startBlock);
-
-    return lockDuration === lockDurationInBlocks;
-  }
 }

lib/bitcoin/models/BitcoinVersionModel.ts

@@ -3,7 +3,7 @@ import ProtocolParameters from './ProtocolParameters';
 /**
  * Defines an bitcoin implementation version and its starting blockchain time.
  */
-export default interface VersionModel {
+export default interface BitcoinVersionModel {
     /** The inclusive starting logical blockchain time that this version applies to. */
     startingBlockchainTime: number;
     version: string;

lib/bitcoin/models/ProtocolParameters.ts

@@ -8,14 +8,14 @@ export default interface ProtocolParameters {
   /** The initial normalized fee */
   initialNormalizedFee: number;
 
-  /** 
+  /**
    * The look back window for normalized fee calculation
    * If this number is 10, then to calculate block X's normalized fee, it will look at blocks X - 10 to x - 1 to calculate.
    */
-  lookBackWindowInterval: number;
+  feeLookBackWindowInBlocks: number;
 
   /**
    * The fluctuation rate cap. The normalized fee fluctuation cannot exceed this percentage. 1 being 100%.
    */
-  fluctuationRate: number;
+  feeMaxFluctuationMultiplierPerBlock: number;
 }

lib/bitcoin/versions/latest/NormalizedFeeCalculator.ts

@@ -1,3 +1,4 @@
+import BlockMetadata from '../../models/BlockMetadata';
 import IBlockMetadataStore from '../../interfaces/IBlockMetadataStore';
 import IFeeCalculator from '../../interfaces/IFeeCalculator';
 
@@ -10,8 +11,8 @@ export default class NormalizedFeeCalculator implements IFeeCalculator {
     private blockMetadataStore: IBlockMetadataStore,
     private genesisBlockNumber: number,
     private initialNormalizedFee: number,
-    private lookBackWindowInterval: number,
-    private fluctuationRate: number) {}
+    private feeLookBackWindowInBlocks: number,
+    private feeMaxFluctuationMultiplierPerBlock: number) {}
 
   /**
    * Initializes the Bitcoin processor.
@@ -21,17 +22,26 @@ export default class NormalizedFeeCalculator implements IFeeCalculator {
   }
 
   public async getNormalizedFee (block: number): Promise<number> {
+    // DB call optimization
+    // https://github.com/decentralized-identity/sidetree/issues/936
     if (block < this.genesisBlockNumber) {
       // No normalized fee for blocks that exist before genesis
       return 0;
-    } else if (block < this.genesisBlockNumber + this.lookBackWindowInterval) {
+    } else if (block < this.genesisBlockNumber + this.feeLookBackWindowInBlocks) {
       // if within look back interval of genesis, use the initial fee
       return this.initialNormalizedFee;
     }
 
+    const blocksToAverage = await this.getLookBackBlocks(block);
+    return this.calculateNormalizedFee(blocksToAverage);
+  }
+
+  private async getLookBackBlocks (block: number): Promise<BlockMetadata[]> {
     // look back the interval
-    const blocksToAverage = await this.blockMetadataStore.get(block - this.lookBackWindowInterval, block);
+    return await this.blockMetadataStore.get(block - this.feeLookBackWindowInBlocks, block);
+  }
 
+  private calculateNormalizedFee (blocksToAverage: BlockMetadata[]): number {
     let totalFee = 0;
     let totalTransactionCount = 0;
 
@@ -43,22 +53,20 @@ export default class NormalizedFeeCalculator implements IFeeCalculator {
     // TODO: #926 investigate potential rounding differences between languages and implemetations
     // https://github.com/decentralized-identity/sidetree/issues/926
     const unadjustedFee = Math.floor(totalFee / totalTransactionCount);
-
     const previousFee = blocksToAverage[blocksToAverage.length - 1].normalizedFee;
-
-    return this.limitTenPercentPerYear(unadjustedFee, previousFee);
+    return this.adjustFeeToWithinFluctuationRate(unadjustedFee, previousFee);
   }
 
-  private limitTenPercentPerYear (unadjustedFee: number, previousFee: number): number {
-    const previousFeeAdjustedUp = Math.floor(previousFee * (1 + this.fluctuationRate));
-    const previousFeeAdjustedDown = Math.floor(previousFee * (1 - this.fluctuationRate));
+  private adjustFeeToWithinFluctuationRate (unadjustedFee: number, previousFee: number): number {
+    const maxAllowedFee = Math.floor(previousFee * (1 + this.feeMaxFluctuationMultiplierPerBlock));
+    const minAllowedFee = Math.floor(previousFee * (1 - this.feeMaxFluctuationMultiplierPerBlock));
 
-    if (unadjustedFee > previousFeeAdjustedUp) {
-      return previousFeeAdjustedUp;
+    if (unadjustedFee > maxAllowedFee) {
+      return maxAllowedFee;
     }
 
-    if (unadjustedFee < previousFeeAdjustedDown) {
-      return previousFeeAdjustedDown;
+    if (unadjustedFee < minAllowedFee) {
+      return minAllowedFee;
     }
 
     return unadjustedFee;

lib/common/models/TransactionModel.ts

@@ -8,8 +8,13 @@ export default interface TransactionModel {
   anchorString: string;
   transactionFeePaid: number;
 
-  // Normalized fee sohuld always be populated in core layer when core makes call to transactions endpoint.
-  // It may not be populated in blockchain service. This allows flexibility for the value to be computed on the spot or stored.
+  /**
+   * Normalized fee sohuld always be populated in core layer when core makes call to transactions endpoint.
+   * It may not be populated in blockchain service. This allows flexibility for the value to be computed on the spot or stored.
+   * To remove potentially dangerous behavior. Make a seperate model
+   * https://github.com/decentralized-identity/sidetree/issues/937
+   */
+
   normalizedTransactionFee?: number;
 
   writer: string;

tests/JasmineSidetreeErrorValidator.ts

@@ -47,10 +47,12 @@ export default class JasmineSidetreeErrorValidator {
     expectedContainedStringInMessage?: string
   ): Promise<void> {
     let validated: boolean = false;
+    let actualError;
 
     try {
       await functionToExecute();
     } catch (e) {
+      actualError = e;
       if (e instanceof SidetreeError) {
         expect(e.code).toEqual(expectedErrorCode);
 
@@ -63,7 +65,7 @@ export default class JasmineSidetreeErrorValidator {
     }
 
     if (!validated) {
-      fail(`Expected error '${expectedErrorCode}' did not occur.`);
+      fail(`Expected error '${expectedErrorCode}' did not occur. Instead got '${actualError.code}'`);
     }
   }
 }