Moving window normalized fee

docs/bitcoin.md

@@ -5,10 +5,9 @@
 
 ### Protocol parameters
 
-| Protocol parameters                  | Description                              |
-| ------------------------------------ | ---------------------------------------- |
-| minimumValueTimeLockDurationInBlocks | TODO |
-| maximumValueTimeLockDurationInBlocks | TODO |
+| Protocol parameters                  | Description                                              |
+| ------------------------------------ | ---------------------------------------------------------|
+| valueTimeLockDurationInBlocks        | The duration which a value time lock is required to have |
 
 ### Configuration parameters
 * valueTimeLockUpdateEnabled

lib/bitcoin/BitcoinProcessor.ts

@@ -4,6 +4,7 @@ import BitcoinBlockModel from './models/BitcoinBlockModel';
 import BitcoinClient from './BitcoinClient';
 import BitcoinServiceStateModel from './models/BitcoinServiceStateModel';
 import BitcoinTransactionModel from './models/BitcoinTransactionModel';
+import BitcoinVersionModel from './models/BitcoinVersionModel';
 import BlockMetadata from './models/BlockMetadata';
 import ErrorCode from './ErrorCode';
 import IBitcoinConfig from './IBitcoinConfig';
@@ -14,7 +15,6 @@ import MongoDbBlockMetadataStore from './MongoDbBlockMetadataStore';
 import MongoDbLockTransactionStore from './lock/MongoDbLockTransactionStore';
 import MongoDbServiceStateStore from '../common/MongoDbServiceStateStore';
 import MongoDbTransactionStore from '../common/MongoDbTransactionStore';
-import ProtocolParameters from './ProtocolParameters';
 import RequestError from './RequestError';
 import ResponseStatus from '../common/enums/ResponseStatus';
 import ServiceInfoProvider from '../common/ServiceInfoProvider';
@@ -28,7 +28,6 @@ import TransactionModel from '../common/models/TransactionModel';
 import TransactionNumber from './TransactionNumber';
 import ValueTimeLockModel from '../common/models/ValueTimeLockModel';
 import VersionManager from './VersionManager';
-import VersionModel from '../common/models/VersionModel';
 
 /**
  * Object representing a blockchain time and hash
@@ -95,8 +94,8 @@ export default class BitcoinProcessor {
   /** at least 100 blocks per page unless reaching the last block */
   private static readonly pageSizeInBlocks = 100;
 
-  public constructor (private config: IBitcoinConfig, versionModels: VersionModel[]) {
-    this.versionManager = new VersionManager(versionModels);
+  public constructor (private config: IBitcoinConfig, versionModels: BitcoinVersionModel[]) {
+    this.versionManager = new VersionManager(versionModels, config);
 
     this.genesisBlockNumber = config.genesisBlockNumber;
 
@@ -127,9 +126,7 @@ export default class BitcoinProcessor {
     this.lockResolver =
       new LockResolver(
         this.versionManager,
-        this.bitcoinClient,
-        ProtocolParameters.minimumValueTimeLockDurationInBlocks,
-        ProtocolParameters.maximumValueTimeLockDurationInBlocks);
+        this.bitcoinClient);
 
     this.mongoDbLockTransactionStore = new MongoDbLockTransactionStore(config.mongoDbConnectionString, config.databaseName);
 
@@ -144,15 +141,15 @@ export default class BitcoinProcessor {
       config.valueTimeLockUpdateEnabled,
       BitcoinClient.convertBtcToSatoshis(config.valueTimeLockAmountInBitcoins), // Desired lock amount in satoshis
       BitcoinClient.convertBtcToSatoshis(valueTimeLockTransactionFeesInBtc),    // Txn Fees amount in satoshis
-      ProtocolParameters.maximumValueTimeLockDurationInBlocks                   // Desired lock duration in blocks
+      this.versionManager
     );
   }
 
   /**
    * Initializes the Bitcoin processor
    */
   public async initialize () {
-    await this.versionManager.initialize();
+    await this.versionManager.initialize(this.blockMetadataStore);
     await this.serviceStateStore.initialize();
     await this.blockMetadataStore.initialize();
     await this.transactionStore.initialize();
@@ -256,10 +253,11 @@ export default class BitcoinProcessor {
 
     // Write the block metadata to DB.
     const timer = timeSpan(); // Start timer to measure time taken to write block metadata.
-    await this.blockMetadataStore.add(validatedBlocks);
-    console.info(`Inserted metadata of ${validatedBlocks.length} blocks to DB. Duration: ${timer.rounded()} ms.`);
 
-    this.lastProcessedBlock = lastBlockInfo;
+    // ValidatedBlocks are in descending order, this flips that and make it ascending by height for the purpose of normalized fee calculation
+    const validatedBlocksOrderedByHeight = validatedBlocks.reverse();
+    await this.writeBlocksToMetadataStoreWithFee(validatedBlocksOrderedByHeight);
+    console.info(`Inserted metadata of ${validatedBlocks.length} blocks to DB. Duration: ${timer.rounded()} ms.`);
     console.log('finished fast processing');
   }
 
@@ -276,12 +274,13 @@ export default class BitcoinProcessor {
           {
             height: block.height,
             hash: block.hash,
+            normalizedFee: -1, // set to -1 to mark that it is not calculated yet
             totalFee: BitcoinProcessor.getBitcoinBlockTotalFee(block),
             transactionCount: block.transactions.length,
             previousHash: block.previousHash
           }
         );
-        await this.processSidetreeTransactionsInBlock(block);
+        await this.processSidetreeTransactionsInBlock(block, -1);
       }
     }
   }
@@ -351,14 +350,14 @@ export default class BitcoinProcessor {
    * Iterates through the transactions within the given block and process the sidetree transactions
    * @param block the block to process
    */
-  private async processSidetreeTransactionsInBlock (block: BitcoinBlockModel) {
+  private async processSidetreeTransactionsInBlock (block: BitcoinBlockModel, normalizedFee: number) {
     const transactions = block.transactions;
     // iterate through transactions
     for (let transactionIndex = 0; transactionIndex < transactions.length; transactionIndex++) {
       const transaction = transactions[transactionIndex];
 
       try {
-        const sidetreeTxToAdd = await this.getSidetreeTransactionModelIfExist(transaction, transactionIndex, block.height);
+        const sidetreeTxToAdd = await this.getSidetreeTransactionModelIfExist(transaction, transactionIndex, block.height, normalizedFee);
 
         // If there are transactions found then add them to the transaction store
         if (sidetreeTxToAdd) {
@@ -524,19 +523,45 @@ export default class BitcoinProcessor {
   }
 
   /**
-   * Return proof-of-fee value for a particular block.
+   * Modifies the given array and update the normalized fees, then write to block metadata store.
+   * @param blocks the ordered block metadata to set the normalized fee for.
    */
-  public async getNormalizedFee (block: number): Promise<TransactionFeeModel> {
+  private async writeBlocksToMetadataStoreWithFee (blocks: BlockMetadata[]) {
+    for (const block of blocks) {
+
+      const normalizedFee = (await this.getNormalizedFee(block.height)).normalizedTransactionFee;
+      block.normalizedFee = normalizedFee;
 
+      // update normalized fee in transactions
+      const transactions = await this.transactionStore.getTransactionsStartingFrom(block.height, block.height + 1);
+      await this.transactionStore.removeTransactionByTransactionTimeHash(block.hash);
+      for (const transaction of transactions) {
+        transaction.normalizedTransactionFee = normalizedFee;
+        await this.transactionStore.addTransaction(transaction);
+      }
+
+      this.blockMetadataStore.add([block]);
+      this.lastProcessedBlock = {
+        hash: block.hash,
+        height: block.height,
+        previousHash: block.previousHash
+      };
+    }
+  }
+
+  /**
+   * Calculate and return proof-of-fee value for a particular block.
+   * @param block The block height to get normalized fee for
+   */
+  public async getNormalizedFee (block: number): Promise<TransactionFeeModel> {
     if (block < this.genesisBlockNumber) {
       const error = `The input block number must be greater than or equal to: ${this.genesisBlockNumber}`;
       console.error(error);
       throw new RequestError(ResponseStatus.BadRequest, SharedErrorCode.BlockchainTimeOutOfRange);
     }
+    const normalizedTransactionFee = await this.versionManager.getFeeCalculator(block).getNormalizedFee(block);
 
-    const normalizedTransactionFee = this.versionManager.getFeeCalculator(block).getNormalizedFee(block);
-
-    return { normalizedTransactionFee };
+    return { normalizedTransactionFee: normalizedTransactionFee };
   }
 
   /**
@@ -767,7 +792,9 @@ export default class BitcoinProcessor {
         `Previous hash from blockchain: ${blockData.previousHash}. Expected value: ${previousBlockHash}`);
     }
 
-    await this.processSidetreeTransactionsInBlock(blockData);
+    const normalizedFee = (await this.getNormalizedFee(blockHeight)).normalizedTransactionFee;
+
+    await this.processSidetreeTransactionsInBlock(blockData, normalizedFee);
 
     // Compute the total fee paid and total transaction count.
     const transactionCount = blockData.transactions.length;
@@ -777,6 +804,7 @@ export default class BitcoinProcessor {
       hash: blockHash,
       height: blockHeight,
       previousHash: blockData.previousHash,
+      normalizedFee,
       totalFee,
       transactionCount
     };
@@ -787,21 +815,21 @@ export default class BitcoinProcessor {
   private async getSidetreeTransactionModelIfExist (
     transaction: BitcoinTransactionModel,
     transactionIndex: number,
-    transactionBlock: number): Promise<TransactionModel | undefined> {
+    transactionBlock: number,
+    normalizedFee: number): Promise<TransactionModel | undefined> {
 
     const sidetreeData = await this.sidetreeTransactionParser.parse(transaction);
 
     if (sidetreeData) {
       const transactionFeePaid = await this.bitcoinClient.getTransactionFeeInSatoshis(transaction.id);
-      const normalizedFeeModel = await this.getNormalizedFee(transactionBlock);
 
       return {
         transactionNumber: TransactionNumber.construct(transactionBlock, transactionIndex),
         transactionTime: transactionBlock,
         transactionTimeHash: transaction.blockHash,
         anchorString: sidetreeData.data,
         transactionFeePaid: transactionFeePaid,
-        normalizedTransactionFee: normalizedFeeModel.normalizedTransactionFee,
+        normalizedTransactionFee: normalizedFee,
         writer: sidetreeData.writer
       };
     }

lib/bitcoin/VersionManager.ts

@@ -1,37 +1,47 @@
 import BitcoinErrorCode from './ErrorCode';
+import BitcoinVersionModel from './models/BitcoinVersionModel';
+import IBitcoinConfig from './IBitcoinConfig';
+import IBlockMetadataStore from './interfaces/IBlockMetadataStore';
 import IFeeCalculator from './interfaces/IFeeCalculator';
+import ProtocolParameters from './models/ProtocolParameters';
 import SidetreeError from '../common/SidetreeError';
-import VersionModel from '../common/models/VersionModel';
 
 /**
  * The class that handles code versioning.
  */
 export default class VersionManager {
   // Reverse sorted implementation versions. ie. latest version first.
-  private versionsReverseSorted: VersionModel[];
+  private versionsReverseSorted: BitcoinVersionModel[];
 
   private feeCalculators: Map<string, IFeeCalculator>;
+  private protocolParameters: Map<string, ProtocolParameters>;
 
-  public constructor (versions: VersionModel[]) {
+  public constructor (versions: BitcoinVersionModel[], private config: IBitcoinConfig) {
     // Reverse sort versions.
     this.versionsReverseSorted = versions.sort((a, b) => b.startingBlockchainTime - a.startingBlockchainTime);
 
     this.feeCalculators = new Map();
+    this.protocolParameters = new Map();
   }
 
   /**
    * Loads all the implementation versions.
    */
-  public async initialize () {
+  public async initialize (
+    blockMetadataStore: IBlockMetadataStore
+  ) {
     // NOTE: In principal each version of the interface implementations can have different constructors,
     // but we currently keep the constructor signature the same as much as possible for simple instance construction,
     // but it is not inherently "bad" if we have to have conditional constructions for each if we have to.
     for (const versionModel of this.versionsReverseSorted) {
       const version = versionModel.version;
+      const initialNormalizedFee = versionModel.protocolParameters.initialNormalizedFee;
+
+      this.protocolParameters.set(version, versionModel.protocolParameters);
 
       /* tslint:disable-next-line */
       const FeeCalculator = await this.loadDefaultExportsForVersion(version, 'NormalizedFeeCalculator');
-      const feeCalculator = new FeeCalculator();
+      const feeCalculator = new FeeCalculator(blockMetadataStore, this.config.genesisBlockNumber, initialNormalizedFee);
       this.feeCalculators.set(version, feeCalculator);
     }
   }
@@ -45,6 +55,15 @@ export default class VersionManager {
     return feeCalculator;
   }
 
+  /**
+   * Gets the corresponding version of the lock duration based on the given block height.
+   */
+  public getLockDurationInBlocks (blockHeight: number): number {
+    const version = this.getVersionString(blockHeight);
+    const protocolParameter = this.protocolParameters.get(version)!;
+    return protocolParameter.valueTimeLockDurationInBlocks;
+  }
+
   /**
    * Gets the corresponding implementation version string given the blockchain time.
    */

lib/bitcoin/interfaces/IFeeCalculator.ts

@@ -5,5 +5,5 @@ export default interface IFeeCalculator {
   /**
    * Returns the fee for a particular block height.
    */
-  getNormalizedFee (block: number): number;
+  getNormalizedFee (block: number): Promise<number>;
 }

lib/bitcoin/lock/LockMonitor.ts

@@ -10,6 +10,7 @@ import SavedLockModel from '../models/SavedLockedModel';
 import SavedLockType from '../enums/SavedLockType';
 import SidetreeError from '../../common/SidetreeError';
 import ValueTimeLockModel from './../../common/models/ValueTimeLockModel';
+import VersionManager from '../VersionManager';
 
 /* global NodeJS */
 
@@ -38,8 +39,8 @@ export default class LockMonitor {
 
   /**
    * Constructor for LockMonitor.
-   * @param valueTimeLockUpdateEnabled When this parameter is set to `false`, parameters `lockPeriodInBlocks`,
-   *                                   `transactionFeesAmountInSatoshis` and `desiredLockAmountInSatoshis` will be ignored.
+   * @param valueTimeLockUpdateEnabled When this parameter is set to `false`, parameters `transactionFeesAmountInSatoshis`
+   *                                   and `desiredLockAmountInSatoshis` will be ignored.
    */
   constructor (
     private bitcoinClient: BitcoinClient,
@@ -49,7 +50,7 @@ export default class LockMonitor {
     private valueTimeLockUpdateEnabled: boolean,
     private desiredLockAmountInSatoshis: number,
     private transactionFeesAmountInSatoshis: number,
-    private lockPeriodInBlocks: number) {
+    private versionManager: VersionManager) {
 
     if (!Number.isInteger(desiredLockAmountInSatoshis)) {
       throw new SidetreeError(ErrorCode.LockMonitorDesiredLockAmountIsNotWholeNumber, `${desiredLockAmountInSatoshis}`);
@@ -271,7 +272,8 @@ export default class LockMonitor {
     console.info(LogColor.lightBlue(`Current wallet balance: ${LogColor.green(walletBalance)}`));
     console.info(LogColor.lightBlue(`Creating a new lock for amount: ${LogColor.green(totalLockAmount)} satoshis.`));
 
-    const lockTransaction = await this.bitcoinClient.createLockTransaction(totalLockAmount, this.lockPeriodInBlocks);
+    const height = await this.bitcoinClient.getCurrentBlockHeight();
+    const lockTransaction = await this.bitcoinClient.createLockTransaction(totalLockAmount, this.versionManager.getLockDurationInBlocks(height));
 
     return this.saveThenBroadcastTransaction(lockTransaction, SavedLockType.Create, desiredLockAmountInSatoshis);
   }
@@ -353,11 +355,12 @@ export default class LockMonitor {
     const currentLockIdentifier = LockIdentifierSerializer.deserialize(currentValueTimeLock.identifier);
     const currentLockDuration = currentValueTimeLock.unlockTransactionTime - currentValueTimeLock.lockTransactionTime;
 
+    const newLockDuration = this.versionManager.getLockDurationInBlocks(currentValueTimeLock.unlockTransactionTime);
     const relockTransaction =
       await this.bitcoinClient.createRelockTransaction(
         currentLockIdentifier.transactionId,
         currentLockDuration,
-        this.lockPeriodInBlocks);
+        newLockDuration);
 
     // If the transaction fee is making the relock amount less than the desired amount
     if (currentValueTimeLock.amountLocked - relockTransaction.transactionFee < desiredLockAmountInSatoshis) {

lib/bitcoin/lock/LockResolver.ts

@@ -26,9 +26,7 @@ export default class LockResolver {
 
   constructor (
     private versionManager: VersionManager,
-    private bitcoinClient: BitcoinClient,
-    private minimumLockDurationInBlocks: number,
-    private maximumLockDurationInBlocks: number) {
+    private bitcoinClient: BitcoinClient) {
   }
 
   /**
@@ -87,15 +85,17 @@ export default class LockResolver {
     // (C). verify that the lock duration is valid
     const unlockAtBlock = lockStartBlock + scriptVerifyResult.lockDurationInBlocks!;
 
+    const lockDurationInBlocks = this.versionManager.getLockDurationInBlocks(lockStartBlock);
+
     if (!this.isLockDurationValid(lockStartBlock, unlockAtBlock)) {
       throw new SidetreeError(
         ErrorCode.LockResolverDurationIsInvalid,
         // eslint-disable-next-line max-len
-        `Lock start block: ${lockStartBlock}. Unlock block: ${unlockAtBlock}. Allowed range: [${this.minimumLockDurationInBlocks} - ${this.maximumLockDurationInBlocks}.]`
+        `Lock start block: ${lockStartBlock}. Unlock block: ${unlockAtBlock}. Invalid duration: ${unlockAtBlock - lockStartBlock}. Allowed duration: ${lockDurationInBlocks}`
       );
     }
 
-    const normalizedFee = this.versionManager.getFeeCalculator(lockStartBlock).getNormalizedFee(lockStartBlock);
+    const normalizedFee = await this.versionManager.getFeeCalculator(lockStartBlock).getNormalizedFee(lockStartBlock);
 
     return {
       identifier: serializedLockIdentifier,
@@ -192,7 +192,8 @@ export default class LockResolver {
     //  lock-duration: unlockBlock - startBlock = 10 blocks
 
     const lockDuration = unlockBlock - startBlock;
+    const lockDurationInBlocks = this.versionManager.getLockDurationInBlocks(startBlock);
 
-    return lockDuration >= this.minimumLockDurationInBlocks && lockDuration <= this.maximumLockDurationInBlocks;
+    return lockDuration === lockDurationInBlocks;
   }
 }