From dac737e7591de3704b934f7411b08398861f7709 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Thu, 2 May 2024 15:49:07 +0300 Subject: [PATCH 1/3] i2155 --- .../nodeView/ErgoNodeViewHolder.scala | 2 +- .../nodeView/mempool/ErgoMemPool.scala | 29 +++++++++++--- .../nodeView/mempool/ErgoMemPoolSpec.scala | 38 +++++++++++++++++-- .../mempool/MempoolTransactionsTest.scala | 4 +- 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index e68b9d4693..19af8bba61 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -402,7 +402,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti .flatMap(extractTransactions) .filter(tx => !appliedTxs.exists(_.id == tx.id)) .map(tx => UnconfirmedTransaction(tx, None)) - memPool.remove(appliedTxs).put(rolledBackTxs) + memPool.removeWithDoubleSpends(appliedTxs).put(rolledBackTxs) } /** diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index df08cd4858..76f2d543d1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -108,13 +108,32 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, /** * Remove transaction from the pool */ - def remove(tx: ErgoTransaction): ErgoMemPool = { - log.debug(s"Removing transaction ${tx.id} from the mempool") - new ErgoMemPool(pool.remove(tx), updateStatsOnRemoval(tx), sortingOption) + def removeTxAndDoubleSpends(tx: ErgoTransaction): ErgoMemPool = { + def removeTx(mp: ErgoMemPool, tx: ErgoTransaction): ErgoMemPool = { + log.debug(s"Removing transaction ${tx.id} from the mempool") + new ErgoMemPool(mp.pool.remove(tx), mp.updateStatsOnRemoval(tx), sortingOption) + } + + val poolWithoutTx = removeTx(this, tx) + val doubleSpentTransactionIds = tx.inputs.flatMap(i => + poolWithoutTx.pool.inputs.get(i.boxId) + ).toSet + val doubleSpentTransactions = doubleSpentTransactionIds.flatMap { txId => + poolWithoutTx.pool.orderedTransactions.get(txId) + } + doubleSpentTransactions.foldLeft(poolWithoutTx) { case (pool, tx) => + removeTx(pool, tx.transaction) + } } - def remove(txs: TraversableOnce[ErgoTransaction]): ErgoMemPool = { - txs.foldLeft(this) { case (acc, tx) => acc.remove(tx) } + def removeWithDoubleSpends(txs: TraversableOnce[ErgoTransaction]): ErgoMemPool = { + txs.foldLeft(this) { case (memPool, tx) => + if (memPool.contains(tx.id)) { + memPool.removeTxAndDoubleSpends(tx) + } else { + memPool + } + } } /** diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala index 5ee89a9a4c..486c626803 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala @@ -1,11 +1,11 @@ package org.ergoplatform.nodeView.mempool import org.ergoplatform.{ErgoBoxCandidate, Input} -import org.ergoplatform.nodeView.mempool.ErgoMemPoolUtils.{SortingOption, ProcessingOutcome} +import org.ergoplatform.nodeView.mempool.ErgoMemPoolUtils.{ProcessingOutcome, SortingOption} import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} import org.ergoplatform.nodeView.state.wrapped.WrappedUtxoState import org.ergoplatform.settings.ErgoSettings -import org.ergoplatform.utils.ErgoTestHelpers +import org.ergoplatform.utils.{ErgoTestHelpers, RandomWrapper} import org.scalatest.flatspec.AnyFlatSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks import sigmastate.Values.{ByteArrayConstant, TrueLeaf} @@ -297,11 +297,41 @@ class ErgoMemPoolSpec extends AnyFlatSpec } pool.size shouldBe (family_depth + 1) * txs.size allTxs.foreach { tx => - pool = pool.remove(tx.transaction) + pool = pool.removeTxAndDoubleSpends(tx.transaction) } pool.size shouldBe 0 } + it should "correctly remove doublespents of a transaction from pool" in { + val (us, bh) = createUtxoState(settings) + val genesis = validFullBlock(None, us, bh) + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get + val boxes = wus.takeBoxes(4) + + val limit = 10000 + + val tx1 = validTransactionsFromBoxes(limit, boxes.take(1), new RandomWrapper) + ._1.map(tx => UnconfirmedTransaction(tx, None)).head + + val tx2 = validTransactionsFromBoxes(limit, boxes.takeRight(2), new RandomWrapper) + ._1.map(tx => UnconfirmedTransaction(tx, None)).head + + val tx3 = validTransactionsFromBoxes(limit, boxes.take(1), new RandomWrapper) + ._1.map(tx => UnconfirmedTransaction(tx, None)).head + + tx1.transaction.inputs.head.boxId shouldBe tx3.transaction.inputs.head.boxId + + var pool = ErgoMemPool.empty(settings) + Seq(tx2, tx3).foreach { tx => + pool = pool.put(tx) + } + + pool = pool.removeTxAndDoubleSpends(tx1.transaction) + pool.contains(tx1.transaction) shouldBe false + pool.contains(tx2.transaction) shouldBe true + pool.contains(tx3.transaction) shouldBe false + } + it should "return results take / getAll / getAllPrioritized sorted by priority" in { val feeProp = settings.chainSettings.monetary.feeProposition @@ -376,7 +406,7 @@ class ErgoMemPoolSpec extends AnyFlatSpec pool.stats.snapTakenTxns shouldBe MemPoolStatistics(System.currentTimeMillis(),0,System.currentTimeMillis()).snapTakenTxns allTxs.foreach { tx => - pool = pool.remove(tx.transaction) + pool = pool.removeTxAndDoubleSpends(tx.transaction) } pool.size shouldBe 0 pool.stats.takenTxns shouldBe (family_depth + 1) * txs.size diff --git a/src/test/scala/scorex/testkit/properties/mempool/MempoolTransactionsTest.scala b/src/test/scala/scorex/testkit/properties/mempool/MempoolTransactionsTest.scala index 891c854c7e..c8ccb676bd 100644 --- a/src/test/scala/scorex/testkit/properties/mempool/MempoolTransactionsTest.scala +++ b/src/test/scala/scorex/testkit/properties/mempool/MempoolTransactionsTest.scala @@ -84,7 +84,7 @@ trait MempoolTransactionsTest property("Size of mempool should decrease when removing a present transaction") { forAll(memPoolGenerator, unconfirmedTxSeqGenerator) { (mp: ErgoMemPool, unconfirmedTxs: Seq[UnconfirmedTransaction]) => val m: ErgoMemPool = mp.put(unconfirmedTxs) - val m2: ErgoMemPool = m.remove(unconfirmedTxs.headOption.get.transaction) + val m2: ErgoMemPool = m.removeTxAndDoubleSpends(unconfirmedTxs.headOption.get.transaction) m2.size shouldBe unconfirmedTxs.size - 1 } } @@ -92,7 +92,7 @@ trait MempoolTransactionsTest property("Size of mempool should not decrease when removing a non-present transaction") { forAll(memPoolGenerator, unconfirmedTxSeqGenerator, unconfirmedTxGenerator) { (mp: ErgoMemPool, unconfirmedTxs: Seq[UnconfirmedTransaction], unconfirmedTx: UnconfirmedTransaction) => val m: ErgoMemPool = mp.put(unconfirmedTxs) - val m2: ErgoMemPool = m.remove(unconfirmedTx.transaction) + val m2: ErgoMemPool = m.removeTxAndDoubleSpends(unconfirmedTx.transaction) m2.size shouldBe unconfirmedTxs.size } } From 1d034609897dccb6f9b0f5254ca7c09cd3bc80e4 Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Thu, 2 May 2024 21:12:53 +0300 Subject: [PATCH 2/3] lowering down default mempoolcleanup duration value --- .../org/ergoplatform/modifiers/history/header/Header.scala | 6 ++++-- src/main/resources/application.conf | 2 +- src/main/scala/org/ergoplatform/local/CleanupWorker.scala | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala index 092dcd219f..7f155c206f 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala @@ -53,8 +53,10 @@ case class Header(override val version: Header.Version, override val extensionRoot: Digest32, powSolution: AutolykosSolution, override val votes: Array[Byte], //3 bytes - override val sizeOpt: Option[Int] = None) extends HeaderWithoutPow(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp, - nBits, height, extensionRoot, votes) with PreHeader with BlockSection { + override val sizeOpt: Option[Int] = None) extends + HeaderWithoutPow(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp, + nBits, height, extensionRoot, votes) + with PreHeader with BlockSection { override def serializedId: Array[Header.Version] = Algos.hash(bytes) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 37c2406a53..d9160f10a0 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -80,7 +80,7 @@ ergo { # Interval for mempool transaction re-check. We check transaction when it is entering the mempool, and then # re-check it every interval value - mempoolCleanupDuration = 20m + mempoolCleanupDuration = 30s # Mempool transaction sorting scheme ("random", "bySize", or "byExecutionCost") mempoolSorting = "random" diff --git a/src/main/scala/org/ergoplatform/local/CleanupWorker.scala b/src/main/scala/org/ergoplatform/local/CleanupWorker.scala index 0db0c9fce8..ecc9c16e00 100644 --- a/src/main/scala/org/ergoplatform/local/CleanupWorker.scala +++ b/src/main/scala/org/ergoplatform/local/CleanupWorker.scala @@ -66,8 +66,8 @@ class CleanupWorker(nodeViewHolderRef: ActorRef, val now = System.currentTimeMillis() - val allPoolTxs = mempool.getAllPrioritized // Check transactions sorted by priority. Parent transaction comes before its children. + val allPoolTxs = mempool.getAllPrioritized val txsToValidate = allPoolTxs.filter { utx => (now - utx.lastCheckedTime) > TimeLimit }.toList From f5f514421e911ab496326a611cc2b268a083da5a Mon Sep 17 00:00:00 2001 From: Alexander Chepurnoy Date: Wed, 22 May 2024 00:35:06 +0300 Subject: [PATCH 3/3] improved comments --- .../org/ergoplatform/modifiers/history/header/Header.scala | 6 ++---- src/main/resources/application.conf | 2 +- .../org/ergoplatform/nodeView/mempool/ErgoMemPool.scala | 7 +++++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala index e486e35fa5..c3bb9c24d8 100644 --- a/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala +++ b/ergo-core/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala @@ -54,10 +54,8 @@ case class Header(override val version: Header.Version, override val extensionRoot: Digest32, powSolution: AutolykosSolution, override val votes: Array[Byte], //3 bytes - override val sizeOpt: Option[Int] = None) extends - HeaderWithoutPow(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp, - nBits, height, extensionRoot, votes) - with PreHeader with BlockSection { + override val sizeOpt: Option[Int] = None) extends HeaderWithoutPow(version, parentId, ADProofsRoot, stateRoot, transactionsRoot, timestamp, + nBits, height, extensionRoot, votes) with PreHeader with BlockSection { override def serializedId: Array[Header.Version] = Algos.hash(bytes) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index d9160f10a0..cbc5dd4382 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -79,7 +79,7 @@ ergo { mempoolCapacity = 1000 # Interval for mempool transaction re-check. We check transaction when it is entering the mempool, and then - # re-check it every interval value + # re-check it every interval value (but only on new block arrival) mempoolCleanupDuration = 30s # Mempool transaction sorting scheme ("random", "bySize", or "byExecutionCost") diff --git a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala index 76f2d543d1..58f008bfec 100644 --- a/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala +++ b/src/main/scala/org/ergoplatform/nodeView/mempool/ErgoMemPool.scala @@ -106,7 +106,7 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, } /** - * Remove transaction from the pool + * Remove transaction from the pool along with its double-spends */ def removeTxAndDoubleSpends(tx: ErgoTransaction): ErgoMemPool = { def removeTx(mp: ErgoMemPool, tx: ErgoTransaction): ErgoMemPool = { @@ -126,9 +126,12 @@ class ErgoMemPool private[mempool](private[mempool] val pool: OrderedTxPool, } } + /** + * Remove provided transactions and their doublespends from the pool + */ def removeWithDoubleSpends(txs: TraversableOnce[ErgoTransaction]): ErgoMemPool = { txs.foldLeft(this) { case (memPool, tx) => - if (memPool.contains(tx.id)) { + if (memPool.contains(tx.id)) { // tx could be removed earlier in this loop as double-spend of another tx memPool.removeTxAndDoubleSpends(tx) } else { memPool