Merge pull request #1027 from ergoplatform/release/3.2.1

Candidate for 3.2.1
ergoplatform · Feb 19, 2020 · 8a4d14b · 8a4d14b
2 parents d817053 + 93d11f0
commit 8a4d14b
Show file tree

Hide file tree

Showing 89 changed files with 854 additions and 729 deletions.
diff --git a/avldb/README.md b/avldb/README.md
@@ -1,4 +1,4 @@
-# AVL-IODB
+# AVL-DB
 
 To run benchmarks use command `sbt "project benchmarks" "jmh:run -i 3 -wi 3 -f1 -t1 -rf csv .*AVLTreeBatchPerformance"`
 

diff --git a/...b/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/AVLTreeBatchPerformance.scala b/...b/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/AVLTreeBatchPerformance.scala
@@ -26,7 +26,7 @@ object AVLTreeBatchPerformance extends {
 
     @Setup(Level.Iteration)
     def up: Unit = {
-      val (p, s, _) = getPersistentProverWithLSMStore(1000, proverCnt)
+      val (p, s, _) = persistentProverWithVersionedStore(1000, proverCnt)
       store = s
       prover = p
       operations = generateOps(proverCnt until (proverCnt + opsCnt))

diff --git a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala
@@ -33,7 +33,8 @@ object Helper {
     inserts ++ updates
   }
 
-  def getPersistentProverWithLSMStore(keepVersions: Int, baseOperationsCount: Int = 0): (Prover, LDBVersionedStore, VersionedLDBAVLStorage[Digest32]) = {
+  def persistentProverWithVersionedStore(keepVersions: Int,
+                                         baseOperationsCount: Int = 0): (Prover, LDBVersionedStore, VersionedLDBAVLStorage[Digest32]) = {
     val dir = java.nio.file.Files.createTempDirectory("bench_testing_" + scala.util.Random.alphanumeric.take(15)).toFile
     dir.deleteOnExit()
     val store = new LDBVersionedStore(dir, keepVersions = keepVersions)
@@ -64,7 +65,7 @@ object Helper {
     }
   }
 
-  def getProver(baseOperationsCount: Int = 0): BatchAVLProver[Digest32, HF] = {
+  def createProver(baseOperationsCount: Int = 0): BatchAVLProver[Digest32, HF] = {
     val prover = new BatchAVLProver[Digest32, HF](kl, Some(vl))
     if (baseOperationsCount > 0) {
       val step = 5000

diff --git a/avldb/benchmarks/src/test/scala/MemoryFootprint.scala b/avldb/benchmarks/src/test/scala/MemoryFootprint.scala
@@ -23,7 +23,7 @@ class MemoryFootprint extends Bench[Double] {
       using(sizes) config(
         exec.benchRuns -> 4,
         exec.independentSamples -> 2
-      ) in { getProver }
+      ) in { createProver }
     }
   }
 }
diff --git a/avldb/build.sbt b/avldb/build.sbt
@@ -5,8 +5,7 @@ name := "avldb"
 libraryDependencies ++= Seq(
   "javax.xml.bind" % "jaxb-api" % "2.4.0-b180830.0359",
   "ch.qos.logback" % "logback-classic" % "1.2.3",
-  "org.scorexfoundation" %% "scrypto" % "2.1.4",
-  "org.scorexfoundation" %% "iodb" % "0.3.2"
+  "org.scorexfoundation" %% "scrypto" % "2.1.4"
 )
 
 libraryDependencies ++= Seq(

diff --git a/avldb/src/main/java/scorex/ByteUtils.java b/avldb/src/main/java/scorex/ByteUtils.java
@@ -0,0 +1,45 @@
+package scorex;
+
+import java.util.Comparator;
+
+public class ByteUtils {
+    /**
+     * Compares primitive Byte Arrays.
+     * It uses unsigned binary comparation; so the byte with negative value is always higher than byte with non-negative value.
+     */
+    public static final Comparator<byte[]> BYTE_ARRAY_COMPARATOR = (o1, o2) -> compare(o1, o2);
+
+    public static int compare(byte[] o1, byte[] o2) {
+        final int len = Math.min(o1.length, o2.length);
+        for (int i = 0; i < len; i++) {
+            int b1 = o1[i] & 0xFF;
+            int b2 = o2[i] & 0xFF;
+            if (b1 != b2)
+                return b1 - b2;
+        }
+        return o1.length - o2.length;
+    }
+
+
+    public static int byteArrayHashCode(byte[] data) {
+        //do not use Arrays.hashCode, it generates too many collisions (31 is too low)
+        int h = 1;
+        for (byte b : data) {
+            h = h * (-1640531527) + b;
+        }
+        return h;
+    }
+
+    public static long getLong(byte[] buf, int pos) {
+        return
+                ((((long) buf[pos++]) << 56) |
+                        (((long) buf[pos++] & 0xFF) << 48) |
+                        (((long) buf[pos++] & 0xFF) << 40) |
+                        (((long) buf[pos++] & 0xFF) << 32) |
+                        (((long) buf[pos++] & 0xFF) << 24) |
+                        (((long) buf[pos++] & 0xFF) << 16) |
+                        (((long) buf[pos++] & 0xFF) << 8) |
+                        (((long) buf[pos] & 0xFF)));
+
+    }
+}
diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala
@@ -30,7 +30,7 @@ class VersionedLDBAVLStorage[D <: Digest](store: LDBVersionedStore,
   private val fixedSizeValueMode = nodeParameters.valueSize.isDefined
 
   override def rollback(version: ADDigest): Try[(ProverNodes[D], Int)] = Try {
-    store.rollback(version)
+    store.rollbackTo(version)
 
     val top = VersionedLDBAVLStorage.fetch[D](ADKey @@ store.get(TopNodeKey).get)(hf, store, nodeParameters)
     val topHeight = Ints.fromByteArray(store.get(TopNodeHeight).get)

diff --git a/avldb/src/main/scala/scorex/db/ByteArrayWrapper.scala b/avldb/src/main/scala/scorex/db/ByteArrayWrapper.scala
@@ -0,0 +1,55 @@
+package scorex.db
+
+
+import java.io.Serializable
+import java.nio.ByteBuffer
+import java.util
+
+import scorex.ByteUtils
+
+object ByteArrayWrapper {
+
+  def fromLong(id: Long): ByteArrayWrapper = {
+    val b = ByteBuffer.allocate(8)
+    b.putLong(0, id)
+    ByteArrayWrapper(b.array())
+  }
+}
+
+/**
+  * Wraps byte array and provides hashCode, equals and compare methods.
+  */
+case class ByteArrayWrapper(data: Array[Byte])
+  extends Serializable
+    with Comparable[ByteArrayWrapper]
+    with Ordered[ByteArrayWrapper] {
+
+  /** alternative constructor which takes array size and creates new empty array */
+  def this(size:Int) = this(new Array[Byte](size))
+
+  def size = data.length
+
+  require(data != null)
+
+  //TODO wrapped data immutable?
+
+  override def equals(o: Any): Boolean =
+    o.isInstanceOf[ByteArrayWrapper] &&
+      util.Arrays.equals(data, o.asInstanceOf[ByteArrayWrapper].data)
+
+  override def hashCode: Int = ByteUtils.byteArrayHashCode(data)
+
+  override def compareTo(o: ByteArrayWrapper): Int = ByteUtils.BYTE_ARRAY_COMPARATOR.compare(this.data, o.data)
+
+  override def compare(that: ByteArrayWrapper): Int = compareTo(that)
+
+  override def toString: String = {
+    val v = if (size == 8) {
+      //if size is 8, display as a number
+      ByteUtils.getLong(data, 0).toString + "L"
+    } else {
+      javax.xml.bind.DatatypeConverter.printHexBinary(data)
+    }
+    getClass.getSimpleName + "[" + v + "]"
+  }
+}
diff --git a/avldb/src/main/scala/scorex/db/LDBFactory.scala b/avldb/src/main/scala/scorex/db/LDBFactory.scala
@@ -48,7 +48,7 @@ case class StoreRegistry(factory: DBFactory) extends DBFactory with ScorexLoggin
 
     def delete(key: Array[Byte], options: WriteOptions): Snapshot = impl.delete(key, options)
 
-    def getSnapshot: Snapshot = impl.getSnapshot()
+    def getSnapshot: Snapshot = impl.getSnapshot
 
     def getApproximateSizes(ranges: Range*): Array[Long] = impl.getApproximateSizes(ranges: _*)
 

diff --git a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala
@@ -288,7 +288,7 @@ class LDBVersionedStore(val dir: File, val keepVersions: Int) extends KVStore {
   }
 
   // Rollback to the specified version: undo all changes done after specified version
-  def rollback(versionID: VersionID): Try[Unit] = Try {
+  def rollbackTo(versionID: VersionID): Try[Unit] = Try {
     lock.writeLock().lock()
     try {
       val versionIndex = versions.indexWhere(_.sameElements(versionID))

diff --git a/...rc/test/scala/scorex/crypto/authds/avltree/batch/AVLStorageWithPersistentProverSpec.scala b/...rc/test/scala/scorex/crypto/authds/avltree/batch/AVLStorageWithPersistentProverSpec.scala
@@ -62,9 +62,6 @@ class AVLStorageWithPersistentProverSpec extends PropSpec with Matchers {
   private val iMods32 = (0 until 100)
     .map(i => Insert(ADKey @@ Random.randomBytes(), ADValue @@ Array.fill(32)(i.toByte)))
 
-  private val iMods64 = (0 until 100)
-    .map(i => Insert(ADKey @@ Random.randomBytes(), ADValue @@ Array.fill(64)(i.toByte)))
-
   private val iMods128 = (0 until 100)
     .map(i => Insert(ADKey @@ Random.randomBytes(), ADValue @@ Array.fill(128)(i.toByte)))
 

diff --git a/...c/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala b/...c/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala
@@ -245,7 +245,7 @@ class VersionedLDBAVLStorageSpecification extends PropSpec
 
   }
 
-  def removeFromLargerSetSingleRandomElementTest(createStore: (Int) => LDBVersionedStore): Unit = {
+  def removeFromLargerSetSingleRandomElementTest(createStore: Int => LDBVersionedStore): Unit = {
     val minSetSize = 10000
     val maxSetSize = 200000
 

diff --git a/...cala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala b/...cala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala
@@ -18,12 +18,12 @@ class VersionedLDBAVLStorageStatefulSpecification extends PropSpec {
     .withMaxSize(50)
     .withMinSuccessfulTests(15)
 
-  property("IODBAVLStorage: rollback in stateful environment with LSM") {
-    WithLSM.property().check(params)
+  property("LDBAVLStorage: rollback in stateful environment") {
+    WithLDB.property().check(params)
   }
 }
 
-object WithLSM extends VersionedIODBAVLStorageStatefulCommands with TestHelper {
+object WithLDB extends VersionedLDBAVLStorageStatefulCommands with TestHelper {
 
   override protected val KL = 32
   override protected val VL = 8
@@ -34,7 +34,7 @@ object WithLSM extends VersionedIODBAVLStorageStatefulCommands with TestHelper {
   }
 }
 
-trait VersionedIODBAVLStorageStatefulCommands extends Commands { this: TestHelper =>
+trait VersionedLDBAVLStorageStatefulCommands extends Commands { this: TestHelper =>
 
   override type State = Operations
   override type Sut = PersistentBatchAVLProver[Digest32, HF]
@@ -161,7 +161,6 @@ trait VersionedIODBAVLStorageStatefulCommands extends Commands { this: TestHelpe
       require(digest1.sameElements(updatedDigest))
       ops.foreach(sut.performOneOperation)
       sut.checkTree(postProof = false)
-      val sameProof = sut.generateProofAndUpdateStorage()
       sut.checkTree(postProof = true)
       val updatedPostDigest = sut.digest
 

diff --git a/...tree/batch/IODBStorageSpecification.scala → ...batch/VersionedStorageSpecification.scala b/...tree/batch/IODBStorageSpecification.scala → ...batch/VersionedStorageSpecification.scala
@@ -9,7 +9,7 @@ import scorex.db.LDBVersionedStore
 
 import scala.collection.mutable.ArrayBuffer
 
-class IODBStorageSpecification extends PropSpec
+class VersionedStorageSpecification extends PropSpec
   with ScalaCheckPropertyChecks
   with ScalaCheckDrivenPropertyChecks
   with Matchers
@@ -30,14 +30,14 @@ class IODBStorageSpecification extends PropSpec
       store.update(nextVersion, Seq(), Seq(pair))
 
       if (version.isDefined) {
-        store.rollback(version.get)
+        store.rollbackTo(version.get)
         store.update(nextVersion, Seq(), Seq(pair))
       }
       version = Some(nextVersion)
       keys.foreach(k => store(k._1).sameElements(k._2) shouldBe true)
     }
   }
 
-  property("IODB with LSM") { storeTest(createVersionedStore()) }
+  property("LevelDB-based versioned storage") { storeTest(createVersionedStore()) }
 
 }
diff --git a/.../test/scala/scorex/crypto/authds/avltree/batch/benchmark/LDBVersionedStoreBenchmark.scala b/.../test/scala/scorex/crypto/authds/avltree/batch/benchmark/LDBVersionedStoreBenchmark.scala
@@ -22,7 +22,7 @@ object LDBVersionedStoreBenchmark extends App with FileHelper {
     val nextVersion = Longs.toByteArray(i)
     store.update(nextVersion, Seq(), mod)
     currentVersion.foreach(v => {
-      store.rollback(Longs.toByteArray(v))
+      store.rollbackTo(Longs.toByteArray(v))
       store.update(nextVersion, Seq(), mod)
     })
     currentVersion = Some(i)

diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/OOMTest.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/OOMTest.scala
@@ -4,12 +4,12 @@ import java.io.File
 import java.nio.file.Files
 
 import com.google.common.primitives.{Longs, Shorts}
-import io.iohk.iodb.ByteArrayWrapper
 import scorex.crypto.authds.{ADDigest, ADKey, ADValue}
 import scorex.crypto.authds.avltree.batch._
 import scorex.util.encode.Base16
 import scorex.crypto.hash.{Blake2b256, Digest32}
-import scorex.db.LDBVersionedStore
+import scorex.db.{ByteArrayWrapper, LDBVersionedStore}
+
 import scala.collection.immutable.SortedMap
 
 object OOMTest extends App {

diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/FileHelper.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/FileHelper.scala
@@ -7,7 +7,7 @@ import scala.util.Random
 trait FileHelper {
 
   def getRandomTempDir: File = {
-    val dir = java.nio.file.Files.createTempDirectory("avliodb_test_" + Random.alphanumeric.take(15).mkString).toFile
+    val dir = java.nio.file.Files.createTempDirectory("avldb_test_" + Random.alphanumeric.take(15).mkString).toFile
     dir.deleteOnExit()
     dir
   }

diff --git a/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala b/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala
@@ -76,7 +76,7 @@ class LDBVersionedStoreSpec extends PropSpec with Matchers {
     store.getWithFilter((_, _) => true).toSeq.length shouldBe 0
 
     var cnt = 0
-    store.processAll({ case (k, v) =>
+    store.processAll({ case (_, _) =>
       cnt = cnt + 1
     })
     cnt shouldBe 0

diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/BenchRunner.scala
@@ -3,6 +3,7 @@ package org.ergoplatform.bench
 import java.io.File
 
 import akka.actor.{ActorRef, ActorSystem}
+import org.ergoplatform.bench.misc.TempDir
 import org.ergoplatform.modifiers.ErgoPersistentModifier
 import org.ergoplatform.nodeView.history.ErgoHistory
 import org.ergoplatform.nodeView.history.storage.modifierprocessors.{FullBlockPruningProcessor, ToDownloadProcessor}

diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/CrawlerRunner.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/CrawlerRunner.scala
@@ -3,7 +3,7 @@ package org.ergoplatform.bench
 import java.io.File
 
 import akka.actor.ActorRef
-import org.ergoplatform.bench.misc.CrawlerConfig
+import org.ergoplatform.bench.misc.{CrawlerConfig, TempDir}
 import org.ergoplatform.http.api.{BlocksApiRoute, ErgoUtilsApiRoute, InfoApiRoute, TransactionsApiRoute}
 import org.ergoplatform.local.{ErgoMinerRef, ErgoStatsCollectorRef}
 import org.ergoplatform.mining.emission.EmissionRules

diff --git a/benchmarks/src/test/scala/org/ergoplatform/bench/misc/TempDir.scala b/benchmarks/src/test/scala/org/ergoplatform/bench/misc/TempDir.scala
@@ -1,4 +1,4 @@
-package org.ergoplatform.bench
+package org.ergoplatform.bench.misc
 
 import scorex.testkit.utils.FileUtils