Merge pull request #1049 from Cryptonomic/fix-build-indexing

Range indexing and project build fixes

conseil-common/src/main/resources/reference.conf

@@ -24,7 +24,7 @@ platforms: [
   {
     name: "tezos"
 
-    network: "mainnet"
+    network: "granadanet"
     network: ${?CONSEIL_XTZ_NETWORK}
 
     enabled: true
@@ -34,10 +34,10 @@ platforms: [
       protocol: "https"
       protocol: ${?CONSEIL_XTZ_NODE_PROTOCOL}
 
-      hostname: "nautilus.cryptonomic.tech"
+      hostname: "tezos-granada.cryptonomic-infra.tech"
       hostname: ${?CONSEIL_XTZ_NODE_HOSTNAME}
 
-      port: 8732
+      port: 443
       port: ${?CONSEIL_XTZ_NODE_PORT}
 
       path-prefix: ""

conseil-lorre/src/main/resources/application.conf

@@ -36,8 +36,8 @@ lorre {
   enabled-features {
     block-rights-fetching-is-on: true
     block-rights-fetching-is-on: ${?CONSEIL_LORRE_BLOCK_RIGHTS_FETCHING_ENABLED}
-    metadata-fetching-is-on: false
-    metadata-fetching-is-on: ${?CONSEIL_LORRE_METADATA_FETCHING_ENABLED}
+    metadata-fetching-is-on: true
+    metadata-fetching-is-on: ${?CONSEIL_LORRE_BLOCK_RIGHTS_FETCHING_ENABLED}
     fork-handling-is-on: false
     fork-handling-is-on: ${?CONSEIL_LORRE_FORK_DETECTION_ENABLED}
     registered-tokens-is-on: false

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/ConfigUtil.scala

@@ -0,0 +1,33 @@
+package tech.cryptonomic.conseil.indexer.config
+
+import com.github.ghik.silencer.silent
+import pureconfig.generic.EnumCoproductHint
+import scopt.Read
+
+import scala.util.Try
+
+private[config] object ConfigUtil {
+
+  object Depth {
+    implicit class DepthOps(val value: String) {
+      def toDepth = value match {
+        case "-1" | "all" | "everything" => Some(Everything)
+        case "0" | "new" | "newest" => Some(Newest)
+        case Natural(n) => Some(Custom(n))
+        case _ => None
+      }
+    }
+
+    /* used by scopt to parse the depth object */
+    @silent("private val depthRead in object ConfigUtil is never used")
+    implicit val depthRead: Read[Option[Depth]] = Read.reads(_.toDepth)
+    @silent("local val depthHint in object ConfigUtil is never used")
+    implicit val depthHint: EnumCoproductHint[Depth] = new EnumCoproductHint[Depth]
+  }
+
+  /** Used to pattern match on natural numbers */
+  object Natural {
+    def unapply(s: String): Option[Int] = Try(s.toInt).filter(_ > 0).toOption
+  }
+
+}

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/LorreAppConfig.scala

@@ -5,10 +5,11 @@ import com.typesafe.config.Config
 import pureconfig._
 import pureconfig.error.{ConfigReaderFailure, ConfigReaderFailures, ConfigValueLocation, ThrowableFailure}
 import pureconfig.generic.auto._
-import pureconfig.generic.{EnumCoproductHint, FieldCoproductHint}
+import pureconfig.generic.FieldCoproductHint
 import scopt.{OptionParser, Read}
 import tech.cryptonomic.conseil.common.config.Platforms._
 import tech.cryptonomic.conseil.common.config.{PlatformConfiguration => _, _}
+import tech.cryptonomic.conseil.indexer.config.ConfigUtil.Depth._
 
 import scala.util.{Success, Try}
 
@@ -17,14 +18,6 @@ trait LorreAppConfig {
   import LorreAppConfig.Loaders._
   import LorreAppConfig._
 
-  /* used by scopt to parse the depth object */
-  implicit private val depthRead: Read[Option[Depth]] = Read.reads {
-    case "-1" | "all" => Some(Everything)
-    case "0" | "new" => Some(Newest)
-    case Natural(n) => Some(Custom(n))
-    case _ => None
-  }
-
   /* used by scopt to parse the blockchain platform object */
   implicit private val platformRead: Read[BlockchainPlatform] = Read.reads { str =>
     Try(BlockchainPlatform.fromString(str)) match {
@@ -80,17 +73,29 @@ trait LorreAppConfig {
     def readArgs(args: Array[String]): ConfigReader.Result[ArgumentsConfig] =
       argsParser.parse(args, ArgumentsConfig()).toRight[ConfigReaderFailures](sys.exit(1))
 
-    @silent("local val depthHint in method loadApplicationConfiguration is never used")
-    implicit val depthHint: EnumCoproductHint[Depth] = new EnumCoproductHint[Depth]
     @silent("local val chainEventHint in method loadApplicationConfiguration is never used")
     implicit val chainEventHint: FieldCoproductHint[ChainEvent] = new FieldCoproductHint[ChainEvent]("type") {
       override def fieldValue(name: String): String = name.head.toLower +: name.tail
     }
+    @silent("local val lorreConfigHint in method loadApplicationConfiguration is never used")
+    implicit val lorreConfigHint = new FieldCoproductHint[LorreConfigurationHelper]("type") {
+      override protected def fieldValue(name: String): String = name.take(name.length - "Helper".length).toLowerCase
+    }
+    @silent("local val lorreConfigReader in method loadApplicationConfiguration is never used")
+    implicit val lorreConfigReader: ConfigReader[LorreConfiguration] =
+      ConfigReader[LorreConfigurationHelper].map(_.toConf)
 
     val loadedConf = for {
       args <- readArgs(commandLineArgs)
       ArgumentsConfig(depth, verbose, headHash, platform, network) = args
-      lorre <- loadConfig[LorreConfiguration](namespace = "lorre").map(_.copy(depth = depth, headHash = headHash))
+      lorre <- loadConfig[LorreConfiguration](namespace = "lorre").map { conf =>
+        /** In general, [[ArgumentsConfig]] should take precedence over [[LorreConfiguration]] */
+        if (depth == Newest && headHash.isDefined) conf.copy(headHash = headHash)
+        else if (depth != Newest) {
+          if (!headHash.isDefined) conf.copy(depth = depth)
+          else conf.copy(depth = depth, headHash = headHash)
+        } else conf
+      }
       nodeRequests <- loadConfig[NetworkCallsConfiguration]("lorre")
       platform <- loadPlatformConfiguration(platform.name, network)
       streamingClient <- loadAkkaStreamingClientConfig(namespace = "akka.streaming-client")
@@ -128,11 +133,6 @@ object LorreAppConfig {
       verbose: VerboseOutput
   )
 
-  /** Used to pattern match on natural numbers */
-  private[config] object Natural {
-    def unapply(s: String): Option[Int] = util.Try(s.toInt).filter(_ > 0).toOption
-  }
-
   private[config] object Loaders extends PlatformConfigurationHint {
 
     /*** Reads a specific platform configuration based on given 'platform' and 'network' */

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/config/LorreConfiguration.scala

@@ -1,6 +1,7 @@
 package tech.cryptonomic.conseil.indexer.config
 
 import tech.cryptonomic.conseil.common.config.ChainEvent
+import tech.cryptonomic.conseil.indexer.config.ConfigUtil.Depth._
 
 import scala.concurrent.duration.FiniteDuration
 
@@ -12,7 +13,7 @@ final case class NetworkCallsConfiguration(
 )
 
 /** generic configuration for the lorre */
-final case class LorreConfiguration(
+case class LorreConfiguration(
     sleepInterval: FiniteDuration,
     bootupRetryInterval: FiniteDuration,
     bootupConnectionCheckTimeout: FiniteDuration,
@@ -28,6 +29,39 @@ final case class LorreConfiguration(
     enabledFeatures: Features
 )
 
+final case class LorreConfigurationHelper(
+    sleepInterval: FiniteDuration,
+    bootupRetryInterval: FiniteDuration,
+    bootupConnectionCheckTimeout: FiniteDuration,
+    feeUpdateInterval: Int,
+    feesAverageTimeWindow: FiniteDuration,
+    depth: String,
+    headHash: Option[String],
+    chainEvents: List[ChainEvent],
+    blockRightsFetching: BakingAndEndorsingRights,
+    tokenContracts: TokenContracts,
+    metadataFetching: TzipMetadata,
+    forkHandling: ForkHandling,
+    enabledFeatures: Features
+) {
+  def toConf: LorreConfiguration =
+    new LorreConfiguration(
+      sleepInterval,
+      bootupRetryInterval,
+      bootupConnectionCheckTimeout,
+      feeUpdateInterval,
+      feesAverageTimeWindow,
+      depth.toDepth.getOrElse(Newest),
+      headHash,
+      chainEvents,
+      blockRightsFetching,
+      tokenContracts,
+      metadataFetching,
+      forkHandling,
+      enabledFeatures
+    )
+}
+
 /** configuration for fetching baking and endorsing rights */
 final case class BakingAndEndorsingRights(
     initDelay: FiniteDuration,

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosDatabaseOperations.scala

@@ -877,9 +877,13 @@ object TezosDatabaseOperations extends ConseilLogSupport {
   def readRegisteredTokensJsonFile(network: String): Option[List[RegisteredToken]] = {
     import io.circe.parser.decode
     import RegisteredTokensFetcher.decoder
+    import java.io.{BufferedReader, InputStreamReader}
+
+    val reader =
+      new BufferedReader(new InputStreamReader(getClass.getResourceAsStream(s"/registered_tokens/$network.json")))
+    val content = Stream.continually(reader.readLine).takeWhile(_ != null).mkString
+    reader.close() // TODO: loose effect
 
-    val file = getClass.getResource(s"/registered_tokens/$network.json")
-    val content = Source.fromFile(file.toURI).getLines.mkString
     decode[List[RegisteredToken]](content) match {
       case Left(error) =>
         logger.error(s"Something wrong with registered tokens file $error")

conseil-lorre/src/main/scala/tech/cryptonomic/conseil/indexer/tezos/TezosIndexer.scala

@@ -331,7 +331,7 @@ object TezosIndexer extends ConseilLogSupport {
 
     val ignoreProcessFailuresOrigin: Option[String] = sys.env.get(LORRE_FAILURE_IGNORE_VAR)
     val ignoreProcessFailures: Boolean =
-      ignoreProcessFailuresOrigin.exists(ignore => ignore == "true" || ignore == "yes")
+      ignoreProcessFailuresOrigin.exists(Seq("true", "yes") contains _)
 
     /* Here we collect all internal service operations and resources, needed to run the indexer */
     val indexedData = new TezosIndexedDataOperations(db)

conseil-lorre/src/test/scala/tech/cryptonomic/conseil/indexer/config/LorreAppConfigTest.scala

@@ -3,7 +3,7 @@ package tech.cryptonomic.conseil.indexer.config
 import com.typesafe.config.{Config, ConfigFactory}
 import tech.cryptonomic.conseil.common.config.Platforms.{TezosConfiguration, TezosNodeConfiguration}
 import tech.cryptonomic.conseil.indexer.config.LorreAppConfig.Loaders._
-import tech.cryptonomic.conseil.indexer.config.LorreAppConfig.Natural
+import tech.cryptonomic.conseil.indexer.config.ConfigUtil.Natural
 import tech.cryptonomic.conseil.common.testkit.ConseilSpec
 
 class LorreAppConfigTest extends ConseilSpec {

docker/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.1"
 
 services:
   conseil-api:
-    image: cryptonomictech/conseil:latest
+    image: cons:latest
     ports:
       - 80:80
     environment:
@@ -60,9 +60,11 @@ services:
       CONSEIL_XTZ_PORT: 80
       CONSEIL_XTZ_KEY: "conseil"
 
-      CONSEIL_XTZ_NETWORK: "mainnet"
+      CONSEIL_API_ALLOW_BLANK_KEYS: "true"
+
+      CONSEIL_XTZ_NETWORK: "granadanet"
       CONSEIL_XTZ_ENABLED: "true"
-      CONSEIL_XTZ_NODE_HOSTNAME: "tezos-staging.cryptonomic-infra.tech"
+      CONSEIL_XTZ_NODE_HOSTNAME: "tezos-granada.cryptonomic-infra.tech"
       CONSEIL_XTZ_NODE_PORT: 443
 
       JVM_XMX: "4G"
@@ -75,7 +77,7 @@ services:
       - conseil-net
 
   conseil-lorre:
-    image: cryptonomictech/conseil:latest
+    image: cons:latest
     environment:
       # List of supported environment variables:
 
@@ -107,13 +109,19 @@ services:
       CONSEIL_XTZ_DB_USER: "conseiluser"
       CONSEIL_XTZ_DB_PASSWORD: "p@ssw0rd"
 
-      CONSEIL_XTZ_NETWORK: "carthagenet"
+      CONSEIL_LORRE_DEPTH: "newest"
+      CONSEIL_LORRE_HEAD_HASH: "head"
+
+      CONSEIL_XTZ_NETWORK: "granadanet"
       CONSEIL_XTZ_ENABLED: "true"
-      CONSEIL_XTZ_NODE_HOSTNAME: "tezos-staging.cryptonomic-infra.tech"
+      CONSEIL_XTZ_NODE_HOSTNAME: "tezos-granada.cryptonomic-infra.tech"
       CONSEIL_XTZ_NODE_PORT: 443
 
       LORRE_RUNNER_PLATFORM: "tezos"
-      LORRE_RUNNER_NETWORK: "carthagenet"
+      LORRE_RUNNER_NETWORK: "granadanet"
+
+      CONSEIL_LORRE_DEPTH: 1320
+      CONSEIL_LORRE_HEAD_HASH: "BMGvXZT9xqJeNNvEupHNLppwbmqnQR4sgd9XHyDWn818MBQDC27"
 
       # Feature-Flag: set to false to disable the concurrent computation of future endorsement/baking rights
       CONSEIL_LORRE_BLOCK_RIGHTS_FETCHING_ENABLED: "true"