Update Oura and other dependencies, improve configs (#156)

* Update oura to 1.8.3, pallas to 0.18.2, dcspark core to recent version with epoch calculation fixes, other dependencies and configs * Fix deny * Better ports configuration
dcSpark · Aug 30, 2023 · d8fc630 · d8fc630
1 parent 33dc328
commit d8fc630
Show file tree

Hide file tree

Showing 25 changed files with 763 additions and 667 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/deny.toml b/deny.toml
@@ -31,6 +31,9 @@ ignore = [
     # time is deprecated but still used
     # by `chrono`
     "RUSTSEC-2020-0071",
+
+    # as of August 2023 there's not patched version & we only use this lib in indexer
+    "RUSTSEC-2023-0052",
 ]
 
 # This section is considered when running `cargo deny check licenses`

diff --git a/deployment/config/example-secrets/example.mainnet.pgpass b/deployment/config/example-secrets/example.mainnet.pgpass
@@ -0,0 +1 @@
+*:5432:carp_mainnet:carp:1234
diff --git a/deployment/config/example-secrets/example.preprod.pgpass b/deployment/config/example-secrets/example.preprod.pgpass
@@ -0,0 +1 @@
+*:5432:carp_preprod:carp:1234
diff --git a/deployment/config/example-secrets/example.preview.pgpass b/deployment/config/example-secrets/example.preview.pgpass
@@ -0,0 +1 @@
+*:5432:carp_preview:carp:1234
diff --git a/...nt/config/indexer/cardano_node_docker.yml → ...g/indexer/cardano_node_docker.mainnet.yml b/...nt/config/indexer/cardano_node_docker.yml → ...g/indexer/cardano_node_docker.mainnet.yml
diff --git a/deployment/config/indexer/cardano_node_docker.preprod.yml b/deployment/config/indexer/cardano_node_docker.preprod.yml
@@ -0,0 +1,20 @@
+source:
+  type: cardano_net
+  relay:
+    - 172.20.0.4
+    - 3001
+      # - relays-new.cardano-mainnet.iohk.io
+      # - 3001
+      # - preview-node.world.dev.cardano.org
+      # - 30002
+      # - preprod-node.world.dev.cardano.org
+      # - 30000
+
+sink:
+  type: cardano
+  db:
+    type: postgres
+    database_url: postgresql://carp:1234@postgres:5432/carp_preprod
+  network: preprod # preview / mainnet / testnet
+
+start_block:
diff --git a/deployment/config/indexer/cardano_node_docker.preview.yml b/deployment/config/indexer/cardano_node_docker.preview.yml
@@ -0,0 +1,20 @@
+source:
+  type: cardano_net
+  relay:
+    - 172.20.0.4
+    - 3001
+      # - relays-new.cardano-mainnet.iohk.io
+      # - 3001
+      # - preview-node.world.dev.cardano.org
+      # - 30002
+      # - preprod-node.world.dev.cardano.org
+      # - 30000
+
+sink:
+  type: cardano
+  db:
+    type: postgres
+    database_url: postgresql://carp:1234@postgres:5432/carp_preview
+  network: preview # mainnet / preprod / testnet
+
+start_block:
diff --git a/deployment/config/indexer/docker.env → deployment/config/indexer/docker.mainnet.env b/deployment/config/indexer/docker.env → deployment/config/indexer/docker.mainnet.env
@@ -2,7 +2,7 @@
 # this parameter is utilized by migration service, cardano-node, backuper
 NETWORK=mainnet
 
-CONFIG_FILE=oura_docker.yml
+CONFIG_FILE=oura_docker.mainnet.yml
 
 CARP_VERSION=2.2.3
 
@@ -21,3 +21,11 @@ PGPASSFILE="$(realpath secrets/.pgpass)"
 DATABASE_URL=postgresql://${PGUSER}:${PGPASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
 # Needed for PgTyped
 PGURI=$DATABASE_URL
+
+CARDANO_NODE_VERSION=8.1.2
+CARDANO_NODE_EXPOSED_PORT=3001
+
+POSTGRES_VERSION=13.6
+POSTGRES_EXPOSED_PORT=5432
+
+CARP_WEB_EXPOSED_PORT=3000
diff --git a/deployment/config/indexer/docker.preprod.env b/deployment/config/indexer/docker.preprod.env
@@ -0,0 +1,31 @@
+# network can be mainnet/preview/preprod/testnet
+# this parameter is utilized by migration service, cardano-node, backuper
+NETWORK=preprod
+
+CONFIG_FILE=oura_docker.preprod.yml
+
+CARP_VERSION=2.2.3
+
+# these credentials are utilized by postgres, carp and carp_web services
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_DB=carp_preprod
+PGUSER=carp
+PGPASSWORD=1234
+PGPASSFILE="$(realpath secrets/.pgpass)"
+
+# note: PGPASSWORD isn't required to run carp
+# since it will be parsed from the PGPASSFILE instead
+# as this command will gracefully fallback to PGPASSFILE if no password is specified
+# However, some dev tools like pgtyped & zapatos don't support .pgpass files
+DATABASE_URL=postgresql://${PGUSER}:${PGPASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
+# Needed for PgTyped
+PGURI=$DATABASE_URL
+
+CARDANO_NODE_VERSION=8.1.2
+CARDANO_NODE_EXPOSED_PORT=3001
+
+POSTGRES_VERSION=13.6
+POSTGRES_EXPOSED_PORT=5432
+
+CARP_WEB_EXPOSED_PORT=3000
diff --git a/deployment/config/indexer/docker.preview.env b/deployment/config/indexer/docker.preview.env
@@ -0,0 +1,31 @@
+# network can be mainnet/preview/preprod/testnet
+# this parameter is utilized by migration service, cardano-node, backuper
+NETWORK=preview
+
+CONFIG_FILE=oura_docker.preview.yml
+
+CARP_VERSION=2.2.3
+
+# these credentials are utilized by postgres, carp and carp_web services
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_DB=carp_preview
+PGUSER=carp
+PGPASSWORD=1234
+PGPASSFILE="$(realpath secrets/.pgpass)"
+
+# note: PGPASSWORD isn't required to run carp
+# since it will be parsed from the PGPASSFILE instead
+# as this command will gracefully fallback to PGPASSFILE if no password is specified
+# However, some dev tools like pgtyped & zapatos don't support .pgpass files
+DATABASE_URL=postgresql://${PGUSER}:${PGPASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
+# Needed for PgTyped
+PGURI=$DATABASE_URL
+
+CARDANO_NODE_VERSION=8.1.2
+CARDANO_NODE_EXPOSED_PORT=3001
+
+POSTGRES_VERSION=13.6
+POSTGRES_EXPOSED_PORT=5432
+
+CARP_WEB_EXPOSED_PORT=3000
diff --git a/deployment/config/indexer/oura_docker.yml → ...nt/config/indexer/oura_docker.mainnet.yml b/deployment/config/indexer/oura_docker.yml → ...nt/config/indexer/oura_docker.mainnet.yml
diff --git a/deployment/config/indexer/oura_docker.preprod.yml b/deployment/config/indexer/oura_docker.preprod.yml
@@ -0,0 +1,13 @@
+source:
+  type: oura
+  socket: "/app/node-ipc/node.socket" # or url
+  bearer: Unix # or Tcp
+
+sink:
+  type: cardano
+  db:
+    type: postgres
+    database_url: postgresql://carp:1234@postgres:5432/carp_preprod
+  network: preprod # preview / mainnet / testnet
+
+start_block:
diff --git a/deployment/config/indexer/oura_docker.preview.yml b/deployment/config/indexer/oura_docker.preview.yml
@@ -0,0 +1,13 @@
+source:
+  type: oura
+  socket: "/app/node-ipc/node.socket" # or url
+  bearer: Unix # or Tcp
+
+sink:
+  type: cardano
+  db:
+    type: postgres
+    database_url: postgresql://carp:1234@postgres:5432/carp_preview
+  network: preview # mainnet / preprod / testnet
+
+start_block:
diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml
@@ -8,7 +8,7 @@ networks:
 services:
   postgres:
     restart: unless-stopped
-    image: postgres:13.6 # 14.2, 13.6, 12.10, 11.15 # replace with correct major version on existing installations, no alpine
+    image: postgres:${POSTGRES_VERSION:-13.6} # 14.2, 13.6, 12.10, 11.15 # replace with correct major version on existing installations, no alpine
     shm_size: 8gb
     environment:
       - POSTGRES_LOGGING=true
@@ -28,14 +28,14 @@ services:
         max-size: "200k"
         max-file: "10"
     ports:
-      - "5432:5432"
+      - "${POSTGRES_EXPOSED_PORT:-5432}:5432"
     networks:
       carp_network:
         ipv4_address: 172.20.0.3
 
   cardano-node:
     restart: unless-stopped
-    image: inputoutput/cardano-node:1.34.1
+    image: inputoutput/cardano-node:${CARDANO_NODE_VERSION:-8.1.2}
     environment:
       - NETWORK=${NETWORK:-mainnet}
     volumes:
@@ -47,7 +47,7 @@ services:
         max-size: "200k"
         max-file: "10"
     ports:
-      - "3001:3001"
+      - "${CARDANO_NODE_EXPOSED_PORT:-3001}:3001"
     networks:
       carp_network:
         ipv4_address: 172.20.0.4
@@ -97,7 +97,7 @@ services:
       context: ../webserver/
       dockerfile: Dockerfile
     ports:
-      - "3000:3000"
+      - "${CARP_WEB_EXPOSED_PORT:-3000}:3000"
     environment:
       - DATABASE_URL=${DATABASE_URL}
     networks:

diff --git a/deployment/mainnet/tmp/.gitignore b/deployment/mainnet/tmp/.gitignore
diff --git a/deployment/preprod/.gitignore b/deployment/preprod/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!tmp
diff --git a/deployment/preview/.gitignore b/deployment/preview/.gitignore
@@ -0,0 +1,3 @@
+*
+!.gitignore
+!tmp
diff --git a/deployment/testnet/tmp/.gitignore b/deployment/testnet/tmp/.gitignore
diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml
@@ -9,9 +9,9 @@ strip = true
 
 [dependencies]
 # [core]
-dcspark-core = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "81d14efde2fac3e00ab74c89da15e961ec231c6a" }
-dcspark-blockchain-source = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "81d14efde2fac3e00ab74c89da15e961ec231c6a" }
-multiverse = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "81d14efde2fac3e00ab74c89da15e961ec231c6a" }
+dcspark-core = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "837b0135462ff7a1018942dca7198ad9c808a84c" }
+dcspark-blockchain-source = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "837b0135462ff7a1018942dca7198ad9c808a84c" }
+multiverse = { git = "https://github.com/dcSpark/dcspark-core.git", rev = "837b0135462ff7a1018942dca7198ad9c808a84c" }
 
 # [local]
 entity = { path = "entity" }
@@ -27,8 +27,8 @@ clap = { version = "3.1", features = ["derive"] }
 ctrlc = { version = "3.2.4", features = ["termination"] }
 dotenv = { version = "0.15.0" }
 hex = { version = "0.4.3" }
-oura = { version = "1.8.0" }
-pallas = { version = "0.17.0" }
+oura = { git = "https://github.com/txpipe/oura.git", rev = "27fb7e876471b713841d96e292ede40101b151d7" }
+pallas = { version = "0.18.2" }
 serde = { version = "1.0.152", features = ["derive", "rc"] }
 serde_json = { version = "1.0.92" }
 serde_yaml = { version = "0.9.17" }

diff --git a/indexer/entity/Cargo.toml b/indexer/entity/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-pallas = "0.17.0"
+pallas = { version = "0.18.2" }
 sea-orm = { git = "https://github.com/dcSpark/sea-orm", branch = "insert-many-returning", features = [
     "runtime-tokio-rustls",
     "sqlx-postgres",

diff --git a/indexer/src/sources/cardano.rs b/indexer/src/sources/cardano.rs
@@ -22,7 +22,6 @@ pub struct CardanoSource {
         WrappedCardanoSource,
         RollbackOrEvent<CardanoSourceEvent, Point>,
     >,
-    configuration: NetworkConfiguration,
 }
 
 #[async_trait]
@@ -63,11 +62,7 @@ impl Source for CardanoSource {
                         tracing::debug!(id = %block_event.id, "block event received");
                         Ok(Some(CardanoEventType::Block {
                             cbor_hex: hex::encode(block_event.raw_block),
-                            epoch: block_event.epoch.or_else(|| {
-                                self.configuration
-                                    .shelley_era_config
-                                    .absolute_slot_to_epoch(block_event.slot_number.into())
-                            }),
+                            epoch: Some(block_event.epoch),
                             epoch_slot: Some(block_event.slot_number.into()),
                             block_number: block_event.block_number.into(),
                             block_hash: block_event.id.to_string(),
@@ -100,9 +95,6 @@ impl CardanoSource {
                     .context("failed to create temporary multiverse")
                     .map(|multiverse| ForkHandlingSource::new(multiverse, 10, cardano_source))
             })
-            .map(|wrapped_source| Self {
-                configuration,
-                wrapped_source,
-            })
+            .map(|wrapped_source| Self { wrapped_source })
     }
 }
diff --git a/indexer/tasks/Cargo.toml b/indexer/tasks/Cargo.toml
@@ -19,7 +19,7 @@ hex = { version = "0.4.3" }
 inventory = { version = "0.2.3" }
 markdown-gen = { git = "https://github.com/dcSpark/markdown-gen-rs", branch = "hbina-add-ability-to-write-raw-str" }
 nameof = { version = "1.2.2" }
-pallas = { version = "0.17.0" }
+pallas = { version = "0.18.2" }
 paste = { version = "1.0.11" }
 sea-orm = { git = "https://github.com/dcSpark/sea-orm", branch = "insert-many-returning", features = [ "macros" ], default-features = false }
 serde = { version = "1.0.152", features = ["derive"] }

diff --git a/indexer/tasks/src/multiera/multiera_datum.rs b/indexer/tasks/src/multiera/multiera_datum.rs
@@ -64,13 +64,20 @@ async fn handle_datum(
                 .or_insert_with(|| datum.encode_fragment().unwrap());
         }
         for output in tx_body.outputs().iter() {
-            match output.datum().as_ref() {
-                Some(DatumOption::Hash(hash)) => {
+            let datum_option = match output.datum() {
+                Some(datum) => DatumOption::from(datum.clone()),
+                None => {
+                    continue;
+                }
+            };
+
+            match datum_option {
+                DatumOption::Hash(hash) => {
                     hash_to_tx
-                        .entry(*hash)
+                        .entry(hash)
                         .or_insert_with(|| cardano_transaction.id);
                 }
-                Some(DatumOption::Data(datum)) => {
+                DatumOption::Data(datum) => {
                     let hash = datum.compute_hash();
                     hash_to_tx
                         .entry(hash)
@@ -79,7 +86,6 @@ async fn handle_datum(
                         .entry(hash)
                         .or_insert_with(|| datum.0.encode_fragment().unwrap());
                 }
-                None => {}
             };
         }
     }

diff --git a/indexer/tasks/src/multiera/utils/common.rs b/indexer/tasks/src/multiera/utils/common.rs
@@ -47,13 +47,18 @@ pub fn get_plutus_datum_for_output(
     output: &MultiEraOutput,
     plutus_data: &[&KeepRaw<alonzo::PlutusData>],
 ) -> Option<alonzo::PlutusData> {
-    match output.datum() {
-        Some(DatumOption::Data(datum)) => Some(datum.0),
-        Some(DatumOption::Hash(hash)) => plutus_data
+    let datum_option = match output.datum() {
+        Some(datum) => DatumOption::from(datum),
+        None => {
+            return None;
+        }
+    };
+    match datum_option {
+        DatumOption::Data(datum) => Some(datum.0),
+        DatumOption::Hash(hash) => plutus_data
             .iter()
             .find(|datum| Hasher::<256>::hash_cbor(datum) == hash)
             .map(|&d| d.clone().unwrap()),
-        None => None,
     }
 }