feat: store metadata cbor as a file.

cardano-foundation · Nov 9, 2024 · 46e3e23 · 46e3e23
1 parent 2dfaab8
commit 46e3e23
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 217 deletions.
diff --git a/blockchain_common/src/main/resources/lob_blockchain_transaction_metadata_schema.json b/blockchain_common/src/main/resources/lob_blockchain_transaction_metadata_schema.json
@@ -1,13 +1,31 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
+  "definitions": {
+    "countryCodePattern": {
+      "type": "string",
+      "pattern": "^[A-Z]{2}$"
+    },
+    "currencyIdPattern": {
+      "type": "string",
+      "pattern": "^ISO_4217:[A-Z]{3}$|^ISO_24165:[A-Z0-9]+(:[A-Z0-9]+)?$"
+    },
+    "bigDecimalPattern": {
+      "type": "string",
+      "pattern": "^[+-]?[0-9]+(\\.[0-9]+)?([eE][+-]?[0-9]+)?$"
+    },
+    "accountingPeriodPattern": {
+      "type": "string",
+      "pattern": "^[0-9]{4}-[0-9]{2}$"
+    }
+  },
   "properties": {
     "metadata": {
       "type": "object",
       "properties": {
         "creation_slot": {
-          "type": "string",
-          "pattern": "^[0-9]+$"
+          "type": "integer",
+          "minimum": 0
         },
         "version": {
           "type": "string"
@@ -19,8 +37,7 @@
       "type": "object",
       "properties": {
         "country_code": {
-          "type": "string",
-          "pattern": "^[A-Z]{2}$"
+          "$ref": "#/definitions/countryCodePattern"
         },
         "name": {
           "type": "string"
@@ -32,8 +49,7 @@
           "type": "string"
         },
         "currency_id": {
-          "type": "string",
-          "pattern": "^ISO_4217:[A-Z]{3}$|^ISO_24165:[A-Z0-9]+(:[A-Z0-9]+)?$"
+          "$ref": "#/definitions/currencyIdPattern"
         }
       },
       "required": ["country_code", "name", "tax_id_number", "id", "currency_id"]
@@ -77,8 +93,7 @@
               "type": "object",
               "properties": {
                 "amount": {
-                  "type": "string",
-                  "pattern": "^[0-9]+(\\.[0-9]{1,2})?$"
+                  "$ref": "#/definitions/bigDecimalPattern"
                 },
                 "event": {
                   "type": "object",
@@ -129,8 +144,7 @@
                           "type": "string"
                         },
                         "id": {
-                          "type": "string",
-                          "pattern": "^ISO_4217:[A-Z]{3}$|^ISO_24165:[A-Z0-9]+(:[A-Z0-9]+)?$"
+                          "$ref": "#/definitions/currencyIdPattern"
                         }
                       },
                       "required": ["cust_code", "id"]
@@ -142,8 +156,7 @@
                           "type": "string"
                         },
                         "rate": {
-                          "type": "string",
-                          "pattern": "^0(\\.\\d{1,3})?$|^1(\\.0{1,3})?$"
+                          "$ref": "#/definitions/bigDecimalPattern"
                         }
                       }
                     },
@@ -166,21 +179,19 @@
                   "type": "string"
                 },
                 "fx_rate": {
-                  "type": "string",
-                  "pattern": "^[0-9]+(\\.[0-9]+)?$"
+                  "$ref": "#/definitions/bigDecimalPattern"
                 }
               },
               "required": ["amount", "event", "document", "id", "fx_rate"]
             }
           },
           "accounting_period": {
-            "type": "string",
-            "pattern": "^[0-9]{4}-[0-9]{2}$"
+            "$ref": "#/definitions/accountingPeriodPattern"
           }
         },
         "required": ["date", "number", "batch_id", "id", "type", "items", "accounting_period"]
       }
     }
   },
   "required": ["metadata", "org", "txs"]
-}
+}
diff --git a/blockchain_common/src/test/resources/test_transactions_valid.json b/blockchain_common/src/test/resources/test_transactions_valid.json
@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "creation_slot": "10278",
+    "creation_slot": 10278,
     "version": "1.0"
   },
   "org": {
@@ -48,7 +48,7 @@
             }
           },
           "id": "9c6e241e1136e20e3b5033861156a1e453dce463575ebbebaa813e2af73fa95c",
-          "fx_rate": "0.91277"
+          "fx_rate": "1E-2"
         }
       ],
       "accounting_period": "2023-10"

diff --git a/...java/org/cardanofoundation/lob/app/blockchain_publisher/service/L1TransactionCreator.java b/...java/org/cardanofoundation/lob/app/blockchain_publisher/service/L1TransactionCreator.java
@@ -100,17 +100,20 @@ private Optional<BlockchainTransactions> createTransaction(String organisationId
             if (transactionLinePeek == null) { // next one is last element
                 continue;
             }
-            val newChunkTxBytesE = serialiseTransactionChunk(organisationId, Stream.concat(transactionsBatch.stream(), Stream.of(transactionLinePeek)).collect(Collectors.toSet()), creationSlot);
+            val newChunkTxBytesE = serialiseTransactionChunk(organisationId, Stream.concat(transactionsBatch.stream(), Stream.of(transactionLinePeek))
+                    .collect(Collectors.toSet()), creationSlot);
+
             if (newChunkTxBytesE.isLeft()) {
                 log.error("Error serialising transaction, abort processing, issue: {}", newChunkTxBytesE.getLeft().getDetail());
+
                 return Optional.empty();
             }
             val newChunkTxBytes = newChunkTxBytesE.get();
 
             if (newChunkTxBytes.length >= CARDANO_MAX_TRANSACTION_SIZE_BYTES) {
                 log.info("Blockchain transaction created, id:{}", TransactionUtil.getTxHash(txBytes));
 
-                final var remaining = calculateRemainingTransactionLines(transactions, transactionsBatch);
+                val remaining = calculateRemainingTransactionLines(transactions, transactionsBatch);
 
                 return Optional.of(new BlockchainTransactions(organisationId, transactionsBatch, remaining, creationSlot, txBytes));
             }
@@ -165,11 +168,14 @@ private Either<Problem, byte[]> serialiseTransactionChunk(String organisationId,
             if (debugStoreOutputTx) {
                 val timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now());
                 val name = STR."lob-tx-metadata-\{runId}-\{timestamp}-\{creationSlot}";
-                val tmpTxFile = Files.createTempFile(name, ".json");
+                val tmpJsonTxFile = Files.createTempFile(name, ".json");
+                val tmpCborFile = Files.createTempFile(name, ".cbor");
 
-                log.info("DebugStoreTx enabled, storing tx metadata to file: {}", tmpTxFile);
+                log.info("DebugStoreTx enabled, storing JSON tx metadata to file: {}", tmpJsonTxFile);
+                Files.writeString(tmpJsonTxFile, json);
 
-                Files.writeString(tmpTxFile, json);
+                log.info("DebugStoreTx enabled, storing CBOR tx metadata to file: {}", tmpCborFile);
+                Files.writeString(tmpCborFile, json);
             }
 
             val isValid = jsonSchemaMetadataChecker.checkTransactionMetadata(json);

diff --git a/...n/java/org/cardanofoundation/lob/app/blockchain_publisher/service/MetadataSerialiser.java b/...n/java/org/cardanofoundation/lob/app/blockchain_publisher/service/MetadataSerialiser.java
@@ -8,6 +8,7 @@
 import org.cardanofoundation.lob.app.support.calc.BigDecimals;
 import org.springframework.stereotype.Service;
 
+import java.math.BigInteger;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
@@ -51,7 +52,7 @@ private MetadataMap createMetadataSection(long creationSlot) {
 
         val now = Instant.now(clock);
 
-        metadataMap.put("creation_slot", String.valueOf(creationSlot));
+        metadataMap.put("creation_slot", BigInteger.valueOf(creationSlot));
         metadataMap.put("timestamp", DateTimeFormatter.ISO_INSTANT.format(now));
         metadataMap.put("version", VERSION);
 
@@ -116,7 +117,6 @@ private static MetadataMap serialise(Document document) {
         document.getVat().ifPresent(vat -> metadataMap.put("vat", serialise(vat)));
         document.getCounterparty().ifPresent(counterparty -> metadataMap.put("counterparty", serialise(counterparty)));
 
-
         return metadataMap;
     }
 
@@ -155,7 +155,6 @@ private static MetadataMap serialise(TransactionItemEntity transactionItemEntity
         transactionItemEntity.getCostCenter().ifPresent(costCenter -> metadataMap.put("cost_center", serialise(costCenter)));
 
         metadataMap.put("fx_rate", BigDecimals.normaliseEngineeringString(transactionItemEntity.getFxRate()));
-
         metadataMap.put("document", serialise(transactionItemEntity.getDocument()));
 
         return metadataMap;

diff --git a/...ation/lob/app/blockchain_publisher/service/dispatch/BlockchainTransactionsDispatcher.java b/...ation/lob/app/blockchain_publisher/service/dispatch/BlockchainTransactionsDispatcher.java
@@ -91,10 +91,9 @@ private Optional<BlockchainTransactions> createAndSendBlockchainTransactions(Str
             return Optional.empty();
         }
 
-        var serialisedTxE = l1TransactionCreator.pullBlockchainTransaction(organisationId, transactions);
-
+        val serialisedTxE = l1TransactionCreator.pullBlockchainTransaction(organisationId, transactions);
         if (serialisedTxE.isEmpty()) {
-            log.warn("Error, there is more passedTransactions to dispatch for organisation:{}", organisationId);
+            log.warn("Error, there is more passedTransactions to dispatch for organisation:{}, actual issue:{}", organisationId, serialisedTxE.getLeft());
 
             return Optional.empty();
         }