Merges and fixing the pipeline

.github/workflows/release.yml

@@ -1,15 +1,9 @@
 name: Release
 on:
-  push:
-    tags:
-      - "v*"
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  release:
+    types: [published]
 
 jobs:
-  # Compiles binaries using a bulder matrix
   binary_build:
     name: Build release binaries
     strategy:
@@ -111,51 +105,6 @@ jobs:
           name: ${{ matrix.name }}
           path: ${{ matrix.name }}
 
-  # Creates a GitHub Container package using multi-arch docker builds
-  container_build:
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: |
-            type=raw,value=latest,enable={{is_default_branch}}
-            type=semver,pattern=v{{major}}
-            type=semver,pattern=v{{major}}.{{minor}}
-            type=semver,pattern=v{{version}}
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          # temporarly removing arm platform until we resolve the ca-certificates issue
-          #platforms: linux/arm64,linux/amd64
-          platforms: linux/amd64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-  # Create GitHub release with Rust build targets and release notes
   github_release:
     name: Create GitHub Release
     needs: [binary_build, container_build]
@@ -188,3 +137,13 @@ jobs:
           files: oura-*/oura-*
           body_path: RELEASE.md
           draft: true
+
+      - name: Deploying to Amazon S3
+        uses: dvelasquez/deploy-s3-action@main
+        with:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+          AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
+          BUCKET_PATH: "/" 
+          DIST_LOCATION_CODE: oura-*/oura-*

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "oura"
 description = "The tail of Cardano"
-version = "2.0.0-alpha.2"
+version = "1.8.1"
 edition = "2021"
 repository = "https://github.com/txpipe/oura"
 homepage = "https://github.com/txpipe/oura"
@@ -12,22 +12,13 @@ authors = ["Santiago Carmuega <[email protected]>"]
 
 
 [dependencies]
-# pallas = "0.19.0-alpha.0"
-# pallas = { path = "../pallas/pallas" }
-pallas = { git = "https://github.com/txpipe/pallas" }
-
-gasket = { version = "^0.4", features = ["derive"] }
-# gasket = { path = "../../construkts/gasket-rs/gasket", features = ["derive"] }
-# gasket = { git = "https://github.com/construkts/gasket-rs.git", features = ["derive"]  }
-
-utxorpc-spec-ledger = "1.0.0-alpha.0"
-# utxorpc-spec-ledger = { path = "../../utxorpc/spec/ledger/gen/rust" }
-# utxorpc-spec-ledger = { git = "https://github.com/utxorpc/spec.git" }
-
+pallas = { git = "https://github.com/Emurgo/pallas", tag = "conway_2" }
+# pallas = { git = "https://github.com/txpipe/pallas" }
+# pallas = { path = "../../pallas/pallas" }
 hex = "0.4.3"
 net2 = "0.2.37"
 bech32 = "0.9.1"
-clap = { version = "4.2.7", features = ["derive"] }
+clap = "3.2.22"
 log = "0.4.17"
 env_logger = "0.10.0"
 crossterm = "0.25"
@@ -38,20 +29,17 @@ config = { version = "0.13.2", default-features = false, features = [
     "json",
 ] }
 serde = { version = "1.0.152", features = ["derive"] }
-serde_json = { version = "1.0.89" }
+serde_json = { version = "1.0.89", features = ["arbitrary_precision"] }
 strum = "0.24"
 strum_macros = "0.24"
 prometheus_exporter = { version = "0.8.5", default-features = false }
 unicode-truncate = "0.2.0"
-thiserror = "1.0.39"
-indicatif = "0.17.3"
-lazy_static = "1.4.0"
-tracing = "0.1.37"
-tracing-subscriber = "0.3.17"
-file-rotate = "0.7.1"
-tokio = { version = "1", features = ["rt"] }
-async-trait = "0.1.68"
-reqwest = { version = "0.11", features = ["json"] }
+
+# feature logs
+file-rotate = { version = "0.7.1", optional = true }
+
+# feature: webhook
+reqwest = { version = "0.11", optional = true, features = ["blocking", "json"] }
 
 # feature: kafkasink
 kafka = { version = "0.8.0", optional = true }
@@ -68,11 +56,14 @@ aws-sdk-sqs = { version = "0.14.0", optional = true }
 aws-sdk-lambda = { version = "0.14.0", optional = true }
 aws-sdk-s3 = { version = "0.14.0", optional = true }
 
+# features: elasticsearch || aws
+tokio = { version = "1.24.2", optional = true, features = ["rt"] }
+
 # required for CI to complete successfully
 openssl = { version = "0.10", optional = true, features = ["vendored"] }
 
 # redis support
-redis = { version = "0.23.0", optional = true, features = ["tokio-comp"] }
+redis = { version = "0.21.6", optional = true, features = ["tokio-comp"] }
 
 # features: gcp
 google-cloud-pubsub = { version = "0.12.0", optional = true }
@@ -81,11 +72,15 @@ google-cloud-googleapis = { version = "0.7.0", optional = true }
 # features: rabbitmqsink
 lapin = { version = "2.1.1", optional = true }
 
-# features: deno
-deno_core = { version = "0.175.0", optional = true }
-serde_v8 = { version = "0.86.0", optional = true }
-deno_runtime = { version = "0.101.0", optional = true }
-
 [features]
-default = ["deno"]
-deno = ["deno_core", "deno_runtime", "serde_v8"]
+default = []
+web = ["reqwest"]
+logs = ["file-rotate"]
+webhook = ["web"]
+kafkasink = ["kafka", "openssl"]
+elasticsink = ["elasticsearch", "tokio"]
+fingerprint = ["murmur3"]
+aws = ["aws-config", "aws-sdk-sqs", "aws-sdk-lambda", "aws-sdk-s3", "tokio"]
+redissink = ["redis", "tokio"]
+gcp = ["google-cloud-pubsub", "google-cloud-googleapis", "tokio", "web"]
+rabbitmqsink = ["lapin", "tokio"]

README.md

@@ -8,9 +8,6 @@
     <hr/>
 </div>
 
-> **Warning**
-> `main` branch is now tracking **V2**. This new version is a complete overhaul of the processing pipeline, multiple breaking changes. If you're looking for **V1**, you can switch to the long-term support branch named `lts/v1`
-
 ## Introduction
 
 We have tools to "explore" the Cardano blockchain, which are useful when you know what you're looking for. We argue that there's a different, complementary use-case which is to "observe" the blockchain and react to particular event patterns.
@@ -35,7 +32,7 @@ In this terminal recording we get to see a few mins of live output from a testne
 
 All the heavy lifting required to communicate with the Cardano node is done by the [Pallas](https://github.com/txpipe/pallas) library, which provides an implementation of the Ouroboros multiplexer and a few of the required mini-protocol state-machines (ChainSync and LocalState in particular).
 
-The data pipeline is implemented by the [Gasket](https://github.com/construkts/gasket-rs) library which provides a framework for building staged, event-driven applications. Under this abstraction, each component of the pipeline (aka: _Stage_) runs in its own thread and communicates with other stages by sending messages (very similar to the _Actor pattern_).
+The data pipeline makes heavy use (maybe a bit too much) of multi-threading and mpsc channels provided by Rust's `std::sync` library.
 
 ## Use Cases
 
@@ -49,7 +46,7 @@ Similar to the well-known db-sync tool provided by IOHK, _Oura_ can be used as a
 
 Given its small memory / cpu footprint, _Oura_ can be deployed side-by-side with your Cardano node even in resource-constrained environments, such as Raspberry PIs.
 
-### As A Trigger Of Custom Actions
+### As a Trigger of Custom Actions
 
 _Oura_ running in `daemon` mode can be configured to use custom filters to pinpoint particular transaction patterns and trigger actions whenever it finds a match. For example: send an email when a particular policy / asset combination appears in a transaction; call an AWS Lambda function when a wallet delegates to a particular pool; send a http-call to a webhook each time a metadata key appears in the TX payload;
 
@@ -69,46 +66,53 @@ Oura is in its essence just a pipeline for processing events. Each stage of the
 
 ## Feature Status
 
-- Data Types
-  - CBOR blocks
-  - CBOR txs
-  - Oura v1 model (for backward-compatibility)
-  - Parsed Txs (structured objects with all tx data)
-  - Generic JSON (any kind of JSON values)
 - Sources
-  - chain-sync from local node
-  - chain-sync + block-fetch from remote relay node
-  - S3 bucket with block data
-  - Kafka topic with block data 
+  - [x] chain-sync full-block (node-to-client)
+  - [x] chain-sync + block-fetch (node-to-node)
+  - [x] Parsing of Shelley-compatible blocks (Shelley, Allegra, Mary, Alonzo)
+  - [x] Parsing of Byron blocks
 - Sinks
-  - Kafka topic
-  - Elasticsearch index / data stream
-  - Rotating log files with compression
-  - Redis streams
-  - AWS SQS queue
-  - AWS Lambda call
-  - AWS S3 objects
-  - GCP PubSub
-  - GCP Cloud Function
-  - Azure Sinks
-  - webhook (http post)
-  - terminal (append-only, tail-like)
+  - [x] Kafka topic
+  - [x] Elasticsearch index / data stream
+  - [x] Rotating log files with compression
+  - [x] Redis streams
+  - [x] AWS SQS queue
+  - [x] AWS Lambda call
+  - [x] AWS S3 objects
+  - [ ] GCP Sinks
+  - [ ] Azure Sinks
+  - [x] webhook (http post)
+  - [x] terminal (append-only, tail-like)
+  - [x] RabbitMQ
+- Events / Parsers
+  - [x] block events (start, end)
+  - [x] transaction events (inputs, outputs, assets)
+  - [x] metadata events (labels, content)
+  - [x] mint events (policy, asset, quantity)
+  - [x] pool registrations events
+  - [x] delegation events
+  - [x] CIP-25 metadata parser (image, files)
+  - [ ] CIP-15 metadata parser
 - Filters
-  - Parse block / tx CBOR 
-  - Split block into txs
-  - Select Txs by matching rules (address, metadata, policies, etc)
-  - Enrich tx data with related inputs
-  - Custom Typescript code (uses Deno)
-  - Custom WASM plugin
-  - Rollback buffer with compensating actions
+  - [x] cherry pick by event type (block, tx, mint, cert, etc)
+  - [x] cherry pick by asset subject (policy, name, etc)
+  - [x] cherry pick by metadata keys
+  - [ ] cherry pick by block property (size, tx count)
+  - [ ] cherry pick by tx property (fee, has native script, has plutus script, etc)
+  - [ ] cherry pick by utxo property (address, asset, amount range)
+  - [ ] enrich events with policy info from external metadata service
+  - [ ] enrich input tx info from Blockfrost API
+  - [ ] enrich addresses descriptions using ADAHandle
 - Other
-  - stateful chain cursor to recover from restarts
-  - buffer stage to hold blocks until they reach a certain depth
-  - pipeline metrics to track the progress and performance
+  - [x] stateful chain cursor to recover from restarts
+  - [x] buffer stage to hold blocks until they reach a certain depth
+  - [x] pipeline metrics to track the progress and performance
 
 ## Known Limitations
 
-- Oura reads events from minted blocks / transactions. Support for querying the mempool is not yet implemented.
+- ~~Oura only knows how to process blocks from the Shelley era. We are working on adding support for Byron in a future release.~~ (available since v1.2)
+- Oura reads events from minted blocks / transactions. Support for querying the mempool is planned for a future release.
+- ~~Oura will notify about chain rollbacks as a new event. The business logic for "undoing" the already processed events is a responsability of the consumer. We're working on adding support for a "buffer" filter stage which can hold blocks until they reach a configurable depth (number of confirmations).~~ (rollback buffer available since v1.2)
 
 ## Contributing