diff --git a/Cargo.lock b/Cargo.lock index 1f312202..148bee26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,6 +521,7 @@ dependencies = [ "ctrlc", "dcspark-blockchain-source", "dcspark-core", + "dotenv", "entity", "hex", "migration", diff --git a/deployment/config/indexer/cardano_node_docker.yml b/deployment/config/indexer/cardano_node_docker.yml index 0f73f1d5..f0f45eec 100644 --- a/deployment/config/indexer/cardano_node_docker.yml +++ b/deployment/config/indexer/cardano_node_docker.yml @@ -14,11 +14,7 @@ sink: type: cardano db: type: postgres - host: postgres # name of postgres container - port: 5432 # internal postgres port - user: carp - password: "1234" - db: carp_mainnet + database_url: postgresql://carp:1234@postgres:5432/carp_mainnet network: mainnet # preview / preprod / testnet start_block: diff --git a/deployment/config/indexer/oura_docker.yml b/deployment/config/indexer/oura_docker.yml index f2dc8080..6f7ce7fb 100644 --- a/deployment/config/indexer/oura_docker.yml +++ b/deployment/config/indexer/oura_docker.yml @@ -7,11 +7,7 @@ sink: type: cardano db: type: postgres - host: postgres # name of postgres container - port: 5432 # internal postgres port - user: carp - password: "1234" - db: carp_mainnet + database_url: postgresql://carp:1234@postgres:5432/carp_mainnet network: mainnet # preview / preprod / testnet start_block: diff --git a/docs/docs/indexer/docker.md b/docs/docs/indexer/docker.md index 5ea9f8ae..5669ac23 100644 --- a/docs/docs/indexer/docker.md +++ b/docs/docs/indexer/docker.md @@ -43,6 +43,7 @@ NETWORK=mainnet # can be cardano_node_docker.yml or oura_docker.yml or # any custom filename located in deployment/config/indexer folder +# alternative to CARP_CONFIG env variable CONFIG_FILE=cardano_node_docker.yml CARP_VERSION=2.2.1 @@ -65,7 +66,7 @@ DATABASE_URL=postgresql://${PGUSER}:${PGPASSWORD}@${POSTGRES_HOST}:${POSTGRES_PO PGURI=$DATABASE_URL ``` -Please set `NETWORK`, `CONFIG_FILE` and postgres variables carefully. `CONFIG_FILE` should be located in `deployment/config/indexer` folder. +Please set `NETWORK`, `CONFIG_FILE` and postgres variables carefully. `CONFIG_FILE` should be located in `deployment/config/indexer` folder. You can use `CARP_CONFIG` env variable instead of `CONFIG_FILE` if you want to use just env variables for configuration. You can also add non-mandatory variables that are useful for backuper: @@ -83,7 +84,7 @@ AWS_SECRET_ACCESS_KEY= ## Carp configuration -We mentioned config files above, they should be stored in `deployment/config/indexer` folder, and you should set `CONFIG_FILE` env variable with the config name. +We mentioned config files above, they should be stored in `deployment/config/indexer` folder, and you should set `CONFIG_FILE` env variable with the config name. Alternatively you can set `CARP_CONFIG` env variable and not use the configuration file The difference from non-docker configuration is in the networking mainly. diff --git a/docs/docs/indexer/run.md b/docs/docs/indexer/run.md index 56ee5418..91b6476d 100644 --- a/docs/docs/indexer/run.md +++ b/docs/docs/indexer/run.md @@ -8,7 +8,9 @@ To run carp you need to configure carp itself, set up / configure cardano-node ( ## Configuration & concepts -Carp itself uses special file for configuration, the examples are stored [there](https://github.com/dcSpark/carp/blob/main/indexer/configs/). Let's dive into configuration a little further: +Carp itself uses special file for configuration, the examples are stored [there](https://github.com/dcSpark/carp/blob/main/indexer/configs/). Besides, carp's config can be set up through `CARP_CONFIG` env variable in json format. Let's dive into configuration a little further: + +File format: ```yaml source: type: oura @@ -19,16 +21,17 @@ sink: type: cardano db: type: postgres - host: localhost - port: 5432 - user: carp - password: "1234" - db: carp_mainnet + database_url: postgresql://carp:1234@localhost:5432/carp_mainnet network: mainnet # preview / preprod / testnet start_block: ``` +Json format: +```json +{"source":{"type":"oura","socket":"relays-new.cardano-mainnet.iohk.io:3001","bearer":"Tcp"},"sink":{"type":"cardano","db":{"type": "postgres","database_url":"postgresql://carp:1234@localhost:5432/carp_mainnet"},"network":"mainnet"},"start_block":null} +``` + As you might see there are several key sections: source and sink. For sink there's only one option at the moment: `cardano` sink. For source there are two options with different configurations: `oura` and `cardano_net`. ### Sink configuration diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 3d9eb8e4..95fbad5a 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -24,10 +24,11 @@ async-trait = { version = "0.1.64" } base64 = { version = "0.21.0" } cardano-multiplatform-lib = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", branch = "metadata-and-addr" } clap = { version = "3.1", features = ["derive"] } -ctrlc = {version = "3.2.4", features = ["termination"] } +ctrlc = { version = "3.2.4", features = ["termination"] } +dotenv = { version = "0.15.0" } hex = { version = "0.4.3" } oura = { version = "1.8.0" } -pallas = {version = "0.17.0" } +pallas = { version = "0.17.0" } serde = { version = "1.0.152", features = ["derive", "rc"] } serde_json = { version = "1.0.92" } serde_yaml = { version = "0.9.17" } diff --git a/indexer/configs/cardano_node.yml b/indexer/configs/cardano_node.yml index f7d6e461..b39261bb 100644 --- a/indexer/configs/cardano_node.yml +++ b/indexer/configs/cardano_node.yml @@ -12,11 +12,7 @@ sink: type: cardano db: type: postgres - host: localhost - port: 5432 - user: carp - password: "1234" - db: carp_mainnet + database_url: postgresql://carp:1234@localhost:5432/carp_mainnet network: mainnet # preview / preprod / testnet start_block: diff --git a/indexer/configs/default.yml b/indexer/configs/default.yml index fabf3d86..4990df99 100644 --- a/indexer/configs/default.yml +++ b/indexer/configs/default.yml @@ -7,11 +7,7 @@ sink: type: cardano db: type: postgres - host: localhost - port: 5432 - user: carp - password: "1234" - db: carp_mainnet + database_url: postgresql://carp:1234@localhost:5432/carp_mainnet network: mainnet # preview / preprod / testnet start_block: diff --git a/indexer/configs/oura.yml b/indexer/configs/oura.yml index fabf3d86..4990df99 100644 --- a/indexer/configs/oura.yml +++ b/indexer/configs/oura.yml @@ -7,11 +7,7 @@ sink: type: cardano db: type: postgres - host: localhost - port: 5432 - user: carp - password: "1234" - db: carp_mainnet + database_url: postgresql://carp:1234@localhost:5432/carp_mainnet network: mainnet # preview / preprod / testnet start_block: diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 7a86ea49..2787a131 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -33,20 +33,14 @@ pub struct Cli { /// path to config file #[clap(long, value_parser)] - config_path: PathBuf, + config_path: Option, } #[derive(Debug, Clone, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] #[serde(deny_unknown_fields)] pub enum DbConfig { - Postgres { - host: String, - port: u64, - user: String, - password: String, - db: String, - }, + Postgres { database_url: String }, } #[derive(Debug, Clone, Deserialize)] @@ -114,19 +108,31 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Execution plan {}", plan); let exec_plan = Arc::new(ExecutionPlan::load_from_file(&plan)?); - tracing::info!("Config file {:?}", config_path); - let file = File::open(&config_path).with_context(|| { - format!( - "Cannot open config file {path}", - path = config_path.display() - ) - })?; - let config: Config = serde_yaml::from_reader(file).with_context(|| { - format!( - "Cannot parse config file {path}", - path = config_path.display() - ) - })?; + let config = if let Some(config_path) = config_path { + tracing::info!("Config file {:?}", config_path); + let file = File::open(&config_path).with_context(|| { + format!( + "Cannot open config file {path}", + path = config_path.display() + ) + })?; + let config: Config = serde_yaml::from_reader(file).with_context(|| { + format!( + "Cannot parse config file {path}", + path = config_path.display() + ) + })?; + config + } else { + dotenv::dotenv().ok(); + + let carp_config = std::env::var("CARP_CONFIG") + .expect("env CARP_CONFIG not found and --config-path not specified"); + let config: Config = serde_json::from_str(&carp_config) + .with_context(|| format!("Cannot parse config string {carp_config}"))?; + + config + }; let (network, mut sink) = match config.sink { SinkConfig::Cardano { ref network, .. } => ( diff --git a/indexer/src/sinks/cardano.rs b/indexer/src/sinks/cardano.rs index 66fb42e3..1e9f4551 100644 --- a/indexer/src/sinks/cardano.rs +++ b/indexer/src/sinks/cardano.rs @@ -42,15 +42,8 @@ impl CardanoSink { _ => todo!("Invalid sink config provided"), }; match db_config { - DbConfig::Postgres { - host, - port, - user, - password, - db, - } => { - let url = format!("postgresql://{user}:{password}@{host}:{port}/{db}"); - let conn = Database::connect(&url).await?; + DbConfig::Postgres { database_url } => { + let conn = Database::connect(&database_url).await?; Ok(Self { db: conn,