merge: #3081

3081: [wip] feat: pass encryption key as base64 string r=sprutton1 a=sprutton1 Needs a bunch of clean up, but it works! ```bash ./buck-out/v2/gen/root/524f8da68ea2a374/bin/pinga/__pinga__/static_pic/pinga --cyclone-encryption-key-base64 $(cat lib/cyclone-server/src/dev.encryption.key | base64) -vvvv 2023-12-19T02:49:41.677935Z INFO ThreadId(61) telemetry_application: updated tracing levels to: "trace,pinga=trace,pinga_server=trace" 2023-12-19T02:49:41.679335Z TRACE ThreadId(02) config_file::layered_load: merging defaults config for defaults=SerdeSource { source: ConfigFile { pg: PgPoolConfig { user: "si", password: ..., dbname: "si", application_name: "si-unknown-app", hostname: "localhost", port: 5432, pool_max_size: 128, pool_timeout_wait_secs: None, pool_timeout_create_secs: None, pool_timeout_recycle_secs: None }, nats: NatsConfig { connection_name: None, creds: None, creds_file: None, subject_prefix: None, url: "localhost" }, crypto: CryptoConfig { encryption_key_base64: None, encryption_key_file: None }, concurrency_limit: 5, instance_id: "01HHZZN91FYRAR148C6YDVM4M5", symmetric_crypto_service: SymmetricCryptoServiceConfigFile { active_key: "/run/pinga/donkey.key", extra_keys: [] } } } ... si_crypto::cyclone::encryption_key: loading cyclone encryption key from base64 string vpq8omDdnI0Ar0KGLRRvXoYK+2sm6FjdiPdnAQGWt3I= ``` Co-authored-by: Scott Prutton <[email protected]>
systeminit · Dec 19, 2023 · 610c8a6 · 610c8a6
2 parents c06ab47 + edb65e7
commit 610c8a6
Show file tree

Hide file tree

Showing 13 changed files with 140 additions and 53 deletions.
diff --git a/bin/council/src/args.rs b/bin/council/src/args.rs
@@ -29,7 +29,7 @@ pub(crate) struct Args {
 
     /// NATS credentials file
     #[arg(long)]
-    pub(crate) nats_creds_file: Option<String>,
+    pub(crate) nats_creds_path: Option<String>,
 
     /// Disable OpenTelemetry on startup
     #[arg(long)]
@@ -47,7 +47,7 @@ impl TryFrom<Args> for Config {
             if let Some(creds) = args.nats_creds {
                 config_map.set("nats.creds", creds);
             }
-            if let Some(creds_file) = args.nats_creds_file {
+            if let Some(creds_file) = args.nats_creds_path {
                 config_map.set("nats.creds_file", creds_file);
             }
             config_map.set("nats.connection_name", NAME);

diff --git a/bin/pinga/src/args.rs b/bin/pinga/src/args.rs
@@ -51,7 +51,7 @@ pub(crate) struct Args {
 
     /// NATS credentials file
     #[arg(long)]
-    pub(crate) nats_creds_file: Option<String>,
+    pub(crate) nats_creds_path: Option<String>,
 
     /// Disable OpenTelemetry on startup
     #[arg(long)]
@@ -61,6 +61,10 @@ pub(crate) struct Args {
     #[arg(long)]
     pub(crate) cyclone_encryption_key_path: Option<String>,
 
+    /// Cyclone encryption key file contents as a base64 encoded string
+    #[arg(long)]
+    pub(crate) cyclone_encryption_key_base64: Option<String>,
+
     /// The number of concurrent jobs that can be processed [default: 10]
     #[arg(long)]
     pub(crate) concurrency: Option<u32>,
@@ -99,11 +103,17 @@ impl TryFrom<Args> for Config {
             if let Some(creds) = args.nats_creds {
                 config_map.set("nats.creds", creds);
             }
-            if let Some(creds_file) = args.nats_creds_file {
+            if let Some(creds_file) = args.nats_creds_path {
                 config_map.set("nats.creds_file", creds_file);
             }
-            if let Some(cyclone_encyption_key_path) = args.cyclone_encryption_key_path {
-                config_map.set("cyclone_encryption_key_path", cyclone_encyption_key_path);
+            if let Some(cyclone_encryption_key_file) = args.cyclone_encryption_key_path {
+                config_map.set("crypto.encryption_key_file", cyclone_encryption_key_file);
+            }
+            if let Some(cyclone_encryption_key_base64) = args.cyclone_encryption_key_base64 {
+                config_map.set(
+                    "crypto.encryption_key_base64",
+                    cyclone_encryption_key_base64,
+                );
             }
             if let Some(concurrency) = args.concurrency {
                 config_map.set("concurrency_limit", i64::from(concurrency));

diff --git a/bin/sdf/src/args.rs b/bin/sdf/src/args.rs
@@ -53,7 +53,7 @@ pub(crate) struct Args {
 
     /// NATS credentials file
     #[arg(long)]
-    pub(crate) nats_creds_file: Option<String>,
+    pub(crate) nats_creds_path: Option<String>,
 
     /// Database migration mode on startup
     #[arg(long, value_parser = PossibleValuesParser::new(MigrationMode::variants()))]
@@ -67,6 +67,10 @@ pub(crate) struct Args {
     #[arg(long)]
     pub(crate) cyclone_encryption_key_path: Option<String>,
 
+    /// Cyclone encryption key file contents
+    #[arg(long)]
+    pub(crate) cyclone_encryption_key_base64: Option<String>,
+
     /// Generates cyclone secret key file (does not run server)
     ///
     /// Will error if set when `generate_cyclone_public_key_path` is not set
@@ -116,11 +120,17 @@ impl TryFrom<Args> for Config {
             if let Some(creds) = args.nats_creds {
                 config_map.set("nats.creds", creds);
             }
-            if let Some(creds_file) = args.nats_creds_file {
+            if let Some(creds_file) = args.nats_creds_path {
                 config_map.set("nats.creds_file", creds_file);
             }
-            if let Some(cyclone_encyption_key_path) = args.cyclone_encryption_key_path {
-                config_map.set("cyclone_encryption_key_path", cyclone_encyption_key_path);
+            if let Some(cyclone_encryption_key_file) = args.cyclone_encryption_key_path {
+                config_map.set("crypto.encryption_key_file", cyclone_encryption_key_file);
+            }
+            if let Some(cyclone_encryption_key_base64) = args.cyclone_encryption_key_base64 {
+                config_map.set(
+                    "crypto.encryption_key_base64",
+                    cyclone_encryption_key_base64,
+                );
             }
             if let Some(pkgs_path) = args.pkgs_path {
                 config_map.set("pkgs_path", pkgs_path);

diff --git a/bin/sdf/src/main.rs b/bin/sdf/src/main.rs
@@ -1,7 +1,6 @@
 #![recursion_limit = "256"]
 
 use std::path::PathBuf;
-use std::sync::Arc;
 
 use color_eyre::Result;
 use sdf_server::{
@@ -73,7 +72,7 @@ async fn run(args: args::Args, mut telemetry: ApplicationTelemetryClient) -> Res
 
     let config = Config::try_from(args)?;
 
-    let encryption_key = Server::load_encryption_key(config.cyclone_encryption_key_path()).await?;
+    let encryption_key = Server::load_encryption_key(config.crypto().clone()).await?;
     let jwt_public_signing_key =
         Server::load_jwt_public_signing_key(config.jwt_signing_public_key_path()).await?;
 
@@ -97,7 +96,7 @@ async fn run(args: args::Args, mut telemetry: ApplicationTelemetryClient) -> Res
         nats_conn,
         job_processor,
         veritech,
-        Arc::from(encryption_key),
+        encryption_key,
         Some(pkgs_path),
         Some(module_index_url),
         symmetric_crypto_service,

diff --git a/bin/veritech/src/args.rs b/bin/veritech/src/args.rs
@@ -27,7 +27,7 @@ pub(crate) struct Args {
 
     /// NATS credentials file
     #[arg(long)]
-    pub(crate) nats_creds_file: Option<String>,
+    pub(crate) nats_creds_path: Option<String>,
 
     /// Disable OpenTelemetry on startup
     #[arg(long)]
@@ -61,7 +61,7 @@ impl TryFrom<Args> for Config {
             if let Some(creds) = args.nats_creds {
                 config_map.set("nats.creds", creds);
             }
-            if let Some(creds_file) = args.nats_creds_file {
+            if let Some(creds_file) = args.nats_creds_path {
                 config_map.set("nats.creds_file", creds_file);
             }
             if args.cyclone_local_firecracker {

diff --git a/lib/pinga-server/src/config.rs b/lib/pinga-server/src/config.rs
@@ -3,10 +3,10 @@ use std::{env, path::Path};
 use buck2_resources::Buck2Resources;
 use derive_builder::Builder;
 use serde::{Deserialize, Serialize};
-use si_crypto::{SymmetricCryptoServiceConfig, SymmetricCryptoServiceConfigFile};
+use si_crypto::{CryptoConfig, SymmetricCryptoServiceConfig, SymmetricCryptoServiceConfigFile};
 use si_data_nats::NatsConfig;
 use si_data_pg::PgPoolConfig;
-use si_std::{CanonicalFile, CanonicalFileError};
+use si_std::CanonicalFileError;
 use telemetry::prelude::*;
 use thiserror::Error;
 
@@ -45,7 +45,8 @@ pub struct Config {
     #[builder(default = "NatsConfig::default()")]
     nats: NatsConfig,
 
-    cyclone_encryption_key_path: CanonicalFile,
+    #[builder(default = "CryptoConfig::default()")]
+    crypto: CryptoConfig,
 
     #[builder(default = "default_concurrency_limit()")]
     concurrency: usize,
@@ -78,10 +79,10 @@ impl Config {
         self.nats.subject_prefix.as_deref()
     }
 
-    /// Gets a reference to the config's cyclone public key path.
+    /// Gets a reference to the config's crypto config.
     #[must_use]
-    pub fn cyclone_encryption_key_path(&self) -> &Path {
-        self.cyclone_encryption_key_path.as_path()
+    pub fn crypto(&self) -> &CryptoConfig {
+        &self.crypto
     }
 
     pub fn symmetric_crypto_service(&self) -> &SymmetricCryptoServiceConfig {
@@ -105,8 +106,8 @@ pub struct ConfigFile {
     pg: PgPoolConfig,
     #[serde(default)]
     nats: NatsConfig,
-    #[serde(default = "default_cyclone_encryption_key_path")]
-    cyclone_encryption_key_path: String,
+    #[serde(default)]
+    crypto: CryptoConfig,
     #[serde(default = "default_concurrency_limit")]
     concurrency_limit: usize,
     #[serde(default = "random_instance_id")]
@@ -120,8 +121,8 @@ impl Default for ConfigFile {
         Self {
             pg: Default::default(),
             nats: Default::default(),
-            cyclone_encryption_key_path: default_cyclone_encryption_key_path(),
             concurrency_limit: default_concurrency_limit(),
+            crypto: Default::default(),
             instance_id: random_instance_id(),
             symmetric_crypto_service: default_symmetric_crypto_config(),
         }
@@ -141,7 +142,7 @@ impl TryFrom<ConfigFile> for Config {
         let mut config = Config::builder();
         config.pg_pool(value.pg);
         config.nats(value.nats);
-        config.cyclone_encryption_key_path(value.cyclone_encryption_key_path.try_into()?);
+        config.crypto(value.crypto);
         config.concurrency(value.concurrency_limit);
         config.instance_id(value.instance_id);
         config.symmetric_crypto_service(value.symmetric_crypto_service.try_into()?);
@@ -153,10 +154,6 @@ fn random_instance_id() -> String {
     Ulid::new().to_string()
 }
 
-fn default_cyclone_encryption_key_path() -> String {
-    "/run/pinga/cyclone_encryption.key".to_string()
-}
-
 fn default_symmetric_crypto_config() -> SymmetricCryptoServiceConfigFile {
     SymmetricCryptoServiceConfigFile {
         active_key: "/run/pinga/donkey.key".into(),
@@ -199,7 +196,7 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> {
         "detected development run",
     );
 
-    config.cyclone_encryption_key_path = cyclone_encryption_key_path;
+    config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
         active_key: symmetric_crypto_service_key,
         extra_keys: vec![],
@@ -224,7 +221,7 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> {
         "detected development run",
     );
 
-    config.cyclone_encryption_key_path = cyclone_encryption_key_path;
+    config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
         active_key: symmetric_crypto_service_key,
         extra_keys: vec![],

diff --git a/lib/pinga-server/src/server.rs b/lib/pinga-server/src/server.rs
@@ -1,4 +1,4 @@
-use std::{io, path::Path, sync::Arc};
+use std::{io, sync::Arc};
 
 use dal::{
     job::{
@@ -11,7 +11,9 @@ use dal::{
 };
 use futures::{FutureExt, Stream, StreamExt};
 use nats_subscriber::{Request, SubscriberError};
-use si_crypto::{SymmetricCryptoError, SymmetricCryptoService, SymmetricCryptoServiceConfig};
+use si_crypto::{
+    CryptoConfig, SymmetricCryptoError, SymmetricCryptoService, SymmetricCryptoServiceConfig,
+};
 use si_data_nats::{NatsClient, NatsConfig, NatsError};
 use si_data_pg::{PgPool, PgPoolConfig, PgPoolError};
 use stream_cancel::StreamExt as StreamCancelStreamExt;
@@ -99,8 +101,7 @@ impl Server {
     pub async fn from_config(config: Config) -> Result<Self> {
         dal::init()?;
 
-        let encryption_key =
-            Self::load_encryption_key(config.cyclone_encryption_key_path()).await?;
+        let encryption_key = Self::load_encryption_key(config.crypto().clone()).await?;
         let nats = Self::connect_to_nats(config.nats()).await?;
         let pg_pool = Self::create_pg_pool(config.pg_pool()).await?;
         let veritech = Self::create_veritech_client(nats.clone());
@@ -194,8 +195,10 @@ impl Server {
     }
 
     #[instrument(name = "pinga.init.load_encryption_key", skip_all)]
-    async fn load_encryption_key(path: impl AsRef<Path>) -> Result<Arc<CycloneEncryptionKey>> {
-        Ok(Arc::new(CycloneEncryptionKey::load(path).await?))
+    async fn load_encryption_key(crypto_config: CryptoConfig) -> Result<Arc<CycloneEncryptionKey>> {
+        Ok(Arc::new(
+            CycloneEncryptionKey::from_config(crypto_config).await?,
+        ))
     }
 
     #[instrument(name = "pinga.init.connect_to_nats", skip_all)]

diff --git a/lib/sdf-server/src/server/config.rs b/lib/sdf-server/src/server/config.rs
@@ -1,3 +1,4 @@
+use si_crypto::CryptoConfig;
 use std::{
     env,
     net::{SocketAddr, ToSocketAddrs},
@@ -69,9 +70,11 @@ pub struct Config {
     #[builder(default = "MigrationMode::default()")]
     migration_mode: MigrationMode,
 
+    #[builder(default = "CryptoConfig::default()")]
+    crypto: CryptoConfig,
+
     jwt_signing_public_key_path: CanonicalFile,
 
-    cyclone_encryption_key_path: CanonicalFile,
     signup_secret: SensitiveString,
     pkgs_path: CanonicalFile,
 }
@@ -113,8 +116,8 @@ impl Config {
 
     /// Gets a reference to the config's cyclone public key path.
     #[must_use]
-    pub fn cyclone_encryption_key_path(&self) -> &Path {
-        self.cyclone_encryption_key_path.as_path()
+    pub fn crypto(&self) -> &CryptoConfig {
+        &self.crypto
     }
 
     /// Gets a reference to the config's signup secret.
@@ -166,8 +169,8 @@ pub struct ConfigFile {
     pub migration_mode: MigrationMode,
     #[serde(default = "default_jwt_signing_public_key_path")]
     pub jwt_signing_public_key_path: String,
-    #[serde(default = "default_cyclone_encryption_key_path")]
-    pub cyclone_encryption_key_path: String,
+    #[serde(default)]
+    pub crypto: CryptoConfig,
     #[serde(default = "default_signup_secret")]
     pub signup_secret: SensitiveString,
     #[serde(default = "default_pkgs_path")]
@@ -187,7 +190,7 @@ impl Default for ConfigFile {
             nats: Default::default(),
             migration_mode: Default::default(),
             jwt_signing_public_key_path: default_jwt_signing_public_key_path(),
-            cyclone_encryption_key_path: default_cyclone_encryption_key_path(),
+            crypto: Default::default(),
             signup_secret: default_signup_secret(),
             pkgs_path: default_pkgs_path(),
             posthog: Default::default(),
@@ -212,7 +215,7 @@ impl TryFrom<ConfigFile> for Config {
         config.nats(value.nats);
         config.migration_mode(value.migration_mode);
         config.jwt_signing_public_key_path(value.jwt_signing_public_key_path.try_into()?);
-        config.cyclone_encryption_key_path(value.cyclone_encryption_key_path.try_into()?);
+        config.crypto(value.crypto);
         config.signup_secret(value.signup_secret);
         config.pkgs_path(value.pkgs_path.try_into()?);
         config.posthog(value.posthog);
@@ -255,10 +258,6 @@ fn default_jwt_signing_public_key_path() -> String {
     "/run/sdf/jwt_signing_public_key.pem".to_string()
 }
 
-fn default_cyclone_encryption_key_path() -> String {
-    "/run/sdf/cyclone_encryption.key".to_string()
-}
-
 fn default_signup_secret() -> SensitiveString {
     DEFAULT_SIGNUP_SECRET.into()
 }
@@ -334,7 +333,7 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> {
     );
 
     config.jwt_signing_public_key_path = jwt_signing_public_key_path;
-    config.cyclone_encryption_key_path = cyclone_encryption_key_path;
+    config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
         active_key: symmetric_crypto_service_key,
         extra_keys: vec![],
@@ -382,7 +381,7 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> {
     );
 
     config.jwt_signing_public_key_path = jwt_signing_public_key_path;
-    config.cyclone_encryption_key_path = cyclone_encryption_key_path;
+    config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
         active_key: symmetric_crypto_service_key,
         extra_keys: vec![],

diff --git a/lib/sdf-server/src/server/server.rs b/lib/sdf-server/src/server/server.rs
@@ -1,5 +1,6 @@
+use si_crypto::CryptoConfig;
 use std::time::Duration;
-use std::{io, net::SocketAddr, path::Path, path::PathBuf};
+use std::{io, net::SocketAddr, path::Path, path::PathBuf, sync::Arc};
 
 use axum::routing::IntoMakeService;
 use axum::Router;
@@ -219,8 +220,12 @@ impl Server<(), ()> {
     }
 
     #[instrument(name = "sdf.init.load_encryption_key", skip_all)]
-    pub async fn load_encryption_key(path: impl AsRef<Path>) -> Result<CycloneEncryptionKey> {
-        Ok(CycloneEncryptionKey::load(path).await?)
+    pub async fn load_encryption_key(
+        crypto_config: CryptoConfig,
+    ) -> Result<Arc<CycloneEncryptionKey>> {
+        Ok(Arc::new(
+            CycloneEncryptionKey::from_config(crypto_config).await?,
+        ))
     }
 
     #[instrument(name = "sdf.init.migrate_database", skip_all)]

diff --git a/lib/si-crypto/src/cyclone.rs b/lib/si-crypto/src/cyclone.rs
@@ -1,3 +1,4 @@
+pub(crate) mod config;
 pub(crate) mod decryption_key;
 pub(crate) mod encryption_key;
 pub(crate) mod key_pair;