merge: #3090

3090: feat: allow passing the secret as a base64 string r=sprutton1 a=sprutton1 This lets us pass the cbor encoded key that we use to encrypt secrets as a base64 string so we can host in ECS. This was a bit more surgery than the last one, but we got there in the end. Co-authored-by: Scott Prutton <[email protected]>
systeminit · Dec 20, 2023 · ae29ff3 · ae29ff3
2 parents bfa5a56 + 0173f6a
commit ae29ff3
Show file tree

Hide file tree

Showing 10 changed files with 97 additions and 22 deletions.
diff --git a/bin/pinga/src/args.rs b/bin/pinga/src/args.rs
@@ -65,6 +65,10 @@ pub(crate) struct Args {
     #[arg(long)]
     pub(crate) cyclone_encryption_key_base64: Option<String>,
 
+    /// Cyclone secret key as base64 string
+    #[arg(long)]
+    pub(crate) cyclone_secret_key_base64: Option<String>,
+
     /// The number of concurrent jobs that can be processed [default: 10]
     #[arg(long)]
     pub(crate) concurrency: Option<u32>,
@@ -115,6 +119,9 @@ impl TryFrom<Args> for Config {
                     cyclone_encryption_key_base64,
                 );
             }
+            if let Some(secret_string) = args.cyclone_secret_key_base64 {
+                config_map.set("symmetric_crypto_service.active_key_base64", secret_string);
+            }
             if let Some(concurrency) = args.concurrency {
                 config_map.set("concurrency_limit", i64::from(concurrency));
             }

diff --git a/bin/sdf/src/args.rs b/bin/sdf/src/args.rs
@@ -71,6 +71,10 @@ pub(crate) struct Args {
     #[arg(long)]
     pub(crate) cyclone_encryption_key_base64: Option<String>,
 
+    /// Cyclone secret key as base64 string
+    #[arg(long)]
+    pub(crate) cyclone_secret_key_base64: Option<String>,
+
     /// Generates cyclone secret key file (does not run server)
     ///
     /// Will error if set when `generate_cyclone_public_key_path` is not set
@@ -132,6 +136,9 @@ impl TryFrom<Args> for Config {
                     cyclone_encryption_key_base64,
                 );
             }
+            if let Some(secret_string) = args.cyclone_secret_key_base64 {
+                config_map.set("symmetric_crypto_service.active_key_base64", secret_string);
+            }
             if let Some(pkgs_path) = args.pkgs_path {
                 config_map.set("pkgs_path", pkgs_path);
             }

diff --git a/lib/dal-test/src/lib.rs b/lib/dal-test/src/lib.rs
@@ -679,7 +679,8 @@ fn detect_and_configure_testing_for_buck2(builder: &mut ConfigBuilder) -> Result
     builder.jwt_signing_private_key_path(jwt_signing_private_key_path);
     builder.symmetric_crypto_service_config(
         SymmetricCryptoServiceConfigFile {
-            active_key: symmetric_crypto_service_key,
+            active_key: Some(symmetric_crypto_service_key),
+            active_key_base64: None,
             extra_keys: vec![],
         }
         .try_into()?,
@@ -725,7 +726,8 @@ fn detect_and_configure_testing_for_cargo(dir: String, builder: &mut ConfigBuild
     builder.jwt_signing_private_key_path(jwt_signing_private_key_path);
     builder.symmetric_crypto_service_config(
         SymmetricCryptoServiceConfigFile {
-            active_key: symmetric_crypto_service_key,
+            active_key: Some(symmetric_crypto_service_key),
+            active_key_base64: None,
             extra_keys: vec![],
         }
         .try_into()?,

diff --git a/lib/dal/examples/dal-pkg-export/main.rs b/lib/dal/examples/dal-pkg-export/main.rs
@@ -130,7 +130,8 @@ async fn create_symmetric_crypto_service() -> Result<SymmetricCryptoService> {
 
     SymmetricCryptoService::from_config(
         &SymmetricCryptoServiceConfigFile {
-            active_key: active_key.to_string_lossy().into_owned(),
+            active_key: Some(active_key.to_string_lossy().into_owned()),
+            active_key_base64: None,
             extra_keys: Default::default(),
         }
         .try_into()?,

diff --git a/lib/dal/examples/dal-pkg-import/main.rs b/lib/dal/examples/dal-pkg-import/main.rs
@@ -119,7 +119,8 @@ async fn create_symmetric_crypto_service() -> Result<SymmetricCryptoService> {
 
     SymmetricCryptoService::from_config(
         &SymmetricCryptoServiceConfigFile {
-            active_key: active_key.to_string_lossy().into_owned(),
+            active_key: Some(active_key.to_string_lossy().into_owned()),
+            active_key_base64: None,
             extra_keys: Default::default(),
         }
         .try_into()?,

diff --git a/lib/pinga-server/src/config.rs b/lib/pinga-server/src/config.rs
@@ -54,6 +54,7 @@ pub struct Config {
     #[builder(default = "random_instance_id()")]
     instance_id: String,
 
+    #[builder(default = "SymmetricCryptoServiceConfig::default()")]
     symmetric_crypto_service: SymmetricCryptoServiceConfig,
 }
 
@@ -156,7 +157,8 @@ fn random_instance_id() -> String {
 
 fn default_symmetric_crypto_config() -> SymmetricCryptoServiceConfigFile {
     SymmetricCryptoServiceConfigFile {
-        active_key: "/run/pinga/donkey.key".into(),
+        active_key: None,
+        active_key_base64: None,
         extra_keys: vec![],
     }
 }
@@ -198,7 +200,8 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> {
 
     config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
-        active_key: symmetric_crypto_service_key,
+        active_key: Some(symmetric_crypto_service_key),
+        active_key_base64: None,
         extra_keys: vec![],
     };
 
@@ -223,7 +226,8 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> {
 
     config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
-        active_key: symmetric_crypto_service_key,
+        active_key: Some(symmetric_crypto_service_key),
+        active_key_base64: None,
         extra_keys: vec![],
     };
 

diff --git a/lib/sdf-server/src/server/config.rs b/lib/sdf-server/src/server/config.rs
@@ -65,6 +65,7 @@ pub struct Config {
     #[builder(default = "PosthogConfig::default()")]
     posthog: PosthogConfig,
 
+    #[builder(default = "SymmetricCryptoServiceConfig::default()")]
     symmetric_crypto_service: SymmetricCryptoServiceConfig,
 
     #[builder(default = "MigrationMode::default()")]
@@ -268,7 +269,8 @@ fn default_pkgs_path() -> String {
 
 fn default_symmetric_crypto_config() -> SymmetricCryptoServiceConfigFile {
     SymmetricCryptoServiceConfigFile {
-        active_key: "/run/sdf/donkey.key".into(),
+        active_key: None,
+        active_key_base64: None,
         extra_keys: vec![],
     }
 }
@@ -335,7 +337,8 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> {
     config.jwt_signing_public_key_path = jwt_signing_public_key_path;
     config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
-        active_key: symmetric_crypto_service_key,
+        active_key: Some(symmetric_crypto_service_key),
+        active_key_base64: None,
         extra_keys: vec![],
     };
     config.pkgs_path = pkgs_path;
@@ -383,7 +386,8 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> {
     config.jwt_signing_public_key_path = jwt_signing_public_key_path;
     config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
     config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
-        active_key: symmetric_crypto_service_key,
+        active_key: Some(symmetric_crypto_service_key),
+        active_key_base64: None,
         extra_keys: vec![],
     };
     config.pkgs_path = pkgs_path;

diff --git a/lib/si-cli/src/engine/docker_engine.rs b/lib/si-cli/src/engine/docker_engine.rs
@@ -422,6 +422,7 @@ impl ContainerEngine for DockerEngine {
                 "SI_PINGA__CRYPTO__ENCRYPTION_KEY_FILE=/run/pinga/cyclone_encryption.key",
                 "SI_PINGA__NATS__URL=nats",
                 "SI_PINGA__PG__HOSTNAME=postgres",
+                "SI_PINGA__SYMMETRIC_CRYPTO_SERVICE__ACTIVE_KEY=/run/pinga/donkey.key",
                 "OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317",
             ])
             .volumes([format!("{}:/run/pinga:z", data_dir.display())])
@@ -452,6 +453,7 @@ impl ContainerEngine for DockerEngine {
                 "SI_SDF__CRYPTO__ENCRYPTION_KEY_FILE=/run/sdf/cyclone_encryption.key",
                 "SI_SDF__NATS__URL=nats",
                 "SI_SDF__PG__HOSTNAME=postgres",
+                "SI_SDF__SYMMETRIC_CRYPTO_SERVICE__ACTIVE_KEY=/run/sdf/donkey.key",
                 "OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317",
             ])
             .network_mode("bridge")

diff --git a/lib/si-cli/src/engine/podman_engine.rs b/lib/si-cli/src/engine/podman_engine.rs
@@ -605,6 +605,10 @@ impl ContainerEngine for PodmanEngine {
                 ),
                 ("SI_PINGA__NATS__URL", "nats"),
                 ("SI_PINGA__PG__HOSTNAME", "postgres"),
+                (
+                    "SI_PINGA__SYMMETRIC_CRYPTO_SERVICE__ACTIVE_KEY",
+                    "/run/pinga/donkey.key",
+                ),
                 ("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otelcol:4317"),
             ]))
             .mounts(vec![ContainerMount {
@@ -657,6 +661,10 @@ impl ContainerEngine for PodmanEngine {
                 ),
                 ("SI_SDF__NATS__URL", "nats"),
                 ("SI_SDF__PG__HOSTNAME", "postgres"),
+                (
+                    "SI_SDF__SYMMETRIC_CRYPTO_SERVICE__ACTIVE_KEY",
+                    "/run/sdf/donkey.key",
+                ),
                 ("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otelcol:4317"),
             ]))
             .portmappings(vec![PortMapping {

diff --git a/lib/si-crypto/src/symmetric.rs b/lib/si-crypto/src/symmetric.rs
@@ -1,6 +1,8 @@
 //! Symmetric key cryptography.
+use telemetry::prelude::*;
 
-use std::{collections::HashMap, fs::File, path::PathBuf, sync::Arc};
+use base64::{engine::general_purpose, Engine};
+use std::{collections::HashMap, fs::File, io::Cursor, path::PathBuf, sync::Arc};
 
 use serde::{Deserialize, Serialize};
 use si_hash::Hash;
@@ -15,15 +17,21 @@ pub use sodiumoxide::crypto::secretbox::Nonce as SymmetricNonce;
 #[remain::sorted]
 #[derive(Error, Debug)]
 pub enum SymmetricCryptoError {
+    /// When a base64 encoded key fails to be decoded.
+    #[error("failed to decode base64 encoded key")]
+    Base64Decode(#[source] base64::DecodeError),
     /// When a file fails to be canonicalized
     #[error(transparent)]
     CanonicalFile(#[from] CanonicalFileError),
     /// When a cipertext fails to decrypt
     #[error("error when decrypting ciphertext")]
     DecryptionFailed,
-    /// When deserializing from a key file format fails
-    #[error("error deserializing key file: {0}")]
+    /// When deserializing from a key format fails
+    #[error("error deserializing key : {0}")]
     Deserialize(#[from] ciborium::de::Error<std::io::Error>),
+    /// When failing to supply appropriate values to form_config
+    #[error("error loading from_config, must supply a filepath or base64 string")]
+    FromConfig,
     /// When an error is returned while reading or writing to a key file
     #[error("io error: {0}")]
     Io(#[from] std::io::Error),
@@ -55,11 +63,12 @@ pub struct SymmetricCryptoService {
 /// loaded key. In this way the service can take an arbitrary number of keys which is useful in
 /// operations such as key rotation where at least 2 keys are needed (the new key and the old
 /// keys).
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct SymmetricCryptoServiceConfig {
     /// The path to the active key file which will be used for all encryption.
-    pub active_key: CanonicalFile,
-
+    pub active_key: Option<CanonicalFile>,
+    /// The base64 representation of the active key file which will be used for all encryption.
+    pub active_key_base64: Option<String>,
     /// Extra keys which can be used when decrypting data.
     pub extra_keys: Vec<CanonicalFile>,
 }
@@ -68,8 +77,9 @@ pub struct SymmetricCryptoServiceConfig {
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct SymmetricCryptoServiceConfigFile {
     /// The path to the active key file which will be used for all encryption.
-    pub active_key: String,
-
+    pub active_key: Option<String>,
+    /// The base64 representation of the active key file which will be used for all encryption.
+    pub active_key_base64: Option<String>,
     /// Extra keys which can be used when decrypting data.
     pub extra_keys: Vec<String>,
 }
@@ -78,8 +88,14 @@ impl TryFrom<SymmetricCryptoServiceConfigFile> for SymmetricCryptoServiceConfig
     type Error = CanonicalFileError;
 
     fn try_from(value: SymmetricCryptoServiceConfigFile) -> Result<Self, Self::Error> {
-        let active_key = value.active_key.try_into()?;
-
+        let mut active_key: Option<CanonicalFile> = None;
+        let mut active_key_base64: Option<String> = None;
+        if let Some(key) = value.active_key {
+            active_key = Some(key.try_into()?);
+        }
+        if let Some(key) = value.active_key_base64 {
+            active_key_base64 = Some(key);
+        }
         let mut extra_keys = Vec::new();
         for extra_key_str in value.extra_keys {
             extra_keys.push(extra_key_str.try_into()?);
@@ -88,6 +104,7 @@ impl TryFrom<SymmetricCryptoServiceConfigFile> for SymmetricCryptoServiceConfig
         Ok(Self {
             active_key,
             extra_keys,
+            active_key_base64,
         })
     }
 }
@@ -120,8 +137,11 @@ impl SymmetricCryptoService {
     /// - A key file could not be successfully parsed
     /// - The [`SymmetricKey`] could not be successfully resolved from loading the key file
     pub async fn from_config(config: &SymmetricCryptoServiceConfig) -> SymmetricCryptoResult<Self> {
-        let active_key = SymmetricKey::load(&config.active_key).await?;
-
+        let active_key = match (&config.active_key, &config.active_key_base64) {
+            (Some(key), None) => Ok(SymmetricKey::load(key).await?),
+            (None, Some(b64_string)) => Ok(SymmetricKey::decode(b64_string.to_string()).await?),
+            _ => Err(SymmetricCryptoError::FromConfig),
+        }?;
         let mut extra_keys = vec![];
 
         for key_path in config.extra_keys.iter() {
@@ -213,6 +233,18 @@ impl SymmetricKey {
     pub async fn load(path: impl Into<PathBuf>) -> SymmetricCryptoResult<Self> {
         Ok(SymmetricKeyFile::load(path).await?.into())
     }
+
+    /// Load a key from a base64 string.
+    ///
+    /// # Errors
+    ///
+    /// Return `Err` if:
+    ///
+    /// - The key string could not be successfully parsed
+    /// - The [`SymmetricKey`] could not be successfully resolved
+    pub async fn decode(key_string: String) -> SymmetricCryptoResult<Self> {
+        Ok(SymmetricKeyFile::decode(key_string).await?.into())
+    }
 }
 
 impl From<SymmetricKeyFile> for SymmetricKey {
@@ -240,6 +272,13 @@ impl SymmetricKeyFile {
         .map_err(Into::into)
     }
 
+    async fn decode(key_string: String) -> SymmetricCryptoResult<Self> {
+        let buf = general_purpose::STANDARD
+            .decode(key_string)
+            .map_err(SymmetricCryptoError::Base64Decode)?;
+        ciborium::from_reader(Cursor::new(&buf)).map_err(Into::into)
+    }
+
     async fn load(path: impl Into<PathBuf>) -> SymmetricCryptoResult<Self> {
         let path = path.into();