Skip to content

Commit

Permalink
merge: #3081
Browse files Browse the repository at this point in the history
3081: [wip] feat: pass encryption key as base64 string r=sprutton1 a=sprutton1

Needs a bunch of clean up, but it works!

```bash
./buck-out/v2/gen/root/524f8da68ea2a374/bin/pinga/__pinga__/static_pic/pinga --cyclone-encryption-key-base64 $(cat lib/cyclone-server/src/dev.encryption.key | base64) -vvvv

2023-12-19T02:49:41.677935Z  INFO ThreadId(61) telemetry_application: updated tracing levels to: "trace,pinga=trace,pinga_server=trace"
2023-12-19T02:49:41.679335Z TRACE ThreadId(02) config_file::layered_load: merging defaults config for defaults=SerdeSource { source: ConfigFile { pg: PgPoolConfig { user: "si", password: ..., dbname: "si", application_name: "si-unknown-app", hostname: "localhost", port: 5432, pool_max_size: 128, pool_timeout_wait_secs: None, pool_timeout_create_secs: None, pool_timeout_recycle_secs: None }, nats: NatsConfig { connection_name: None, creds: None, creds_file: None, subject_prefix: None, url: "localhost" }, crypto: CryptoConfig { encryption_key_base64: None, encryption_key_file: None }, concurrency_limit: 5, instance_id: "01HHZZN91FYRAR148C6YDVM4M5", symmetric_crypto_service: SymmetricCryptoServiceConfigFile { active_key: "/run/pinga/donkey.key", extra_keys: [] } } }
...
si_crypto::cyclone::encryption_key: loading cyclone encryption key from base64 string vpq8omDdnI0Ar0KGLRRvXoYK+2sm6FjdiPdnAQGWt3I=
```

Co-authored-by: Scott Prutton <[email protected]>
  • Loading branch information
si-bors-ng[bot] and sprutton1 authored Dec 19, 2023
2 parents c06ab47 + edb65e7 commit 610c8a6
Show file tree
Hide file tree
Showing 13 changed files with 140 additions and 53 deletions.
4 changes: 2 additions & 2 deletions bin/council/src/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ pub(crate) struct Args {

/// NATS credentials file
#[arg(long)]
pub(crate) nats_creds_file: Option<String>,
pub(crate) nats_creds_path: Option<String>,

/// Disable OpenTelemetry on startup
#[arg(long)]
Expand All @@ -47,7 +47,7 @@ impl TryFrom<Args> for Config {
if let Some(creds) = args.nats_creds {
config_map.set("nats.creds", creds);
}
if let Some(creds_file) = args.nats_creds_file {
if let Some(creds_file) = args.nats_creds_path {
config_map.set("nats.creds_file", creds_file);
}
config_map.set("nats.connection_name", NAME);
Expand Down
18 changes: 14 additions & 4 deletions bin/pinga/src/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ pub(crate) struct Args {

/// NATS credentials file
#[arg(long)]
pub(crate) nats_creds_file: Option<String>,
pub(crate) nats_creds_path: Option<String>,

/// Disable OpenTelemetry on startup
#[arg(long)]
Expand All @@ -61,6 +61,10 @@ pub(crate) struct Args {
#[arg(long)]
pub(crate) cyclone_encryption_key_path: Option<String>,

/// Cyclone encryption key file contents as a base64 encoded string
#[arg(long)]
pub(crate) cyclone_encryption_key_base64: Option<String>,

/// The number of concurrent jobs that can be processed [default: 10]
#[arg(long)]
pub(crate) concurrency: Option<u32>,
Expand Down Expand Up @@ -99,11 +103,17 @@ impl TryFrom<Args> for Config {
if let Some(creds) = args.nats_creds {
config_map.set("nats.creds", creds);
}
if let Some(creds_file) = args.nats_creds_file {
if let Some(creds_file) = args.nats_creds_path {
config_map.set("nats.creds_file", creds_file);
}
if let Some(cyclone_encyption_key_path) = args.cyclone_encryption_key_path {
config_map.set("cyclone_encryption_key_path", cyclone_encyption_key_path);
if let Some(cyclone_encryption_key_file) = args.cyclone_encryption_key_path {
config_map.set("crypto.encryption_key_file", cyclone_encryption_key_file);
}
if let Some(cyclone_encryption_key_base64) = args.cyclone_encryption_key_base64 {
config_map.set(
"crypto.encryption_key_base64",
cyclone_encryption_key_base64,
);
}
if let Some(concurrency) = args.concurrency {
config_map.set("concurrency_limit", i64::from(concurrency));
Expand Down
18 changes: 14 additions & 4 deletions bin/sdf/src/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ pub(crate) struct Args {

/// NATS credentials file
#[arg(long)]
pub(crate) nats_creds_file: Option<String>,
pub(crate) nats_creds_path: Option<String>,

/// Database migration mode on startup
#[arg(long, value_parser = PossibleValuesParser::new(MigrationMode::variants()))]
Expand All @@ -67,6 +67,10 @@ pub(crate) struct Args {
#[arg(long)]
pub(crate) cyclone_encryption_key_path: Option<String>,

/// Cyclone encryption key file contents
#[arg(long)]
pub(crate) cyclone_encryption_key_base64: Option<String>,

/// Generates cyclone secret key file (does not run server)
///
/// Will error if set when `generate_cyclone_public_key_path` is not set
Expand Down Expand Up @@ -116,11 +120,17 @@ impl TryFrom<Args> for Config {
if let Some(creds) = args.nats_creds {
config_map.set("nats.creds", creds);
}
if let Some(creds_file) = args.nats_creds_file {
if let Some(creds_file) = args.nats_creds_path {
config_map.set("nats.creds_file", creds_file);
}
if let Some(cyclone_encyption_key_path) = args.cyclone_encryption_key_path {
config_map.set("cyclone_encryption_key_path", cyclone_encyption_key_path);
if let Some(cyclone_encryption_key_file) = args.cyclone_encryption_key_path {
config_map.set("crypto.encryption_key_file", cyclone_encryption_key_file);
}
if let Some(cyclone_encryption_key_base64) = args.cyclone_encryption_key_base64 {
config_map.set(
"crypto.encryption_key_base64",
cyclone_encryption_key_base64,
);
}
if let Some(pkgs_path) = args.pkgs_path {
config_map.set("pkgs_path", pkgs_path);
Expand Down
5 changes: 2 additions & 3 deletions bin/sdf/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#![recursion_limit = "256"]

use std::path::PathBuf;
use std::sync::Arc;

use color_eyre::Result;
use sdf_server::{
Expand Down Expand Up @@ -73,7 +72,7 @@ async fn run(args: args::Args, mut telemetry: ApplicationTelemetryClient) -> Res

let config = Config::try_from(args)?;

let encryption_key = Server::load_encryption_key(config.cyclone_encryption_key_path()).await?;
let encryption_key = Server::load_encryption_key(config.crypto().clone()).await?;
let jwt_public_signing_key =
Server::load_jwt_public_signing_key(config.jwt_signing_public_key_path()).await?;

Expand All @@ -97,7 +96,7 @@ async fn run(args: args::Args, mut telemetry: ApplicationTelemetryClient) -> Res
nats_conn,
job_processor,
veritech,
Arc::from(encryption_key),
encryption_key,
Some(pkgs_path),
Some(module_index_url),
symmetric_crypto_service,
Expand Down
4 changes: 2 additions & 2 deletions bin/veritech/src/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub(crate) struct Args {

/// NATS credentials file
#[arg(long)]
pub(crate) nats_creds_file: Option<String>,
pub(crate) nats_creds_path: Option<String>,

/// Disable OpenTelemetry on startup
#[arg(long)]
Expand Down Expand Up @@ -61,7 +61,7 @@ impl TryFrom<Args> for Config {
if let Some(creds) = args.nats_creds {
config_map.set("nats.creds", creds);
}
if let Some(creds_file) = args.nats_creds_file {
if let Some(creds_file) = args.nats_creds_path {
config_map.set("nats.creds_file", creds_file);
}
if args.cyclone_local_firecracker {
Expand Down
29 changes: 13 additions & 16 deletions lib/pinga-server/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ use std::{env, path::Path};
use buck2_resources::Buck2Resources;
use derive_builder::Builder;
use serde::{Deserialize, Serialize};
use si_crypto::{SymmetricCryptoServiceConfig, SymmetricCryptoServiceConfigFile};
use si_crypto::{CryptoConfig, SymmetricCryptoServiceConfig, SymmetricCryptoServiceConfigFile};
use si_data_nats::NatsConfig;
use si_data_pg::PgPoolConfig;
use si_std::{CanonicalFile, CanonicalFileError};
use si_std::CanonicalFileError;
use telemetry::prelude::*;
use thiserror::Error;

Expand Down Expand Up @@ -45,7 +45,8 @@ pub struct Config {
#[builder(default = "NatsConfig::default()")]
nats: NatsConfig,

cyclone_encryption_key_path: CanonicalFile,
#[builder(default = "CryptoConfig::default()")]
crypto: CryptoConfig,

#[builder(default = "default_concurrency_limit()")]
concurrency: usize,
Expand Down Expand Up @@ -78,10 +79,10 @@ impl Config {
self.nats.subject_prefix.as_deref()
}

/// Gets a reference to the config's cyclone public key path.
/// Gets a reference to the config's crypto config.
#[must_use]
pub fn cyclone_encryption_key_path(&self) -> &Path {
self.cyclone_encryption_key_path.as_path()
pub fn crypto(&self) -> &CryptoConfig {
&self.crypto
}

pub fn symmetric_crypto_service(&self) -> &SymmetricCryptoServiceConfig {
Expand All @@ -105,8 +106,8 @@ pub struct ConfigFile {
pg: PgPoolConfig,
#[serde(default)]
nats: NatsConfig,
#[serde(default = "default_cyclone_encryption_key_path")]
cyclone_encryption_key_path: String,
#[serde(default)]
crypto: CryptoConfig,
#[serde(default = "default_concurrency_limit")]
concurrency_limit: usize,
#[serde(default = "random_instance_id")]
Expand All @@ -120,8 +121,8 @@ impl Default for ConfigFile {
Self {
pg: Default::default(),
nats: Default::default(),
cyclone_encryption_key_path: default_cyclone_encryption_key_path(),
concurrency_limit: default_concurrency_limit(),
crypto: Default::default(),
instance_id: random_instance_id(),
symmetric_crypto_service: default_symmetric_crypto_config(),
}
Expand All @@ -141,7 +142,7 @@ impl TryFrom<ConfigFile> for Config {
let mut config = Config::builder();
config.pg_pool(value.pg);
config.nats(value.nats);
config.cyclone_encryption_key_path(value.cyclone_encryption_key_path.try_into()?);
config.crypto(value.crypto);
config.concurrency(value.concurrency_limit);
config.instance_id(value.instance_id);
config.symmetric_crypto_service(value.symmetric_crypto_service.try_into()?);
Expand All @@ -153,10 +154,6 @@ fn random_instance_id() -> String {
Ulid::new().to_string()
}

fn default_cyclone_encryption_key_path() -> String {
"/run/pinga/cyclone_encryption.key".to_string()
}

fn default_symmetric_crypto_config() -> SymmetricCryptoServiceConfigFile {
SymmetricCryptoServiceConfigFile {
active_key: "/run/pinga/donkey.key".into(),
Expand Down Expand Up @@ -199,7 +196,7 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> {
"detected development run",
);

config.cyclone_encryption_key_path = cyclone_encryption_key_path;
config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
active_key: symmetric_crypto_service_key,
extra_keys: vec![],
Expand All @@ -224,7 +221,7 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> {
"detected development run",
);

config.cyclone_encryption_key_path = cyclone_encryption_key_path;
config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
active_key: symmetric_crypto_service_key,
extra_keys: vec![],
Expand Down
15 changes: 9 additions & 6 deletions lib/pinga-server/src/server.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::{io, path::Path, sync::Arc};
use std::{io, sync::Arc};

use dal::{
job::{
Expand All @@ -11,7 +11,9 @@ use dal::{
};
use futures::{FutureExt, Stream, StreamExt};
use nats_subscriber::{Request, SubscriberError};
use si_crypto::{SymmetricCryptoError, SymmetricCryptoService, SymmetricCryptoServiceConfig};
use si_crypto::{
CryptoConfig, SymmetricCryptoError, SymmetricCryptoService, SymmetricCryptoServiceConfig,
};
use si_data_nats::{NatsClient, NatsConfig, NatsError};
use si_data_pg::{PgPool, PgPoolConfig, PgPoolError};
use stream_cancel::StreamExt as StreamCancelStreamExt;
Expand Down Expand Up @@ -99,8 +101,7 @@ impl Server {
pub async fn from_config(config: Config) -> Result<Self> {
dal::init()?;

let encryption_key =
Self::load_encryption_key(config.cyclone_encryption_key_path()).await?;
let encryption_key = Self::load_encryption_key(config.crypto().clone()).await?;
let nats = Self::connect_to_nats(config.nats()).await?;
let pg_pool = Self::create_pg_pool(config.pg_pool()).await?;
let veritech = Self::create_veritech_client(nats.clone());
Expand Down Expand Up @@ -194,8 +195,10 @@ impl Server {
}

#[instrument(name = "pinga.init.load_encryption_key", skip_all)]
async fn load_encryption_key(path: impl AsRef<Path>) -> Result<Arc<CycloneEncryptionKey>> {
Ok(Arc::new(CycloneEncryptionKey::load(path).await?))
async fn load_encryption_key(crypto_config: CryptoConfig) -> Result<Arc<CycloneEncryptionKey>> {
Ok(Arc::new(
CycloneEncryptionKey::from_config(crypto_config).await?,
))
}

#[instrument(name = "pinga.init.connect_to_nats", skip_all)]
Expand Down
25 changes: 12 additions & 13 deletions lib/sdf-server/src/server/config.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use si_crypto::CryptoConfig;
use std::{
env,
net::{SocketAddr, ToSocketAddrs},
Expand Down Expand Up @@ -69,9 +70,11 @@ pub struct Config {
#[builder(default = "MigrationMode::default()")]
migration_mode: MigrationMode,

#[builder(default = "CryptoConfig::default()")]
crypto: CryptoConfig,

jwt_signing_public_key_path: CanonicalFile,

cyclone_encryption_key_path: CanonicalFile,
signup_secret: SensitiveString,
pkgs_path: CanonicalFile,
}
Expand Down Expand Up @@ -113,8 +116,8 @@ impl Config {

/// Gets a reference to the config's cyclone public key path.
#[must_use]
pub fn cyclone_encryption_key_path(&self) -> &Path {
self.cyclone_encryption_key_path.as_path()
pub fn crypto(&self) -> &CryptoConfig {
&self.crypto
}

/// Gets a reference to the config's signup secret.
Expand Down Expand Up @@ -166,8 +169,8 @@ pub struct ConfigFile {
pub migration_mode: MigrationMode,
#[serde(default = "default_jwt_signing_public_key_path")]
pub jwt_signing_public_key_path: String,
#[serde(default = "default_cyclone_encryption_key_path")]
pub cyclone_encryption_key_path: String,
#[serde(default)]
pub crypto: CryptoConfig,
#[serde(default = "default_signup_secret")]
pub signup_secret: SensitiveString,
#[serde(default = "default_pkgs_path")]
Expand All @@ -187,7 +190,7 @@ impl Default for ConfigFile {
nats: Default::default(),
migration_mode: Default::default(),
jwt_signing_public_key_path: default_jwt_signing_public_key_path(),
cyclone_encryption_key_path: default_cyclone_encryption_key_path(),
crypto: Default::default(),
signup_secret: default_signup_secret(),
pkgs_path: default_pkgs_path(),
posthog: Default::default(),
Expand All @@ -212,7 +215,7 @@ impl TryFrom<ConfigFile> for Config {
config.nats(value.nats);
config.migration_mode(value.migration_mode);
config.jwt_signing_public_key_path(value.jwt_signing_public_key_path.try_into()?);
config.cyclone_encryption_key_path(value.cyclone_encryption_key_path.try_into()?);
config.crypto(value.crypto);
config.signup_secret(value.signup_secret);
config.pkgs_path(value.pkgs_path.try_into()?);
config.posthog(value.posthog);
Expand Down Expand Up @@ -255,10 +258,6 @@ fn default_jwt_signing_public_key_path() -> String {
"/run/sdf/jwt_signing_public_key.pem".to_string()
}

fn default_cyclone_encryption_key_path() -> String {
"/run/sdf/cyclone_encryption.key".to_string()
}

fn default_signup_secret() -> SensitiveString {
DEFAULT_SIGNUP_SECRET.into()
}
Expand Down Expand Up @@ -334,7 +333,7 @@ fn buck2_development(config: &mut ConfigFile) -> Result<()> {
);

config.jwt_signing_public_key_path = jwt_signing_public_key_path;
config.cyclone_encryption_key_path = cyclone_encryption_key_path;
config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
active_key: symmetric_crypto_service_key,
extra_keys: vec![],
Expand Down Expand Up @@ -382,7 +381,7 @@ fn cargo_development(dir: String, config: &mut ConfigFile) -> Result<()> {
);

config.jwt_signing_public_key_path = jwt_signing_public_key_path;
config.cyclone_encryption_key_path = cyclone_encryption_key_path;
config.crypto.encryption_key_file = cyclone_encryption_key_path.parse().ok();
config.symmetric_crypto_service = SymmetricCryptoServiceConfigFile {
active_key: symmetric_crypto_service_key,
extra_keys: vec![],
Expand Down
11 changes: 8 additions & 3 deletions lib/sdf-server/src/server/server.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use si_crypto::CryptoConfig;
use std::time::Duration;
use std::{io, net::SocketAddr, path::Path, path::PathBuf};
use std::{io, net::SocketAddr, path::Path, path::PathBuf, sync::Arc};

use axum::routing::IntoMakeService;
use axum::Router;
Expand Down Expand Up @@ -219,8 +220,12 @@ impl Server<(), ()> {
}

#[instrument(name = "sdf.init.load_encryption_key", skip_all)]
pub async fn load_encryption_key(path: impl AsRef<Path>) -> Result<CycloneEncryptionKey> {
Ok(CycloneEncryptionKey::load(path).await?)
pub async fn load_encryption_key(
crypto_config: CryptoConfig,
) -> Result<Arc<CycloneEncryptionKey>> {
Ok(Arc::new(
CycloneEncryptionKey::from_config(crypto_config).await?,
))
}

#[instrument(name = "sdf.init.migrate_database", skip_all)]
Expand Down
1 change: 1 addition & 0 deletions lib/si-crypto/src/cyclone.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub(crate) mod config;
pub(crate) mod decryption_key;
pub(crate) mod encryption_key;
pub(crate) mod key_pair;
Loading

0 comments on commit 610c8a6

Please sign in to comment.