diff --git a/Cargo.lock b/Cargo.lock index a330d2d99a5..70f5bf1a903 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -463,9 +463,9 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b" +source = "git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9#24a74d0c76b6a63961ecef76acb1516b6e66c5c9" dependencies = [ - "bhyve_api_sys 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)", + "bhyve_api_sys 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9)", "libc", "strum", ] @@ -483,7 +483,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b" +source = "git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9#24a74d0c76b6a63961ecef76acb1516b6e66c5c9" dependencies = [ "libc", "strum", @@ -1434,7 +1434,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=e10f8793f8414fdb9a165219f17e45fa014d088b#e10f8793f8414fdb9a165219f17e45fa014d088b" +source = "git+https://github.com/oxidecomputer/crucible?rev=e58ca3693cb9ce0438947beba10e97ee38a0966b#e58ca3693cb9ce0438947beba10e97ee38a0966b" dependencies = [ "anyhow", "chrono", @@ -1450,7 +1450,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=e10f8793f8414fdb9a165219f17e45fa014d088b#e10f8793f8414fdb9a165219f17e45fa014d088b" +source = "git+https://github.com/oxidecomputer/crucible?rev=e58ca3693cb9ce0438947beba10e97ee38a0966b#e58ca3693cb9ce0438947beba10e97ee38a0966b" dependencies = [ "anyhow", "chrono", @@ -1467,7 +1467,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=e10f8793f8414fdb9a165219f17e45fa014d088b#e10f8793f8414fdb9a165219f17e45fa014d088b" +source = "git+https://github.com/oxidecomputer/crucible?rev=e58ca3693cb9ce0438947beba10e97ee38a0966b#e58ca3693cb9ce0438947beba10e97ee38a0966b" dependencies = [ "crucible-workspace-hack", "libc", @@ -3539,7 +3539,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "bhyve_api 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)", + "bhyve_api 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9)", "byteorder", "camino", "camino-tempfile", @@ -5677,7 +5677,7 @@ dependencies = [ "pq-sys", "pretty_assertions", "progenitor-client", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)", + "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9)", "rand 0.8.5", "rcgen", "ref-cast", @@ -5932,7 +5932,7 @@ dependencies = [ "oximeter-producer", "oxnet", "pretty_assertions", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)", + "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9)", "propolis-mock-server", "rand 0.8.5", "rcgen", @@ -7444,7 +7444,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b" +source = "git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9#24a74d0c76b6a63961ecef76acb1516b6e66c5c9" dependencies = [ "async-trait", "base64 0.21.7", @@ -7486,7 +7486,7 @@ dependencies = [ [[package]] name = "propolis-mock-server" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b" +source = "git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9#24a74d0c76b6a63961ecef76acb1516b6e66c5c9" dependencies = [ "anyhow", "atty", @@ -7496,7 +7496,7 @@ dependencies = [ "futures", "hyper 0.14.28", "progenitor", - "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b)", + "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9)", "rand 0.8.5", "reqwest", "schemars", @@ -7528,7 +7528,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=66d1ee7d4a5829dbbf02a152091ea051023b5b8b#66d1ee7d4a5829dbbf02a152091ea051023b5b8b" +source = "git+https://github.com/oxidecomputer/propolis?rev=24a74d0c76b6a63961ecef76acb1516b6e66c5c9#24a74d0c76b6a63961ecef76acb1516b6e66c5c9" dependencies = [ "schemars", "serde", diff --git a/Cargo.toml b/Cargo.toml index 578c4de5a1e..cfa6cc58452 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -289,9 +289,9 @@ cookie = "0.18" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "e10f8793f8414fdb9a165219f17e45fa014d088b" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "e10f8793f8414fdb9a165219f17e45fa014d088b" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "e10f8793f8414fdb9a165219f17e45fa014d088b" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "e58ca3693cb9ce0438947beba10e97ee38a0966b" } csv = "1.3.0" curve25519-dalek = "4" datatest-stable = "0.2.9" @@ -445,9 +445,9 @@ prettyplease = { version = "0.2.20", features = ["verbatim"] } proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b" } -propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b" } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "24a74d0c76b6a63961ecef76acb1516b6e66c5c9" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "24a74d0c76b6a63961ecef76acb1516b6e66c5c9" } +propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "24a74d0c76b6a63961ecef76acb1516b6e66c5c9" } proptest = "1.4.0" quote = "1.0" rand = "0.8.5" diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 9bd99c23d3e..996ee446f20 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -520,13 +520,11 @@ In this example we'll update the recovery silo so we can provision instances dir [source, console] ---- -$ oxide api /v1/system/silos/recovery/quotas --method PUT --input - </crucible.sha256.txt -source.sha256 = "e3f7ace2da974da6379485c6871ffc88ce7430c9ff519d5ac9dc04c55ce9f189" +source.sha256 = "18dc942935af51ea03161498c44b56bb85b4a5d2499a5ff19ba87c0284e34742" output.type = "zone" output.intermediate_only = true @@ -515,10 +515,10 @@ service_name = "crucible_pantry_prebuilt" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "e10f8793f8414fdb9a165219f17e45fa014d088b" +source.commit = "e58ca3693cb9ce0438947beba10e97ee38a0966b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "a5505a51871e910735f449acb6887610a08244773e19d474f66cb00e533842d0" +source.sha256 = "9067426bb77ce091116a034a19abb96ecd98cc8ac228a147bd3c3283e40d527d" output.type = "zone" output.intermediate_only = true @@ -532,7 +532,7 @@ service_name = "crucible_dtrace" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "e10f8793f8414fdb9a165219f17e45fa014d088b" +source.commit = "e58ca3693cb9ce0438947beba10e97ee38a0966b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-dtrace.sha256.txt source.sha256 = "29e79e0df79fc46b244745fac807b0e1817954c0a17b1923d725f257d31010e9" @@ -546,10 +546,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "66d1ee7d4a5829dbbf02a152091ea051023b5b8b" +source.commit = "24a74d0c76b6a63961ecef76acb1516b6e66c5c9" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "168d4f061245bae749926104a77d087d144ee4aea8cc6d2a49284ee26ad5ffe9" +source.sha256 = "e716c1fa5c4ee402f8e3a63f667bbcda744f378305f91ff333bebafe5c20d60b" output.type = "zone" [package.mg-ddm-gz] diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 23c897a6438..407254419cf 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -1023,7 +1023,7 @@ async fn inventory( async fn sled_identifiers( request_context: RequestContext, ) -> Result, HttpError> { - Ok(HttpResponseOk(request_context.context().sled_identifiers().await)) + Ok(HttpResponseOk(request_context.context().sled_identifiers())) } /// Get the internal state of the local bootstore node diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 8bafad8b6bd..d19fed80aac 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -31,12 +31,11 @@ use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::svc::wait_for_service; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; -use omicron_common::address::NEXUS_INTERNAL_PORT; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ - NetworkInterface, SourceNatConfig, + NetworkInterface, SledIdentifiers, SourceNatConfig, }; use omicron_common::backoff; use omicron_common::zpool_name::ZpoolName; @@ -48,7 +47,7 @@ use sled_storage::dataset::ZONE_DATASET; use sled_storage::manager::StorageHandle; use slog::Logger; use std::net::IpAddr; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::SocketAddr; use std::sync::Arc; use tokio::sync::{mpsc, oneshot}; use uuid::Uuid; @@ -977,7 +976,9 @@ impl Instance { /// instance manager's tracking table. /// * `state`: The initial state of this instance. /// * `services`: A set of instance manager-provided services. + /// * `sled_identifiers`: Sled-related metadata used to track statistics. /// * `metadata`: Instance-related metadata used to track statistics. + #[allow(clippy::too_many_arguments)] pub(crate) fn new( log: Logger, id: InstanceUuid, @@ -985,6 +986,7 @@ impl Instance { ticket: InstanceTicket, state: InstanceInitialState, services: InstanceManagerServices, + sled_identifiers: SledIdentifiers, metadata: InstanceMetadata, ) -> Result { info!(log, "initializing new Instance"; @@ -1043,6 +1045,15 @@ impl Instance { let (tx, rx) = mpsc::channel(QUEUE_SIZE); let (tx_monitor, rx_monitor) = mpsc::channel(1); + let metadata = propolis_client::types::InstanceMetadata { + project_id: metadata.project_id, + silo_id: metadata.silo_id, + sled_id: sled_identifiers.sled_id, + sled_model: sled_identifiers.model, + sled_revision: sled_identifiers.revision, + sled_serial: sled_identifiers.serial, + }; + let runner = InstanceRunner { log: log.new(o!("instance_id" => id.to_string())), should_terminate: false, @@ -1062,7 +1073,7 @@ impl Instance { // TODO: we should probably make propolis aligned with // InstanceCpuCount here, to avoid any casting... vcpus: hardware.properties.ncpus.0 as u8, - metadata: metadata.into(), + metadata, }, propolis_id, propolis_addr, @@ -1446,28 +1457,6 @@ impl InstanceRunner { .await?; let gateway = self.port_manager.underlay_ip(); - - // TODO: We should not be using the resolver here to lookup the Nexus IP - // address. It would be preferable for Propolis, and through Propolis, - // Oximeter, to access the Nexus internal interface using a progenitor - // resolver that relies on a DNS resolver. - // - // - With the current implementation: if Nexus' IP address changes, this - // breaks. - // - With a DNS resolver: the metric producer would be able to continue - // sending requests to new servers as they arise. - let metric_ip = self - .nexus_client - .resolver() - .lookup_ipv6(internal_dns::ServiceName::Nexus) - .await?; - let metric_addr = SocketAddr::V6(SocketAddrV6::new( - metric_ip, - NEXUS_INTERNAL_PORT, - 0, - 0, - )); - let config = PropertyGroupBuilder::new("config") .add_property( "datalink", @@ -1485,7 +1474,9 @@ impl InstanceRunner { "astring", &self.propolis_addr.port().to_string(), ) - .add_property("metric_addr", "astring", &metric_addr.to_string()); + // Allow Propolis's `oximeter_producer::Server` to use DNS, based on + // the underlay IP address supplied in `listen_addr` above. + .add_property("metric_addr", "astring", "dns"); let profile = ProfileBuilder::new("omicron").add_service( ServiceBuilder::new("system/illumos/propolis-server").add_instance( @@ -1620,6 +1611,7 @@ mod tests { ByteCount, Generation, Hostname, InstanceCpuCount, }; use omicron_common::api::internal::nexus::{InstanceProperties, VmmState}; + use omicron_common::api::internal::shared::SledIdentifiers; use omicron_common::FileKv; use sled_storage::manager_test_harness::StorageManagerTestHarness; use std::net::Ipv6Addr; @@ -1807,6 +1799,13 @@ mod tests { silo_id: Uuid::new_v4(), project_id: Uuid::new_v4(), }; + let sled_identifiers = SledIdentifiers { + rack_id: Uuid::new_v4(), + sled_id: Uuid::new_v4(), + model: "fake-model".into(), + revision: 1, + serial: "fake-serial".into(), + }; Instance::new( log.new(o!("component" => "Instance")), @@ -1815,6 +1814,7 @@ mod tests { ticket, initial_state, services, + sled_identifiers, metadata, ) .unwrap() @@ -2195,6 +2195,13 @@ mod tests { silo_id: Uuid::new_v4(), project_id: Uuid::new_v4(), }; + let sled_identifiers = SledIdentifiers { + rack_id: Uuid::new_v4(), + sled_id: Uuid::new_v4(), + model: "fake-model".into(), + revision: 1, + serial: "fake-serial".into(), + }; mgr.ensure_registered( instance_id, @@ -2203,6 +2210,7 @@ mod tests { instance_runtime, vmm_runtime, propolis_addr, + sled_identifiers, metadata, ) .await diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index cfb96fb8c9b..baa377a0645 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -28,6 +28,7 @@ use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::VmmRuntimeState; +use omicron_common::api::internal::shared::SledIdentifiers; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; use sled_storage::manager::StorageHandle; @@ -148,6 +149,7 @@ impl InstanceManager { instance_runtime: InstanceRuntimeState, vmm_runtime: VmmRuntimeState, propolis_addr: SocketAddr, + sled_identifiers: SledIdentifiers, metadata: InstanceMetadata, ) -> Result { let (tx, rx) = oneshot::channel(); @@ -160,6 +162,7 @@ impl InstanceManager { instance_runtime, vmm_runtime, propolis_addr, + sled_identifiers, metadata, tx, }) @@ -362,6 +365,7 @@ enum InstanceManagerRequest { instance_runtime: InstanceRuntimeState, vmm_runtime: VmmRuntimeState, propolis_addr: SocketAddr, + sled_identifiers: SledIdentifiers, metadata: InstanceMetadata, tx: oneshot::Sender>, }, @@ -485,10 +489,20 @@ impl InstanceManagerRunner { instance_runtime, vmm_runtime, propolis_addr, + sled_identifiers, metadata, tx, }) => { - tx.send(self.ensure_registered(instance_id, propolis_id, hardware, instance_runtime, vmm_runtime, propolis_addr, metadata).await).map_err(|_| Error::FailedSendClientClosed) + tx.send(self.ensure_registered( + instance_id, + propolis_id, + hardware, + instance_runtime, + vmm_runtime, + propolis_addr, + sled_identifiers, + metadata + ).await).map_err(|_| Error::FailedSendClientClosed) }, Some(EnsureUnregistered { instance_id, tx }) => { self.ensure_unregistered(tx, instance_id).await @@ -572,6 +586,7 @@ impl InstanceManagerRunner { instance_runtime: InstanceRuntimeState, vmm_runtime: VmmRuntimeState, propolis_addr: SocketAddr, + sled_identifiers: SledIdentifiers, metadata: InstanceMetadata, ) -> Result { info!( @@ -638,6 +653,7 @@ impl InstanceManagerRunner { ticket, state, services, + sled_identifiers, metadata, )?; let _old = diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index f98d5c82e5f..8ce16e91d24 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -93,12 +93,6 @@ pub struct InstanceMetadata { pub project_id: Uuid, } -impl From for propolis_client::types::InstanceMetadata { - fn from(md: InstanceMetadata) -> Self { - Self { silo_id: md.silo_id, project_id: md.project_id } - } -} - /// The body of a request to ensure that a instance and VMM are known to a sled /// agent. #[derive(Serialize, Deserialize, JsonSchema)] diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 32d3d29c5df..9acfa24b3dc 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -321,6 +321,24 @@ impl SledAgent { .contains_key(&instance_id.into_untyped_uuid()) .await { + let metadata = propolis_client::types::InstanceMetadata { + project_id: metadata.project_id, + silo_id: metadata.silo_id, + sled_id: self.id, + sled_model: self + .config + .hardware + .baseboard + .model() + .to_string(), + sled_revision: self.config.hardware.baseboard.revision(), + sled_serial: self + .config + .hardware + .baseboard + .identifier() + .to_string(), + }; let properties = propolis_client::types::InstanceProperties { id: propolis_id.into_untyped_uuid(), name: hardware.properties.hostname.to_string(), @@ -329,7 +347,7 @@ impl SledAgent { bootrom_id: Uuid::default(), memory: hardware.properties.memory.to_whole_mebibytes(), vcpus: hardware.properties.ncpus.0 as u8, - metadata: metadata.into(), + metadata, }; let body = propolis_client::types::InstanceEnsureRequest { properties, diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 163a8c356e8..92dcfd2dfe0 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -985,6 +985,7 @@ impl SledAgent { instance_runtime, vmm_runtime, propolis_addr, + self.sled_identifiers(), metadata, ) .await @@ -1214,7 +1215,7 @@ impl SledAgent { /// interested in the switch identifiers, MGS is the current best way to do /// that, by asking for the local switch's slot, and then that switch's SP /// state. - pub(crate) async fn sled_identifiers(&self) -> SledIdentifiers { + pub(crate) fn sled_identifiers(&self) -> SledIdentifiers { let baseboard = self.inner.hardware.baseboard(); SledIdentifiers { rack_id: self.inner.start_request.body.rack_id,