From 47c50ec4a85796b3e0455d9175a54ed80e4d583c Mon Sep 17 00:00:00 2001 From: Garik Asplund Date: Mon, 29 Apr 2024 14:28:41 -0700 Subject: [PATCH 1/4] draft of single component rebuilds for spin watch Signed-off-by: Garik Asplund --- src/commands/watch.rs | 21 +++++---- src/commands/watch/buildifier.rs | 68 ++++++++++++++++++++-------- src/commands/watch/reconfiguriser.rs | 2 +- src/commands/watch/uppificator.rs | 2 +- 4 files changed, 63 insertions(+), 30 deletions(-) diff --git a/src/commands/watch.rs b/src/commands/watch.rs index 3a20b86794..9d3131e39d 100644 --- a/src/commands/watch.rs +++ b/src/commands/watch.rs @@ -101,10 +101,13 @@ impl WatchCommand { // We create the build-related objects even if skip_build is on - the cost is insignificant, // and it saves having to check options all over the place. - let (artifact_tx, artifact_rx) = tokio::sync::watch::channel(Uuid::new_v4()); + let (artifact_tx, artifact_rx) = + tokio::sync::watch::channel((Uuid::new_v4(), "artifact".to_owned())); let (pause_tx, pause_rx) = tokio::sync::mpsc::channel(1); - let (source_code_tx, source_code_rx) = tokio::sync::watch::channel(Uuid::new_v4()); - let (manifest_tx, manifest_rx) = tokio::sync::watch::channel(Uuid::new_v4()); + let (source_code_tx, source_code_rx) = + tokio::sync::watch::channel((Uuid::new_v4(), "".to_owned())); + let (manifest_tx, manifest_rx) = + tokio::sync::watch::channel((Uuid::new_v4(), "manifest".to_owned())); let (stop_tx, stop_rx) = tokio::sync::watch::channel(Uuid::new_v4()); let mut buildifier = Buildifier { @@ -239,7 +242,7 @@ impl WatchCommand { manifest_file: &Path, manifest_dir: &Path, filter_factory: Box, - notifier: Arc>, + notifier: Arc>, impact_description: &'static str, ) -> anyhow::Result<(ReconfigurableWatcher, tokio::task::JoinHandle<()>)> { let rtf = RuntimeConfigFactory { @@ -269,7 +272,7 @@ pub struct RuntimeConfigFactory { manifest_file: PathBuf, manifest_dir: PathBuf, filter_factory: Box, - notifier: Arc>, + notifier: Arc>, impact_description: &'static str, debounce: Duration, } @@ -295,19 +298,19 @@ impl RuntimeConfigFactory { } // This is the watchexec action handler that triggers the Uppificator -// to reload or Builidifer to rebuild by sending a notification. +// to reload or Buildifier to rebuild by sending a notification. // It is a struct rather than a closure because this makes it easier // for the compiler to confirm that all the data lives long enough and // is thread-safe for async stuff. struct NotifyOnFileChange { despurifier: despurifier::Despurifier, - notifier: Arc>, + notifier: Arc>, impact_description: &'static str, } impl NotifyOnFileChange { fn new( - notifier: Arc>, + notifier: Arc>, impact_description: &'static str, ) -> Self { Self { @@ -331,7 +334,7 @@ impl watchexec::handler::Handler for NotifyOnFileChan self.impact_description, paths_of(&action) ); - _ = self.notifier.send(Uuid::new_v4()); + _ = self.notifier.send((Uuid::new_v4(), paths_of(&action))); } action.outcome(watchexec::action::Outcome::DoNothing); Ok::<(), Box<(dyn std::error::Error + 'static)>>(()) diff --git a/src/commands/watch/buildifier.rs b/src/commands/watch/buildifier.rs index 74bd433d07..88d5676428 100644 --- a/src/commands/watch/buildifier.rs +++ b/src/commands/watch/buildifier.rs @@ -3,13 +3,13 @@ use std::path::PathBuf; use uuid::Uuid; use super::uppificator::Pause; - +#[derive(Debug)] pub(crate) struct Buildifier { pub spin_bin: PathBuf, pub manifest: PathBuf, pub clear_screen: bool, pub has_ever_built: bool, - pub watched_changes: tokio::sync::watch::Receiver, // TODO: refine which component(s) a change affects + pub watched_changes: tokio::sync::watch::Receiver<(Uuid, String)>, // TODO: refine which component(s) a change affects pub uppificator_pauser: tokio::sync::mpsc::Sender, } @@ -28,9 +28,13 @@ impl Buildifier { break; } - let build_result = self.build_once().await; + let (_, ref changed_paths) = self.watched_changes.borrow_and_update().clone(); + tracing::debug!("Detected changes in: {}", changed_paths); + + let build_component_result = self.build_component(changed_paths).await; + if !self.has_ever_built { - self.has_ever_built = matches!(build_result, Ok(true)); + self.has_ever_built = matches!(build_component_result, Ok(true)); } if self.has_ever_built { @@ -45,27 +49,53 @@ impl Buildifier { } } - pub(crate) async fn build_once(&mut self) -> std::io::Result { + pub(crate) async fn build_component( + &mut self, + mut component_path: &str, + ) -> std::io::Result { + let manifest = spin_manifest::manifest_from_file(&self.manifest).unwrap(); + let inner_ids: Vec<&str> = manifest.components.keys().map(|key| key.as_ref()).collect(); + + if !inner_ids.iter().any(|id| component_path.contains(id)) && !component_path.is_empty() { + component_path = inner_ids.first().cloned().unwrap_or_default(); + } + + for inner_id in inner_ids { + if component_path.contains(inner_id) { + component_path = inner_id; + break; + } + } + loop { let mut cmd = tokio::process::Command::new(&self.spin_bin); - cmd.arg("build").arg("-f").arg(&self.manifest); + + if component_path.is_empty() { + cmd.arg("build").arg("-f").arg(&self.manifest); + } else { + cmd.arg("build") + .arg("-c") + .arg(component_path) + .arg("-f") + .arg(&self.manifest); + } + let mut child = cmd.group_spawn()?; tokio::select! { - exit_status = child.wait() => { - // It reports its own errors so we only care about success or failure (and then only for - // the initial build). - return Ok(exit_status?.success()); - } - _ = self.watched_changes.changed() => { - tracing::debug!("Cancelling build as there are new changes to process"); - child.kill()?; - if self.clear_screen { - _ = clearscreen::clear(); + exit_status = child.wait() => { + // It reports its own errors so we only care about success or failure (and then only for + // the initial build). + return Ok(exit_status?.success()); + } + _ = self.watched_changes.changed() => { + tracing::debug!("Cancelling build as there are new changes to process"); + child.kill()?; + if self.clear_screen { + _ = clearscreen::clear(); + } + continue; } - continue; - } - } } } diff --git a/src/commands/watch/reconfiguriser.rs b/src/commands/watch/reconfiguriser.rs index 9107a0f201..d0a69ced63 100644 --- a/src/commands/watch/reconfiguriser.rs +++ b/src/commands/watch/reconfiguriser.rs @@ -1,7 +1,7 @@ use uuid::Uuid; pub(crate) struct Reconfiguriser { - pub manifest_changes: tokio::sync::watch::Receiver, + pub manifest_changes: tokio::sync::watch::Receiver<(Uuid, String)>, pub artifact_watcher: super::ReconfigurableWatcher, pub build_watcher: super::ReconfigurableWatcher, } diff --git a/src/commands/watch/uppificator.rs b/src/commands/watch/uppificator.rs index 2e22bc39e0..fd3abbc266 100644 --- a/src/commands/watch/uppificator.rs +++ b/src/commands/watch/uppificator.rs @@ -7,7 +7,7 @@ pub(crate) struct Uppificator { pub up_args: Vec, pub manifest: PathBuf, pub clear_screen: bool, - pub watched_changes: tokio::sync::watch::Receiver, + pub watched_changes: tokio::sync::watch::Receiver<(Uuid, String)>, pub pause_feed: tokio::sync::mpsc::Receiver, pub stopper: tokio::sync::watch::Receiver, } From 915017cebdec38c94ab545965ff71de5e3e29644 Mon Sep 17 00:00:00 2001 From: Garik Asplund Date: Mon, 29 Apr 2024 22:05:49 -0700 Subject: [PATCH 2/4] grabbing component ids via routes with hashmap for rebuilds Signed-off-by: Garik Asplund --- src/commands/watch.rs | 2 +- src/commands/watch/buildifier.rs | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/commands/watch.rs b/src/commands/watch.rs index 9d3131e39d..ca2a5fc617 100644 --- a/src/commands/watch.rs +++ b/src/commands/watch.rs @@ -105,7 +105,7 @@ impl WatchCommand { tokio::sync::watch::channel((Uuid::new_v4(), "artifact".to_owned())); let (pause_tx, pause_rx) = tokio::sync::mpsc::channel(1); let (source_code_tx, source_code_rx) = - tokio::sync::watch::channel((Uuid::new_v4(), "".to_owned())); + tokio::sync::watch::channel((Uuid::new_v4(), "THIS_IS_ THE-FIRST BUILD".to_owned())); let (manifest_tx, manifest_rx) = tokio::sync::watch::channel((Uuid::new_v4(), "manifest".to_owned())); let (stop_tx, stop_rx) = tokio::sync::watch::channel(Uuid::new_v4()); diff --git a/src/commands/watch/buildifier.rs b/src/commands/watch/buildifier.rs index 88d5676428..b084133140 100644 --- a/src/commands/watch/buildifier.rs +++ b/src/commands/watch/buildifier.rs @@ -1,5 +1,5 @@ use command_group::AsyncCommandGroup; -use std::path::PathBuf; +use std::{collections::HashMap, path::PathBuf}; use uuid::Uuid; use super::uppificator::Pause; @@ -54,23 +54,37 @@ impl Buildifier { mut component_path: &str, ) -> std::io::Result { let manifest = spin_manifest::manifest_from_file(&self.manifest).unwrap(); - let inner_ids: Vec<&str> = manifest.components.keys().map(|key| key.as_ref()).collect(); - if !inner_ids.iter().any(|id| component_path.contains(id)) && !component_path.is_empty() { - component_path = inner_ids.first().cloned().unwrap_or_default(); - } - - for inner_id in inner_ids { - if component_path.contains(inner_id) { + let id_to_workdir: HashMap<_, _> = manifest + .components + .iter() + .filter_map(|(id, component)| { + component.build.as_ref().map(|build_config| { + ( + id.as_ref(), + build_config.workdir.clone().unwrap_or("".to_owned()), + ) + }) + }) + .collect(); + + for (inner_id, workdir) in id_to_workdir.iter() { + + if !workdir.is_empty() && component_path.contains(workdir) { component_path = inner_id; break; } + if component_path != "THIS_IS_ THE-FIRST BUILD" && component_path.contains(workdir) { + component_path = inner_id; + } } + println!("COMPONENT PATH: {:?}", &component_path); + loop { let mut cmd = tokio::process::Command::new(&self.spin_bin); - if component_path.is_empty() { + if component_path == "THIS_IS_ THE-FIRST BUILD" { cmd.arg("build").arg("-f").arg(&self.manifest); } else { cmd.arg("build") From 9b2b26f4b239ad6f50df091659c53a74bfc7f800 Mon Sep 17 00:00:00 2001 From: Garik Asplund Date: Tue, 30 Apr 2024 11:01:28 -0700 Subject: [PATCH 3/4] multiple components rebuilt including src after changes Signed-off-by: Garik Asplund --- src/commands/watch/buildifier.rs | 40 ++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/commands/watch/buildifier.rs b/src/commands/watch/buildifier.rs index b084133140..e32f9b1bb8 100644 --- a/src/commands/watch/buildifier.rs +++ b/src/commands/watch/buildifier.rs @@ -28,10 +28,10 @@ impl Buildifier { break; } - let (_, ref changed_paths) = self.watched_changes.borrow_and_update().clone(); - tracing::debug!("Detected changes in: {}", changed_paths); + let (_, ref changed_path) = self.watched_changes.borrow_and_update().clone(); + tracing::debug!("Detected changes in: {:?}", changed_path); - let build_component_result = self.build_component(changed_paths).await; + let build_component_result = self.build_component(changed_path).await; if !self.has_ever_built { self.has_ever_built = matches!(build_component_result, Ok(true)); @@ -49,10 +49,7 @@ impl Buildifier { } } - pub(crate) async fn build_component( - &mut self, - mut component_path: &str, - ) -> std::io::Result { + pub(crate) async fn build_component(&mut self, component_path: &str) -> std::io::Result { let manifest = spin_manifest::manifest_from_file(&self.manifest).unwrap(); let id_to_workdir: HashMap<_, _> = manifest @@ -68,28 +65,35 @@ impl Buildifier { }) .collect(); - for (inner_id, workdir) in id_to_workdir.iter() { - - if !workdir.is_empty() && component_path.contains(workdir) { - component_path = inner_id; - break; - } - if component_path != "THIS_IS_ THE-FIRST BUILD" && component_path.contains(workdir) { - component_path = inner_id; + let component_paths: Vec<&str> = component_path.split(", ").collect(); + let mut component_ids = Vec::new(); + let source_dir = id_to_workdir + .iter() + .find(|(_, value)| value.is_empty()) + .map(|(key, _)| key); + + for changed_path in &component_paths { + for (inner_id, workdir) in id_to_workdir.iter() { + if !workdir.is_empty() && changed_path.contains(workdir) { + component_ids.push(inner_id); + break; + } } } - println!("COMPONENT PATH: {:?}", &component_path); + if component_ids.len() != component_paths.len() { + component_ids.push(source_dir.unwrap()); + } loop { let mut cmd = tokio::process::Command::new(&self.spin_bin); - if component_path == "THIS_IS_ THE-FIRST BUILD" { + if component_paths.contains(&"THIS_IS_ THE-FIRST BUILD") { cmd.arg("build").arg("-f").arg(&self.manifest); } else { cmd.arg("build") .arg("-c") - .arg(component_path) + .args(&component_ids) .arg("-f") .arg(&self.manifest); } From 07e83bdb01a3f9f211b556d63b11d1dbb71a4ad1 Mon Sep 17 00:00:00 2001 From: Garik Asplund Date: Tue, 30 Apr 2024 11:32:14 -0700 Subject: [PATCH 4/4] accounting for changes in manifest for full rebuild Signed-off-by: Garik Asplund --- src/commands/watch/buildifier.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/commands/watch/buildifier.rs b/src/commands/watch/buildifier.rs index e32f9b1bb8..4d59c52081 100644 --- a/src/commands/watch/buildifier.rs +++ b/src/commands/watch/buildifier.rs @@ -88,7 +88,9 @@ impl Buildifier { loop { let mut cmd = tokio::process::Command::new(&self.spin_bin); - if component_paths.contains(&"THIS_IS_ THE-FIRST BUILD") { + if component_paths.contains(&"THIS_IS_ THE-FIRST BUILD") + || component_paths.contains(&self.manifest.to_str().unwrap()) + { cmd.arg("build").arg("-f").arg(&self.manifest); } else { cmd.arg("build")