From 6086e0f05a59d4d128ff460a91ed2de194ac123f Mon Sep 17 00:00:00 2001 From: WaviestBalloon <31796727+WaviestBalloon@users.noreply.github.com> Date: Sat, 26 Aug 2023 02:36:16 +0100 Subject: [PATCH] Implemented multi-threading to extraction phase of install --- src/args/install.rs | 13 +++++-- src/utils/installation.rs | 80 ++++++++++++++++++++++++++++++--------- src/utils/setup.rs | 2 +- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/src/args/install.rs b/src/args/install.rs index 6dc3a09..1ee1ff5 100644 --- a/src/args/install.rs +++ b/src/args/install.rs @@ -7,7 +7,8 @@ const HELP_TEXT: &str = "\nUsage: --install [type] [?removeolder] [?migratefflag fn download_and_install(version_hash: &str, channel: &str, raw_args: Vec>) { let remove_older = argparse::get_param_value(raw_args.clone(), "removeolder").is_empty(); - let migrate_fflags = argparse::get_param_value(raw_args, "migratefflags").is_empty(); + let migrate_fflags = argparse::get_param_value(raw_args.clone(), "migratefflags").is_empty(); + let disallow_multithreading = argparse::get_param_value(raw_args, "nothreads").is_empty() == false; // If there is no --nothreads flag it will return true, which means we need to invert it status(format!("Resolving package manifest for version hash {}...", version_hash)); let package_manifest = installation::get_package_manifest(version_hash.to_string(), channel.to_string()); @@ -33,12 +34,16 @@ fn download_and_install(version_hash: &str, channel: &str, raw_args: Vec let temp_path = format!("{}/cache/{}-download", root_path, version_hash); if setup::confirm_existence(&temp_path) { - warning(format!("{} is already downloaded. Skipping download.", version_hash)); + warning(format!("{} is already downloaded. Skipping download. Use --purge cache to delete previously downloaded files.", version_hash)); return temp_path; } setup::create_dir(&format!("cache/{}-download", version_hash)); @@ -162,32 +162,76 @@ pub fn download_deployment(binary: &str, version_hash: String, channel: &str) -> return temp_path; // Return the cache path to continue with extraction } -pub fn extract_deployment_zips(binary: &str, temp_path: String, extraction_path: String) { +pub fn extract_deployment_zips(binary: &str, temp_path: String, extraction_path: String, disallow_multithreading: bool) { let bindings: &[_] = if binary == "Player" { &PLAYER_EXTRACT_BINDINGS } else { &STUDIO_EXTRACT_BINDINGS }; status(format!("{} files will be extracted!", bindings.len())); + let start_time = std::time::Instant::now(); progress_bar::init_progress_bar_with_eta(bindings.len()); - for (_index, (package, path)) in bindings.iter().enumerate() { - progress_bar::print_progress_bar_info("•", format!("Extracting {package}...").as_str(), progress_bar::Color::Blue, progress_bar::Style::Bold); - if setup::confirm_existence(&format!("{}/{}", extraction_path, path)) && !path.is_empty() { - progress_bar::print_progress_bar_info("!", format!("{} is already extracted. Skipping extraction.", package).as_str(), progress_bar::Color::Red, progress_bar::Style::Bold); - continue; + println!("{}", disallow_multithreading); + if disallow_multithreading { + for (_index, (package, path)) in bindings.iter().enumerate() { + progress_bar::print_progress_bar_info("•", format!("Extracting {package}...").as_str(), progress_bar::Color::Blue, progress_bar::Style::Bold); + + if setup::confirm_existence(&format!("{}/{}", extraction_path, path)) && !path.is_empty() { + progress_bar::print_progress_bar_info("!", format!("{} is already extracted. Skipping extraction.", package).as_str(), progress_bar::Color::LightYellow, progress_bar::Style::Bold); + continue; + } + if path.to_string() != "" { // Create directory if it doesn't exist during extraction + progress_bar::print_progress_bar_info("•", format!("Creating path for {}/{}", extraction_path, path).as_str(), progress_bar::Color::Blue, progress_bar::Style::Bold); + setup::create_dir(&format!("{}/{}", extraction_path, path)); + } + process::Command::new("unzip") + .arg(format!("{}/{}", temp_path, package)) + .arg("-d") + .arg(format!("{}/{}", extraction_path, path)) + .output() + .expect("Failed to execute unzip command"); + + progress_bar::inc_progress_bar(); } - if path.to_string() != "" { // Create directory if it doesn't exist during extraction - progress_bar::print_progress_bar_info("•", format!("Creating path for {}/{}", extraction_path, path).as_str(), progress_bar::Color::Blue, progress_bar::Style::Bold); - setup::create_dir(&format!("{}/{}", extraction_path, path)); + } else { + warning("Multi-threading is enabled for this part! This may cause issues with some files not being extracted properly; If you encounter any issues, re-run this command with the --nothreads flag"); + let threads_available = available_parallelism().unwrap(); + let mut threads = vec![]; + let chunked_files = bindings.chunks(threads_available.into()); + + status(format!("{} threads available, {} chunks created from bindings", threads_available, threads_available)); + for (_index, chunk) in chunked_files.enumerate() { + status(format!("Preparing thread {}...", _index)); + let extract_bind = extraction_path.clone(); + let temp_path_bind = temp_path.clone(); + threads.push(thread::spawn(move || { + for (package, path) in chunk.iter() { + if setup::confirm_existence(&format!("{}/{}", extract_bind, path)) && !path.is_empty() { + warning(format!("[Thread {_index}] {} is already extracted. Skipping extraction.", package)); + continue; + } + if path.to_string() != "" { // Create directory if it doesn't exist during extraction + setup::create_dir(&format!("{}/{}", extract_bind, path)); + } + status(format!("[Thread {_index}] Extracting {}...", package)); + process::Command::new("unzip") + .arg(format!("{}/{}", temp_path_bind, package)) + .arg("-d") + .arg(format!("{}/{}", extract_bind, path)) + .output() + .expect("Failed to execute unzip command"); + + } + + success(format!("[Thread {_index}] Thread quitting!")); + })); } - process::Command::new("unzip") - .arg(format!("{}/{}", temp_path, package)) - .arg("-d") - .arg(format!("{}/{}", extraction_path, path)) - .output() - .expect("Failed to execute unzip command"); - progress_bar::inc_progress_bar(); + for thread in threads { // Wait for all threads to finish + let _ = thread.join(); + } } + progress_bar::finalize_progress_bar(); + success(format!("Decompression task finished in {} milliseconds!", start_time.elapsed().as_millis())); } pub fn get_package_manifest(version_hash: String, channel: String) -> String { diff --git a/src/utils/setup.rs b/src/utils/setup.rs index f7d0567..41210df 100644 --- a/src/utils/setup.rs +++ b/src/utils/setup.rs @@ -42,7 +42,7 @@ pub fn confirm_existence(providedpath: &str) -> bool { // Check whether a item e if providedpath.contains(get_applejuice_dir().to_string().as_str()) { // Sometimes we provide the EXACT path, so we need to check for that and overwrite the other exact path path = providedpath.to_string(); } - + match fs::metadata(path.clone()) { Ok(_) => { return true;