diff --git a/rust/fastsim-core/Cargo.toml b/rust/fastsim-core/Cargo.toml index 82d30c20..4678c5c5 100644 --- a/rust/fastsim-core/Cargo.toml +++ b/rust/fastsim-core/Cargo.toml @@ -40,6 +40,7 @@ include_dir = "0.7.3" itertools = "0.12.0" ndarray-stats = "0.5.1" tempfile = "3.8.1" +url = "2.5.0" [package.metadata] include = [ diff --git a/rust/fastsim-core/src/cycle.rs b/rust/fastsim-core/src/cycle.rs index cf50eedc..f87133ce 100644 --- a/rust/fastsim-core/src/cycle.rs +++ b/rust/fastsim-core/src/cycle.rs @@ -629,6 +629,8 @@ pub struct RustCycle { impl SerdeAPI for RustCycle { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin", "csv"]; const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json", "csv"]; + // is this enough, or do I have to copy paste in the whole to_cache mathod? + const CACHE_FOLDER: &'static str = &"cycles"; // TODO: make this get called somewhere fn init(&mut self) -> anyhow::Result<()> { @@ -726,19 +728,6 @@ impl SerdeAPI for RustCycle { ), } } - - /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder - /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file - fn to_cache>(url: S) { - let url = url.as_ref(); - let url_parts: Vec<&str> = url.split("/").collect(); - let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); - let data_subdirectory = create_project_subdir("cycles").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); - let file_path = data_subdirectory.join(file_name); - // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add - // a bool argument so user can determine whether the file should overwrite an existing file or not - download_file_from_url(url, &file_path); - } } impl TryFrom>> for RustCycle { diff --git a/rust/fastsim-core/src/lib.rs b/rust/fastsim-core/src/lib.rs index 67812580..e6e6bd8d 100644 --- a/rust/fastsim-core/src/lib.rs +++ b/rust/fastsim-core/src/lib.rs @@ -29,7 +29,6 @@ //! ``` extern crate ndarray; -extern crate tempfile; #[macro_use] pub mod macros; diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 076dc45e..79d0db9d 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -5,6 +5,7 @@ use tempfile::tempdir; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json"]; + const CACHE_FOLDER: &'static str = &"rust_objects"; /// Runs any initialization steps that might be needed fn init(&mut self) -> anyhow::Result<()> { @@ -156,42 +157,57 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { Ok(bincode::deserialize(encoded)?) } - /// instantiates an object from a url + /// instantiates an object from a url + /// accepts yaml and json file types + /// # Arguments + /// - url: url (either as a string or url type) to object fn from_url>(url: S) -> anyhow::Result { let url = url.as_ref(); + let url_parts: Vec<&str> = url.split(".").collect(); + let file_extension = url_parts.last().unwrap_or_else(||panic!("Could not determine file type.")).to_lowercase().as_str(); + ensure!( + Self::ACCEPTED_STR_FORMATS.contains(&file_extension) || file_extension == "yml", + "Unsupported format {file_extension:?}, must be one of {:?}", + Self::ACCEPTED_STR_FORMATS + ); + let file_name = "temporary_object.".to_string() + file_extension; let temp_dir = tempdir()?; - let mut file_path = PathBuf::new(); - // do these file types need to be specific to the object? - // TODO: either make funciton work for csv files, or remove from supported file list - if url.ends_with("yaml"){ - let file_path = temp_dir.path().join("temporary_object.yaml"); - } else if url.ends_with("csv"){ - let file_path = temp_dir.path().join("temporary_object.csv"); - } else if url.ends_with("json"){ - let file_path = temp_dir.path().join("temporary_object.json"); - } else { - bail!("Unsupported file type, must be a yaml, json, or csv file."); - } + let file_path = temp_dir.path().join(file_name); download_file_from_url(url, &file_path); - // only works for json and yaml - // seems like I might also be able to use from_reader instead -- which one is preferable? + // seems like I might also be able to use from_reader instead -- which + // one is preferable? + // try switching over to from_reader(), this will require finding + // a crate which takes a url and spits out a Rust object + // that implements std::io::Read Self::from_file(file_path) } - /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder - /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file - /// to save to a folder other than rust_objects for a specific object type, override this default - /// implementation for the Rust object, and in the object-specific implementation, replace - /// "rust_objects" with your choice of folder name - fn to_cache>(url: S) { - let url = url.as_ref(); - let url_parts: Vec<&str> = url.split("/").collect(); - let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); - let data_subdirectory = create_project_subdir("rust_objects").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); + /// takes an instantiated Rust object and saves it in the FASTSim data directory in + /// a rust_objects folder + /// WARNING: if there is a file already in the data subdirectory with the + /// same name, it will be replaced by the new file + /// to save to a folder other than rust_objects for a specific object type, + /// in the object-specific SerdeAPI implementation, redefine the + /// CACHE_FOLDER constant to be your choice of folder name + /// # Arguments + /// - self (rust object) + /// - file_path: path to file within subdirectory. If only the file name is + /// listed, file will sit directly within the subdirectory of + /// the FASTSim data directory. If a path is given, the file will live + /// within the path specified, within the subdirectory CACHE_FOLDER of the + /// FASTSim data directory. + fn to_cache>(&self, file_path: P) -> anyhow::Result<()> { + let file_name = file_path.as_ref().file_name().with_context(||"Could not determine file name")?.to_str().context("Could not determine file name.")?; + let mut subpath = PathBuf::new(); + let file_path_internal = file_path.as_ref().to_str().context("Could not determine file name.")?; + if file_name == file_path_internal { + subpath = PathBuf::from(Self::CACHE_FOLDER); + } else { + subpath = Path::new(Self::CACHE_FOLDER).join(file_path_internal.strip_suffix(file_name).context("Could not determine path to subdirectory.")?); + } + let data_subdirectory = create_project_subdir(subpath).with_context(||"Could not find or build Fastsim data subdirectory.")?; let file_path = data_subdirectory.join(file_name); - // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add - // a bool argument so user can determine whether the file should overwrite an existing file or not - download_file_from_url(url, &file_path); + self.to_file(file_path) } } diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index 80681efa..98cb560c 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -7,6 +7,7 @@ use ndarray::*; use ndarray_stats::QuantileExt; use regex::Regex; use std::collections::HashSet; +use url::{Url, Host, Position}; use crate::imports::*; #[cfg(feature = "pyo3")] @@ -525,6 +526,29 @@ pub fn create_project_subdir>(subpath: P) -> anyhow::Result, P: AsRef>(url: S, subpath: P) -> anyhow::Result<()> { + let url = Url::parse(url.as_ref())?; + let file_name = url + .path_segments() + .and_then(|segments| segments.last()) + .with_context(|| "Could not parse filename from URL: {url:?}")?; + let data_subdirectory = create_project_subdir(subpath).with_context(||"Could not find or build Fastsim data subdirectory.")?; + let file_path = data_subdirectory.join(file_name); + download_file_from_url(url.as_ref(), &file_path)?; + Ok(()) +} + #[cfg(feature = "pyo3")] pub mod array_wrappers { use crate::proc_macros::add_pyo3_api; diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index acc74ece..ee0092d0 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -546,6 +546,7 @@ pub struct RustVehicle { /// RustVehicle rust methods impl RustVehicle { + const VEHICLE_DIRECTORY_URL: &'static str = &"https://github.com/NREL/fastsim-vehicles/blob/main/public/"; /// Sets the following parameters: /// - `ess_mass_kg` /// - `mc_mass_kg` @@ -1034,22 +1035,19 @@ impl RustVehicle { v.set_derived().unwrap(); v } - /// Downloads specified vehicle from vehicle repo into fastsim/python/fastsim/resources/vehdb. - /// - /// Arguments: - /// ---------- - /// vehicle (string): name of the vehicle to be downloaded - // make separate function that gets list of yaml files off of github, then user can choose from that for input -- Rust function - // exposed to python, put in utilities - pub fn from_github(vehicle_file_name: String) { - let mut vehicle_url: String = "https://github.com/NREL/fastsim-vehicles/blob/main/public/".to_string(); - vehicle_url.push_str(&vehicle_file_name); - // find or create fastsim data directory with subdirectory "vehicles" and return path - let data_directory: Option = create_project_subdir("vehicles"); - // see if you can find file_name in vehicle data - // if file doesn't already exist in directory, and once we have file path to directory: - // download_file_from_url(vehicle_url, data_directory) + /// Downloads specified vehicle from vehicle repo into + /// fastsim/python/fastsim/resources/vehdb. + /// # Arguments + /// - vehicle_file_name: file name for vehicle to be downloaded + /// - url: url for vehicle to be downloaded, if None, assumed to be + /// downloaded from vehicle FASTSim repo + /// - cache: If True, function will first check for vehicle in FASTSim data + /// directory, and if vehicle is already there, will use local version. If + /// vehicle is not stored locally, will download and store vehicle for later + /// use. If False, will not check for vehicle locally or store vehicle + /// locally for later use + pub fn from_github_or_url>(vehicle_file_name: S, url: Option, cache: bool) { } } @@ -1079,21 +1077,12 @@ impl Default for RustVehicle { } impl SerdeAPI for RustVehicle { + // is this enough, or do I have to copy paste in the whole to_cache mathod? + const CACHE_FOLDER: &'static str = &"vehicles"; + fn init(&mut self) -> anyhow::Result<()> { self.set_derived() } - /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder - /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file - fn to_cache>(url: S) { - let url = url.as_ref(); - let url_parts: Vec<&str> = url.split("/").collect(); - let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); - let data_subdirectory = create_project_subdir("vehicles").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); - let file_path = data_subdirectory.join(file_name); - // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add - // a bool argument so user can determine whether the file should overwrite an existing file or not - download_file_from_url(url, &file_path); - } } #[cfg(test)]