diff --git a/Cargo.lock b/Cargo.lock index 5a91816..c47707e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -608,9 +608,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.13.25" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "2994bee4a3a6a51eb90c218523be382fd7ea09b16380b9312e9dbe955ff7c7d1" dependencies = [ "bitflags", "libc", @@ -723,7 +723,7 @@ dependencies = [ "clap", "compress-tools", "derive_more", - "dirs 3.0.2", + "dirs 4.0.0", "fs2", "fs_extra", "futures", @@ -740,7 +740,7 @@ dependencies = [ "rayon", "regex", "reqwest", - "semver 0.11.0", + "semver 1.0.14", "serde", "serde_json", "serde_yaml", @@ -1003,9 +1003,9 @@ dependencies = [ [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.14.0+1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "47a00859c70c8a4f7218e6d1cc32875c4b55f6799445b842b0d8ed5e4c3d959b" dependencies = [ "cc", "libc", @@ -1350,9 +1350,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.40" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -1615,18 +1615,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.138" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1578c6245786b9d168c5447eeacfb96856573ca56c9d68fdcf394be134882a47" +checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.138" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023e9b1467aef8a10fb88f25611870ada9800ef7e22afce356bb0d2387b6f27c" +checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" dependencies = [ "proc-macro2", "quote", @@ -1659,9 +1659,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ "indexmap", "ryu", @@ -1742,9 +1742,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" [[package]] name = "syn" -version = "1.0.98" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -2007,9 +2007,9 @@ dependencies = [ [[package]] name = "urlencoding" -version = "1.3.3" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" [[package]] name = "vcpkg" diff --git a/Cargo.toml b/Cargo.toml index f70e2f3..1c563ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = ["crates/app", "crates/common", "crates/procmacro", "crates/generator" default-members = ["crates/app"] [profile.release] -opt-level = 'z' +strip = true +opt-level = "z" lto = true codegen-units = 1 diff --git a/Makefile b/Makefile index ad32771..5298820 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ HUBER_ARTIFACT := $(shell $(CURDIR)/hack/huber-artifact-name.sh) MANAGED_PKG_ROOT_DIR := $(CURDIR)/generated PLATFORMS ?= linux/arm64 # for multi arch HUBER_BIN=$(CURDIR)/target/debug/huber +FORCE_GENERATE ?= false .PHONY: help help: @@ -18,7 +19,7 @@ setup-dev: ## Setup development environment $(CURDIR)/hack/setup-dev.sh .PHONY: build -build: fmt ## Build binaries +build: fix fmt ## Build binaries cargo build $(CARGO_OPTS) .PHONY: test @@ -50,12 +51,14 @@ clean: ## Clean build caches fix: ## Fix code cargo fix --allow-dirty || cargo fix --allow-staged +# Examples: +# FORCE_GENERATE=true|false make generate .PHONY: generate generate: ## Generate managed package list @echo "! Must have GITHUB_TOKEN to automatically generate package description" GITHUB_TOKEN=$(GITHUB_TOKEN) cargo build -vv --package=huber-generator GITHUB_KEY=$(GITHUB_KEY) $(MAKE) build && \ - (MANAGED_PKG_ROOT_DIR=$(CURDIR)/generated $(HUBER_BIN) search | xargs -0 $(CURDIR)/hack/generate-packages.md.sh) + (MANAGED_PKG_ROOT_DIR=$(CURDIR)/generated FORCE_GENERATE=$(FORCE_GENERATE) $(HUBER_BIN) search | xargs -0 $(CURDIR)/hack/generate-packages.md.sh) .PHONY: checksum checksum: ## Generate checksum files for built executables @@ -66,10 +69,14 @@ udep: ## Check unused dependencies cargo install cargo-udeps --locked cargo +nightly udeps +# Examples: +# PLATFORMS=linux/arm64 make build-multiarch .PHONY: build-multiarch build-multiarch: ## Build binaries for linux multiple architectures PLATFORMS=$(PLATFORMS) BUILD_TARGET=debug MAKE_TARGET="test build" $(CURDIR)/hack/build-multiarch.sh +# Examples: +# PLATFORMS=linux/arm64 make release-multiarch .PHONY: release-multiarch release-multiarch: ## Release binaries for linux multiple archite PLATFORMS=$(PLATFORMS) BUILD_TARGET=release OUTPUT_DIR=$(OUTPUT_DIR) MAKE_TARGET=release $(CURDIR)/hack/build-multiarch.sh @@ -80,8 +87,10 @@ release-multiarch: ## Release binaries for linux multiple archite verify: ## Verify Huber commands via the local package generated folder MANAGED_PKG_ROOT_DIR=$(MANAGED_PKG_ROOT_DIR) $(HUBER_BIN) $(CMD) +# Examples: +# CMD=search make verify-compatible .PHONY: verify-compatible -verify-compatible: ## Verify the current Huber commands via the local package generated folder +verify-compatible: ## Verify the installed Huber commands compatible with the new local package generated folder MANAGED_PKG_ROOT_DIR=$(MANAGED_PKG_ROOT_DIR) $(shell which huber) $(CMD) .PHONY: publish diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index a6a8a9f..4e1828b 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -24,11 +24,11 @@ log = "0.4" pretty_env_logger = "0.4" lazy_static = "1.4.0" anyhow = "1.0" -dirs = "3.0.1" -semver = "0.11.0" +dirs = "4.0.0" +semver = "1.0.14" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -serde_yaml = "0.8.13" +serde_yaml = "~0.8.26" reqwest = { version = "0.11", features = ["json"] } tokio = { version = "1.16", features = ["full"] } # reqwest 0.10, hubcaps 0.6 does not support tokio 0.3/1.0 yet, ref: https://github.com/seanmonstar/reqwest/issues/1060 #hubcaps = { git = "https://github.com/softprops/hubcaps", rev = "324d02fbb371782c82e9a13b839b86894475c91f" } # wait for hubcaps formal release > 0.6 @@ -38,11 +38,11 @@ derive_more = "=0.99.17" # remove = until https://github.com/OSSystems/compress- regex = "1.4.1" futures = "0.3" async-trait = "0.1.41" -git2 = "0.13" +git2 = "0.15.0" chrono = "0.4" symlink = "0.1.0" is_executable = "1.0.1" -urlencoding = "1" +urlencoding = "2.1.2" url = "2" Inflector = "0.11.4" fs_extra = "1.1" diff --git a/crates/app/src/cmd/reset.rs b/crates/app/src/cmd/reset.rs index ffe8a79..ff73660 100644 --- a/crates/app/src/cmd/reset.rs +++ b/crates/app/src/cmd/reset.rs @@ -9,7 +9,7 @@ use huber_common::result::Result; use huber_procmacro::process_lock; use crate::cmd::{CommandAsyncTrait, CommandTrait}; -use crate::service::update::{UpdateService, UpdateTrait}; +use crate::service::update::{HuberUpdateService, UpdateTrait}; pub(crate) const CMD_NAME: &str = "reset"; @@ -48,7 +48,7 @@ impl CommandAsyncTrait for ResetCmd { ) -> Result<()> { process_lock!(); - let update_service = container.get::().unwrap(); + let update_service = container.get::().unwrap(); progress( "Resetting huber by removing created caches, downloaded files and installed packages", diff --git a/crates/app/src/cmd/self_update.rs b/crates/app/src/cmd/self_update.rs index 8d11c09..d8e2652 100644 --- a/crates/app/src/cmd/self_update.rs +++ b/crates/app/src/cmd/self_update.rs @@ -10,7 +10,7 @@ use huber_procmacro::process_lock; use crate::cmd::{CommandAsyncTrait, CommandTrait}; use crate::service::cache::{CacheAsyncTrait, CacheService}; -use crate::service::update::{UpdateAsyncTrait, UpdateService}; +use crate::service::update::{HuberUpdateService, UpdateAsyncTrait}; pub(crate) const CMD_NAME: &str = "self-update"; @@ -48,7 +48,7 @@ impl CommandAsyncTrait for SelfUpdateCmd { let cache_service = container.get::().unwrap(); let _ = cache_service.update_repositories().await?; - let update_service = container.get::().unwrap(); + let update_service = container.get::().unwrap(); let r = update_service.has_update().await?; if r.0 { diff --git a/crates/app/src/main.rs b/crates/app/src/main.rs index bc6ecfe..cf8bc3a 100644 --- a/crates/app/src/main.rs +++ b/crates/app/src/main.rs @@ -41,7 +41,7 @@ use crate::service::config::ConfigService; use crate::service::package::PackageService; use crate::service::release::ReleaseService; use crate::service::repo::RepoService; -use crate::service::update::UpdateService; +use crate::service::update::HuberUpdateService; mod cmd; mod component; @@ -117,7 +117,7 @@ fn init_services(mut container: DIContainer) -> Arc { create_dep!(PackageService::new(), container); create_dep!(ReleaseService::new(), container); create_dep!(CacheService::new(), container); - create_dep!(UpdateService::new(), container); + create_dep!(HuberUpdateService::new(), container); create_dep!(RepoService::new(), container); create_dep!(ConfigService::new(), container); @@ -127,7 +127,7 @@ fn init_services(mut container: DIContainer) -> Arc { inject_dep!(PackageService, container.clone()); inject_dep!(ReleaseService, container.clone()); inject_dep!(CacheService, container.clone()); - inject_dep!(UpdateService, container.clone()); + inject_dep!(HuberUpdateService, container.clone()); inject_dep!(RepoService, container.clone()); inject_dep!(ConfigService, container.clone()); diff --git a/crates/app/src/service/release.rs b/crates/app/src/service/release.rs index 57a7a03..d0eac11 100644 --- a/crates/app/src/service/release.rs +++ b/crates/app/src/service/release.rs @@ -1,4 +1,3 @@ -use array_tool::vec::Uniq; use std::collections::HashMap; use std::fs::{read_dir, read_link, remove_dir_all, remove_file, File}; use std::io::Write; @@ -468,44 +467,53 @@ impl ReleaseAsyncTrait for ReleaseService { }) .collect(); - let mut download_urls: Vec = vec![]; - let mut asset_urls: Vec = asset_names + let mut asset_download_urls: Vec = vec![]; + let mut ext_asset_urls: Vec = asset_names // external assets not on github .iter() .filter(|it| Url::parse(it).is_ok() && it.starts_with("https")) .map(|it| it.clone()) .collect(); - download_urls.append(&mut asset_urls); + asset_download_urls.append(&mut ext_asset_urls); - if !asset_urls.is_empty() { - asset_names = asset_names + if !ext_asset_urls.is_empty() { + asset_names = asset_names // assets on github .into_iter() - .filter(|it| !asset_urls.contains(it)) + .filter(|it| !ext_asset_urls.contains(it)) .collect(); } // prepare download urls - for a in package_github.assets.iter() { - let decoded_download_url = decode(&a.browser_download_url)?; + for asset in package_github.assets.iter() { + let asset_download_url = decode(&asset.browser_download_url)?; - if !asset_names.contains(&a.name) + if !asset_names.contains(&asset.name) // assets not mentioned in assert names, just ignored && !asset_names - .iter() - .any(|it| decoded_download_url.ends_with(it)) + .iter() + .any(|it| asset_download_url.ends_with(it)) { + debug!( + "Ignored {}, not mentioned or not right arch type defined in the package artifact config", + asset.name + ); continue; } - download_urls.push(decoded_download_url); + asset_download_urls.push(asset_download_url.to_string()); } - if download_urls.len() < asset_names.len() { - return Err(anyhow!("{:?} not found", asset_names.uniq(download_urls))); + if asset_download_urls.is_empty() { + return Err(anyhow!( + "No available artifacts for {} to download, \ + so probably the download path in package artifact config outdated. \ + Please report issue to https://github.com/innobead/huber", + package.name + )); } // download let mut tasks = vec![]; - for download_url in download_urls { + for download_url in asset_download_urls { // download info!("Downloading {}", &download_url); diff --git a/crates/app/src/service/update.rs b/crates/app/src/service/update.rs index 610ac0a..711f3d0 100644 --- a/crates/app/src/service/update.rs +++ b/crates/app/src/service/update.rs @@ -24,29 +24,29 @@ pub(crate) trait UpdateAsyncTrait { } #[derive(Debug)] -pub(crate) struct UpdateService { +pub(crate) struct HuberUpdateService { pub(crate) container: Option>, } -unsafe impl Send for UpdateService {} +unsafe impl Send for HuberUpdateService {} -unsafe impl Sync for UpdateService {} +unsafe impl Sync for HuberUpdateService {} -impl ServiceTrait for UpdateService {} +impl ServiceTrait for HuberUpdateService {} -impl DependencyInjectTrait for UpdateService { +impl DependencyInjectTrait for HuberUpdateService { fn inject(&mut self, container: Arc) { self.container = Some(container); } } -impl UpdateService { +impl HuberUpdateService { pub(crate) fn new() -> Self { Self { container: None } } } -impl UpdateTrait for UpdateService { +impl UpdateTrait for HuberUpdateService { fn reset(&self) -> Result<()> { let config = self.container.get::().unwrap(); @@ -87,20 +87,29 @@ impl UpdateTrait for UpdateService { } #[async_trait] -impl UpdateAsyncTrait for UpdateService { +impl UpdateAsyncTrait for HuberUpdateService { async fn has_update(&self) -> Result<(bool, String)> { let pkg_service = self.container.get::().unwrap(); let release_service = self.container.get::().unwrap(); - let current_version = env!("HUBER_VERSION").trim_start_matches("v"); + let current_version = + Version::parse(env!("HUBER_VERSION").trim_start_matches("v")).unwrap(); let pkg = pkg_service.get("huber")?; match release_service.get_latest(&pkg).await { Err(e) => Err(anyhow!("No update available: {:?}", e)), - Ok(r) => Ok(( - Version::parse(r.version.trim_start_matches("v")) > Version::parse(current_version), - r.version, - )), + Ok(r) => { + let result = Version::parse(r.version.trim_start_matches("v")) + .map(|ver| ver > current_version); + + match result { + Ok(update_needed) => Ok((update_needed, r.version)), + Err(e) => Err(anyhow!( + "A update available, but failed to continue: {:?}", + e + )), + } + } } } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index ea83dc1..bccda93 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -17,7 +17,7 @@ anyhow = "1.0" dirs = "3.0.1" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["preserve_order"] } -serde_yaml = "0.8.13" +serde_yaml = "~0.8.26" #hubcaps = { git = "https://github.com/softprops/hubcaps", rev = "324d02fbb371782c82e9a13b839b86894475c91f" } # wait for hubcaps formal release > 0.6 hubcaps-ex = "0.6.2" prettytable-rs = "0.9" diff --git a/crates/common/src/model/package.rs b/crates/common/src/model/package.rs index d2675fc..2d90188 100644 --- a/crates/common/src/model/package.rs +++ b/crates/common/src/model/package.rs @@ -18,8 +18,10 @@ use crate::str::VersionCompareTrait; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Package { pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub version: Option, + pub description: Option, pub source: PackageSource, pub targets: Vec, @@ -70,19 +72,27 @@ pub enum PackageTargetType { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PackageManagement { + // {version}, {os} can be used in each. Also, an external URL is acceptable pub artifact_templates: Vec, + #[serde(skip_serializing_if = "Option::is_none")] pub executable_templates: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub executable_mappings: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub install_commands: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub uninstall_commands: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub upgrade_commands: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub tag_version_regex_template: Option, + // only keep the {version} part #[serde(skip_serializing_if = "Option::is_none")] pub scan_dirs: Option>, @@ -100,9 +110,11 @@ pub struct GithubPackage { pub tag_name: String, pub target_commitish: String, pub name: String, + #[serde(skip_deserializing)] #[serde(skip_serializing)] pub body: String, + pub draft: bool, pub prerelease: bool, pub created_at: String, diff --git a/crates/generator/Cargo.toml b/crates/generator/Cargo.toml index 3094575..d1a8b22 100644 --- a/crates/generator/Cargo.toml +++ b/crates/generator/Cargo.toml @@ -9,7 +9,7 @@ build = "src/build.rs" [build-dependencies] huber-common = { path = "../common" } serde = { version = "1.0", features = ["derive"] } -serde_yaml = "0.8.13" +serde_yaml = "~0.8.26" tokio = { version = "1.16", features = ["full"] } # reqwest 0.10, hubcaps 0.6 does not support tokio 0.3/1.0 yet, ref: https://github.com/seanmonstar/reqwest/issues/1060 hubcaps-ex = "0.6.2" maplit = "1.0" diff --git a/crates/generator/src/build.rs b/crates/generator/src/build.rs index 37f766d..0341f76 100644 --- a/crates/generator/src/build.rs +++ b/crates/generator/src/build.rs @@ -26,6 +26,10 @@ async fn main() -> Result<()> { .unwrap() .join("generated") .join("packages"); + let force_generated: bool = env::var("FORCE_GENERATE") + .unwrap_or("false".to_string()) + .parse() + .unwrap(); // clean up and prepare // remove_dir_all(generated_dir.clone()).unwrap(); @@ -52,18 +56,20 @@ async fn main() -> Result<()> { source: r.source.to_string(), }); - let gh_pkg_module_rs_file = Path::new(&pkg_dir) - .join("src") - .join("pkg") - .join(format!("{}.rs", r.name)); - let gh_pkg_module_rs_file_changed = Command::new("git") - .args(&["status", "--short", gh_pkg_module_rs_file.to_str().unwrap()]) - .output() - .map(|output| !output.stdout.is_empty()) - .unwrap(); + if !force_generated { + let gh_pkg_module_rs_file = Path::new(&pkg_dir) + .join("src") + .join("pkg") + .join(format!("{}.rs", r.name)); + let gh_pkg_module_rs_file_changed = Command::new("git") + .args(&["status", "--short", gh_pkg_module_rs_file.to_str().unwrap()]) + .output() + .map(|output| !output.stdout.is_empty()) + .unwrap(); - if !gh_pkg_module_rs_file_changed { - continue; + if !gh_pkg_module_rs_file_changed { + continue; + } } update_description(&mut r).await?; @@ -85,7 +91,7 @@ async fn main() -> Result<()> { pkg_indexes.sort_by(|x, y| x.name.partial_cmp(&y.name).unwrap()); index_file - .write_all(&serde_yaml::to_vec(&pkg_indexes).unwrap()) + .write_all(serde_yaml::to_string(&pkg_indexes).unwrap().as_bytes()) .await?; Ok(()) diff --git a/doc/packages.md b/doc/packages.md index 5fcae25..e475606 100644 --- a/doc/packages.md +++ b/doc/packages.md @@ -109,7 +109,7 @@ nerdctl contaiNERD CTL - Docker-compatible CLI for containerd, with support for Compose, Rootless, eStargz, ... https://github.com/containerd/nerdctl node Node.js JavaScript runtime :sparkles::turtle::rocket::sparkles: https://github.com/nodejs/node nomad Nomad is an easy-to-use, flexible, and performant workload orchestrator that can deploy a mix of mic... https://github.com/hashicorp/nomad - norouter NoRouter: IP-over-Stdio. The easiest multi-host & multi-cloud networking ever. No root privilege is ... https://github.com/norouter/norouter + norouter NoRouter: IP-over-Stdio. The easiest multi-host {value} multi-cloud networking ever. No root privilege is ... https://github.com/norouter/norouter nushell A new type of shell https://github.com/nushell/nushell octant Highly extensible platform for developers to better understand the complexity of Kubernetes clusters... https://github.com/vmware-tanzu/octant okteto Develop your applications directly in your Kubernetes Cluster https://github.com/okteto/okteto @@ -135,7 +135,7 @@ rustwasmc Tool for building Rust functions for Node.js. Combine the performance of Rust, safety and portabilit... https://github.com/second-state/rustwasmc sad CLI search and replace | Space Age seD https://github.com/ms-jpq/sad saml2aws CLI tool which enables you to login and retrieve AWS temporary credentials using a SAML IDP https://github.com/Versent/saml2aws - sd Intuitive find & replace CLI (sed alternative) https://github.com/chmln/sd + sd Intuitive find {value} replace CLI (sed alternative) https://github.com/chmln/sd shadowsocks A Rust port of shadowsocks https://github.com/shadowsocks/shadowsocks-rust skaffold Easy and Repeatable Kubernetes Development https://github.com/GoogleContainerTools/skaffold skim Fuzzy Finder in rust! https://github.com/lotabout/skim