From f189bad8f0f8d40f36932bbc526ddc276faf8e13 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Mon, 14 Oct 2024 14:42:39 +0200 Subject: [PATCH 1/2] feat: incremental cmake compilation --- Cargo.lock | 9 +-- Cargo.toml | 18 +++--- .../src/bin/pixi-build-cmake/build_script.j2 | 13 ++-- .../src/bin/pixi-build-cmake/build_script.rs | 1 + .../src/bin/pixi-build-cmake/cmake.rs | 60 ++++++++++--------- .../src/bin/pixi-build-python/python.rs | 52 +++++++++++----- crates/pixi-build/src/cli.rs | 13 +++- 7 files changed, 100 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c76eec..2e418d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1088,7 +1088,6 @@ dependencies = [ [[package]] name = "fancy_display" version = "0.1.0" -source = "git+https://github.com/baszalmstra/pixi?branch=feat/virtual_packages_for_platforms#1d28a0faad2853fbe669598472b4f4f0e91e90a6" dependencies = [ "console", ] @@ -2890,7 +2889,6 @@ dependencies = [ [[package]] name = "pixi_build_types" version = "0.1.0" -source = "git+https://github.com/baszalmstra/pixi?branch=feat/virtual_packages_for_platforms#1d28a0faad2853fbe669598472b4f4f0e91e90a6" dependencies = [ "rattler_conda_types", "serde", @@ -2901,7 +2899,6 @@ dependencies = [ [[package]] name = "pixi_consts" version = "0.1.0" -source = "git+https://github.com/baszalmstra/pixi?branch=feat/virtual_packages_for_platforms#1d28a0faad2853fbe669598472b4f4f0e91e90a6" dependencies = [ "console", "lazy_static", @@ -2912,7 +2909,6 @@ dependencies = [ [[package]] name = "pixi_manifest" version = "0.1.0" -source = "git+https://github.com/baszalmstra/pixi?branch=feat/virtual_packages_for_platforms#1d28a0faad2853fbe669598472b4f4f0e91e90a6" dependencies = [ "console", "dunce", @@ -2944,7 +2940,6 @@ dependencies = [ [[package]] name = "pixi_spec" version = "0.1.0" -source = "git+https://github.com/baszalmstra/pixi?branch=feat/virtual_packages_for_platforms#1d28a0faad2853fbe669598472b4f4f0e91e90a6" dependencies = [ "dirs", "file_url", @@ -3177,8 +3172,8 @@ dependencies = [ [[package]] name = "rattler-build" -version = "0.25.0" -source = "git+https://github.com/baszalmstra/rattler-build?branch=feat/virtual_packages_for_platforms#bbe8f1b9c89ee13427866c7917c8ac3171778927" +version = "0.26.0" +source = "git+https://github.com/prefix-dev/rattler-build?branch=main#3067372ea22be518496f471aa3ff7f99ea2f2159" dependencies = [ "anyhow", "async-once-cell", diff --git a/Cargo.toml b/Cargo.toml index 031399c..a8ca426 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,18 +28,18 @@ jsonrpc-stdio-server = "18.0.0" jsonrpc-http-server = "18.0.0" jsonrpc-core = "18.0.0" -rattler-build = { git = "https://github.com/baszalmstra/rattler-build", branch = "feat/virtual_packages_for_platforms", default-features = false } +rattler-build = { git = "https://github.com/prefix-dev/rattler-build", branch = "main", default-features = false } rattler_conda_types = "0.28.2" rattler_package_streaming = "0.22.10" rattler_virtual_packages = "1.1.7" -#pixi_build_types = { path = "../pixi-build-branch/crates/pixi_build_types" } -#pixi_consts = { path = "../pixi-build-branch/crates/pixi_consts" } -#pixi_manifest = { path = "../pixi-build-branch/crates/pixi_manifest" } -#pixi_spec = { path = "../pixi-build-branch/crates/pixi_spec" } +pixi_build_types = { path = "../pixi-build-branch/crates/pixi_build_types" } +pixi_consts = { path = "../pixi-build-branch/crates/pixi_consts" } +pixi_manifest = { path = "../pixi-build-branch/crates/pixi_manifest" } +pixi_spec = { path = "../pixi-build-branch/crates/pixi_spec" } -pixi_build_types = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } -pixi_consts = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } -pixi_manifest = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } -pixi_spec = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } +#pixi_build_types = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } +#pixi_consts = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } +#pixi_manifest = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } +#pixi_spec = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } diff --git a/crates/pixi-build/src/bin/pixi-build-cmake/build_script.j2 b/crates/pixi-build/src/bin/pixi-build-cmake/build_script.j2 index 970b105..002d960 100644 --- a/crates/pixi-build/src/bin/pixi-build-cmake/build_script.j2 +++ b/crates/pixi-build/src/bin/pixi-build-cmake/build_script.j2 @@ -7,19 +7,22 @@ cmake %CMAKE_ARGS% ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ -DBUILD_SHARED_LIBS=ON ^ - %SRC_DIR% -if errorlevel 1 exit 1 + -B %SRC_DIR%\..\build ^ + -S "{{ source_dir }}" +@if errorlevel 1 exit 1 +cmake --build %SRC_DIR%\..\build --target install +@if errorlevel 1 exit 1 {% else -%} cmake $CMAKE_ARGS \ -GNinja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DBUILD_SHARED_LIBS=ON \ - $SRC_DIR + -B $SRC_DIR/../build \ + -S "{{ source_dir }}" +cmake --build $SRC_DIR/../build --target install {% endif -%} -cmake --build . --target install - {% if build_platform == "windows" -%} if errorlevel 1 exit 1 {% endif %} diff --git a/crates/pixi-build/src/bin/pixi-build-cmake/build_script.rs b/crates/pixi-build/src/bin/pixi-build-cmake/build_script.rs index 0f9c5e5..267f1c5 100644 --- a/crates/pixi-build/src/bin/pixi-build-cmake/build_script.rs +++ b/crates/pixi-build/src/bin/pixi-build-cmake/build_script.rs @@ -4,6 +4,7 @@ use serde::Serialize; #[derive(Serialize)] pub struct BuildScriptContext { pub build_platform: BuildPlatform, + pub source_dir: String, } #[derive(Serialize)] diff --git a/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs b/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs index 88d80d8..8099fbe 100644 --- a/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs +++ b/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs @@ -31,7 +31,7 @@ use rattler_build::{ BuildConfiguration, Directories, Output, PackagingSettings, PlatformWithVirtualPackages, }, recipe::{ - parser::{Build, Dependency, Package, PathSource, Requirements, ScriptContent, Source}, + parser::{Build, Dependency, Package, Requirements, ScriptContent}, Recipe, }, render::resolved_dependencies::DependencyInfo, @@ -43,7 +43,6 @@ use rattler_conda_types::{ use rattler_package_streaming::write::CompressionLevel; use rattler_virtual_packages::VirtualPackageOverrides; use reqwest::Url; -use tempfile::tempdir; use crate::{ build_script::{BuildPlatform, BuildScriptContext}, @@ -102,7 +101,7 @@ impl CMakeBuildBackend { /// recipe. fn requirements( &self, - target_platform: Platform, + host_platform: Platform, channel_config: &ChannelConfig, ) -> miette::Result { let mut requirements = Requirements::default(); @@ -112,17 +111,17 @@ impl CMakeBuildBackend { let run_dependencies = Dependencies::from( default_features .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Run), Some(target_platform))), + .filter_map(|f| f.dependencies(SpecType::Run, Some(host_platform))), ); let mut host_dependencies = Dependencies::from( default_features .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Host), Some(target_platform))), + .filter_map(|f| f.dependencies(SpecType::Host, Some(host_platform))), ); let build_dependencies = Dependencies::from( default_features .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Build), Some(target_platform))), + .filter_map(|f| f.dependencies(SpecType::Build, Some(host_platform))), ); // Ensure build tools are available in the host dependencies section. @@ -167,7 +166,7 @@ impl CMakeBuildBackend { // Add compilers to the dependencies. requirements.build.extend( - self.compiler_packages(target_platform) + self.compiler_packages(host_platform) .into_iter() .map(Dependency::Spec), ); @@ -233,6 +232,7 @@ impl CMakeBuildBackend { } else { BuildPlatform::Unix }, + source_dir: manifest_root.display().to_string(), } .render(); @@ -244,16 +244,18 @@ impl CMakeBuildBackend { name, }, cache: None, - source: vec![Source::Path(PathSource { - // TODO: How can we use a git source? - path: manifest_root.to_path_buf(), - sha256: None, - md5: None, - patches: vec![], - target_directory: None, - file_name: None, - use_gitignore: true, - })], + // source: vec![Source::Path(PathSource { + // // TODO: How can we use a git source? + // path: manifest_root.to_path_buf(), + // sha256: None, + // md5: None, + // patches: vec![], + // target_directory: None, + // file_name: None, + // use_gitignore: true, + // })], + // We hack the source location + source: vec![], build: Build { number: build_number, string: Default::default(), @@ -289,6 +291,7 @@ impl CMakeBuildBackend { channels: Vec, build_platform: Option, host_platform: Option, + work_directory: &Path, ) -> miette::Result { // Parse the package name from the manifest let Some(name) = self.manifest.parsed.project.name.clone() else { @@ -297,17 +300,14 @@ impl CMakeBuildBackend { let name = PackageName::from_str(&name).into_diagnostic()?; // TODO: Setup defaults - let output_dir = tempdir() - .into_diagnostic() - .context("failed to create temporary directory")?; - std::fs::create_dir_all(&output_dir) + std::fs::create_dir_all(work_directory) .into_diagnostic() .context("failed to create output directory")?; let directories = Directories::setup( name.as_normalized(), self.manifest.path.as_path(), - output_dir.path(), - false, + work_directory, + true, &Utc::now(), ) .into_diagnostic() @@ -410,6 +410,7 @@ impl Protocol for CMakeBuildBackend { channels, params.build_platform, params.host_platform, + ¶ms.work_directory, ) .await?, recipe, @@ -425,6 +426,7 @@ impl Protocol for CMakeBuildBackend { .with_logging_output_handler(self.logging_output_handler.clone()) .with_channel_config(channel_config.clone()) .with_testing(false) + .with_keep_build(true) .finish(); let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; @@ -483,16 +485,16 @@ impl Protocol for CMakeBuildBackend { .into_diagnostic() .context("failed to determine channels from the manifest")?, }; - let target_platform = params + let host_platform = params .host_platform .as_ref() .map(|p| p.platform) .unwrap_or_else(Platform::current); - if !self.manifest.supports_target_platform(target_platform) { - miette::bail!("the project does not support the target platform ({target_platform})"); + if !self.manifest.supports_target_platform(host_platform) { + miette::bail!("the project does not support the target platform ({host_platform})"); } - let recipe = self.recipe(target_platform, &channel_config)?; + let recipe = self.recipe(host_platform, &channel_config)?; let output = Output { build_configuration: self .build_configuration( @@ -500,9 +502,10 @@ impl Protocol for CMakeBuildBackend { channels, params.host_platform.clone(), Some(PlatformAndVirtualPackages { - platform: target_platform, + platform: host_platform, virtual_packages: params.build_platform_virtual_packages, }), + ¶ms.work_directory, ) .await?, recipe, @@ -518,6 +521,7 @@ impl Protocol for CMakeBuildBackend { .with_logging_output_handler(self.logging_output_handler.clone()) .with_channel_config(channel_config.clone()) .with_testing(false) + .with_keep_build(true) .finish(); let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; diff --git a/crates/pixi-build/src/bin/pixi-build-python/python.rs b/crates/pixi-build/src/bin/pixi-build-python/python.rs index e468930..2f68de8 100644 --- a/crates/pixi-build/src/bin/pixi-build-python/python.rs +++ b/crates/pixi-build/src/bin/pixi-build-python/python.rs @@ -43,7 +43,6 @@ use rattler_conda_types::{ use rattler_package_streaming::write::CompressionLevel; use rattler_virtual_packages::VirtualPackageOverrides; use reqwest::Url; -use tempfile::tempdir; use crate::build_script::{BuildPlatform, BuildScriptContext, Installer}; @@ -99,6 +98,7 @@ impl PythonBuildBackend { /// recipe. fn requirements( &self, + host_platform: Platform, channel_config: &ChannelConfig, ) -> miette::Result<(Requirements, Installer)> { let mut requirements = Requirements::default(); @@ -108,17 +108,17 @@ impl PythonBuildBackend { let run_dependencies = Dependencies::from( default_features .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Run), None)), + .filter_map(|f| f.dependencies(SpecType::Run, Some(host_platform))), ); let mut host_dependencies = Dependencies::from( default_features .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Host), None)), + .filter_map(|f| f.dependencies(SpecType::Host, Some(host_platform))), ); let build_dependencies = Dependencies::from( default_features .iter() - .filter_map(|f| f.dependencies(Some(SpecType::Build), None)), + .filter_map(|f| f.dependencies(SpecType::Build, Some(host_platform))), ); // Determine the installer to use @@ -175,7 +175,11 @@ impl PythonBuildBackend { } /// Constructs a [`Recipe`] from the current manifest. - fn recipe(&self, channel_config: &ChannelConfig) -> miette::Result { + fn recipe( + &self, + host_platform: Platform, + channel_config: &ChannelConfig, + ) -> miette::Result { let manifest_root = self .manifest .path @@ -193,7 +197,7 @@ impl PythonBuildBackend { let noarch_type = NoArchType::python(); // TODO: Read from config / project. - let (requirements, installer) = self.requirements(channel_config)?; + let (requirements, installer) = self.requirements(host_platform, channel_config)?; let build_platform = Platform::current(); let build_number = 0; @@ -260,6 +264,7 @@ impl PythonBuildBackend { channels: Vec, build_platform: Option, host_platform: Option, + work_directory: &Path, ) -> miette::Result { // Parse the package name from the manifest let Some(name) = self.manifest.parsed.project.name.clone() else { @@ -267,18 +272,14 @@ impl PythonBuildBackend { }; let name = PackageName::from_str(&name).into_diagnostic()?; - // TODO: Setup defaults - let output_dir = tempdir() - .into_diagnostic() - .context("failed to create temporary directory")?; - std::fs::create_dir_all(&output_dir) + std::fs::create_dir_all(work_directory) .into_diagnostic() .context("failed to create output directory")?; let directories = Directories::setup( name.as_normalized(), self.manifest.path.as_path(), - output_dir.path(), - false, + work_directory, + true, &Utc::now(), ) .into_diagnostic() @@ -395,8 +396,17 @@ impl Protocol for PythonBuildBackend { .context("failed to determine channels from the manifest")?, }; + let host_platform = params + .host_platform + .as_ref() + .map(|p| p.platform) + .unwrap_or(Platform::current()); + if !self.manifest.supports_target_platform(host_platform) { + miette::bail!("the project does not support the target platform ({host_platform})"); + } + // TODO: Determine how and if we can determine this from the manifest. - let recipe = self.recipe(&channel_config)?; + let recipe = self.recipe(host_platform, &channel_config)?; let output = Output { build_configuration: self .build_configuration( @@ -404,6 +414,7 @@ impl Protocol for PythonBuildBackend { channels, params.build_platform, params.host_platform, + ¶ms.work_directory, ) .await?, recipe, @@ -478,10 +489,19 @@ impl Protocol for PythonBuildBackend { .context("failed to determine channels from the manifest")?, }; - let recipe = self.recipe(&channel_config)?; + let host_platform = params + .host_platform + .as_ref() + .map(|p| p.platform) + .unwrap_or_else(Platform::current); + if !self.manifest.supports_target_platform(host_platform) { + miette::bail!("the project does not support the target platform ({host_platform})"); + } + + let recipe = self.recipe(host_platform, &channel_config)?; let output = Output { build_configuration: self - .build_configuration(&recipe, channels, None, None) + .build_configuration(&recipe, channels, None, None, ¶ms.work_directory) .await?, recipe, finalized_dependencies: None, diff --git a/crates/pixi-build/src/cli.rs b/crates/pixi-build/src/cli.rs index e8282f6..f06948e 100644 --- a/crates/pixi-build/src/cli.rs +++ b/crates/pixi-build/src/cli.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use clap_verbosity_flag::{InfoLevel, Verbosity}; -use miette::IntoDiagnostic; +use miette::{Context, IntoDiagnostic}; use pixi_build_types::{ procedures::{ conda_build::CondaBuildParams, @@ -14,6 +14,7 @@ use pixi_build_types::{ use rattler_build::console_utils::{get_default_env_filter, LoggingOutputHandler}; use rattler_conda_types::{ChannelConfig, GenericVirtualPackage, Platform}; use rattler_virtual_packages::{VirtualPackage, VirtualPackageOverrides}; +use tempfile::TempDir; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; use crate::{ @@ -116,6 +117,10 @@ async fn get_conda_metadata( .map(GenericVirtualPackage::from) .collect(); + let tempdir = TempDir::new_in(".") + .into_diagnostic() + .context("failed to create a temporary directory in the current directory")?; + protocol .get_conda_metadata(CondaMetadataParams { build_platform: None, @@ -127,6 +132,7 @@ async fn get_conda_metadata( channel_configuration: ChannelConfiguration { base_url: channel_config.channel_alias, }, + work_directory: tempdir.path().to_path_buf(), }) .await } @@ -147,6 +153,10 @@ async fn build(factory: impl ProtocolFactory, manifest_path: &Path) -> miette::R }) .await?; + let work_dir = TempDir::new_in(".") + .into_diagnostic() + .context("failed to create a temporary directory in the current directory")?; + let result = protocol .build_conda(CondaBuildParams { host_platform: None, @@ -156,6 +166,7 @@ async fn build(factory: impl ProtocolFactory, manifest_path: &Path) -> miette::R base_url: channel_config.channel_alias, }, outputs: None, + work_directory: work_dir.path().to_path_buf(), }) .await?; From 1ad8cdf714831f8dc186f35b90769cb15d5a3d33 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Mon, 14 Oct 2024 14:44:02 +0200 Subject: [PATCH 2/2] fix: use correct branch --- Cargo.lock | 5 +++++ Cargo.toml | 18 +++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e418d4..0237e05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1088,6 +1088,7 @@ dependencies = [ [[package]] name = "fancy_display" version = "0.1.0" +source = "git+https://github.com/baszalmstra/pixi?branch=feat/incremental_compilation#f789ae8cb8e2534fa3bed245e0f2af45f71d83af" dependencies = [ "console", ] @@ -2889,6 +2890,7 @@ dependencies = [ [[package]] name = "pixi_build_types" version = "0.1.0" +source = "git+https://github.com/baszalmstra/pixi?branch=feat/incremental_compilation#f789ae8cb8e2534fa3bed245e0f2af45f71d83af" dependencies = [ "rattler_conda_types", "serde", @@ -2899,6 +2901,7 @@ dependencies = [ [[package]] name = "pixi_consts" version = "0.1.0" +source = "git+https://github.com/baszalmstra/pixi?branch=feat/incremental_compilation#f789ae8cb8e2534fa3bed245e0f2af45f71d83af" dependencies = [ "console", "lazy_static", @@ -2909,6 +2912,7 @@ dependencies = [ [[package]] name = "pixi_manifest" version = "0.1.0" +source = "git+https://github.com/baszalmstra/pixi?branch=feat/incremental_compilation#f789ae8cb8e2534fa3bed245e0f2af45f71d83af" dependencies = [ "console", "dunce", @@ -2940,6 +2944,7 @@ dependencies = [ [[package]] name = "pixi_spec" version = "0.1.0" +source = "git+https://github.com/baszalmstra/pixi?branch=feat/incremental_compilation#f789ae8cb8e2534fa3bed245e0f2af45f71d83af" dependencies = [ "dirs", "file_url", diff --git a/Cargo.toml b/Cargo.toml index a8ca426..3ddb755 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,13 +33,13 @@ rattler_conda_types = "0.28.2" rattler_package_streaming = "0.22.10" rattler_virtual_packages = "1.1.7" -pixi_build_types = { path = "../pixi-build-branch/crates/pixi_build_types" } -pixi_consts = { path = "../pixi-build-branch/crates/pixi_consts" } -pixi_manifest = { path = "../pixi-build-branch/crates/pixi_manifest" } -pixi_spec = { path = "../pixi-build-branch/crates/pixi_spec" } - -#pixi_build_types = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } -#pixi_consts = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } -#pixi_manifest = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } -#pixi_spec = { git = "https://github.com/baszalmstra/pixi", branch = "feat/virtual_packages_for_platforms" } +#pixi_build_types = { path = "../pixi-build-branch/crates/pixi_build_types" } +#pixi_consts = { path = "../pixi-build-branch/crates/pixi_consts" } +#pixi_manifest = { path = "../pixi-build-branch/crates/pixi_manifest" } +#pixi_spec = { path = "../pixi-build-branch/crates/pixi_spec" } + +pixi_build_types = { git = "https://github.com/baszalmstra/pixi", branch = "feat/incremental_compilation" } +pixi_consts = { git = "https://github.com/baszalmstra/pixi", branch = "feat/incremental_compilation" } +pixi_manifest = { git = "https://github.com/baszalmstra/pixi", branch = "feat/incremental_compilation" } +pixi_spec = { git = "https://github.com/baszalmstra/pixi", branch = "feat/incremental_compilation" }