diff --git a/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs b/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs index 62e108f..bd80d29 100644 --- a/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs +++ b/crates/pixi-build/src/bin/pixi-build-cmake/cmake.rs @@ -1,6 +1,6 @@ use std::{ borrow::Cow, - collections::BTreeMap, + collections::{BTreeMap, HashMap}, path::{Path, PathBuf}, str::FromStr, sync::Arc, @@ -17,7 +17,9 @@ use pixi_build_backend::{ }; use pixi_build_types::{ procedures::{ - conda_build::{CondaBuildParams, CondaBuildResult, CondaBuiltPackage}, + conda_build::{ + CondaBuildParams, CondaBuildResult, CondaBuiltPackage, CondaOutputIdentifier, + }, conda_metadata::{CondaMetadataParams, CondaMetadataResult}, initialize::{InitializeParams, InitializeResult}, negotiate_capabilities::{NegotiateCapabilitiesParams, NegotiateCapabilitiesResult}, @@ -214,7 +216,8 @@ impl CMakeBuildBackend { vec!["cxx".to_string()] } - /// Constructs a [`Recipe`] from the current manifest. + /// Constructs a [`Recipe`] from the current manifest. The constructed + /// recipe will invoke CMake to build and install the package. fn recipe( &self, host_platform: Platform, @@ -290,32 +293,16 @@ impl CMakeBuildBackend { } /// Returns the build configuration for a recipe - pub async fn build_configuration( + pub fn build_configuration( &self, recipe: &Recipe, channels: Vec, build_platform: Option, host_platform: Option, - work_directory: &Path, variant: BTreeMap, + directories: Directories, ) -> miette::Result { // Parse the package name from the manifest - let name = self.package_manifest.package.name.clone(); - let name = PackageName::from_str(&name).into_diagnostic()?; - // TODO: Setup defaults - std::fs::create_dir_all(work_directory) - .into_diagnostic() - .context("failed to create output directory")?; - let directories = Directories::setup( - name.as_normalized(), - &self.manifest_path, - work_directory, - true, - &Utc::now(), - ) - .into_diagnostic() - .context("failed to setup build directories")?; - let build_platform = build_platform.map(|p| PlatformWithVirtualPackages { platform: p.platform, virtual_packages: p.virtual_packages.unwrap_or_default(), @@ -362,6 +349,44 @@ impl CMakeBuildBackend { sandbox_config: None, }) } + + /// Determine the all the variants that can be built for this package. + /// + /// The variants are computed based on the dependencies of the package and + /// the input variants. Each package that has a `*` as its version we + /// consider as a potential variant. If an input variant configuration for + /// it exists we add it. + pub fn compute_variants( + &self, + input_variant_configuration: Option>>, + host_platform: Platform, + ) -> miette::Result>> { + // Create a variant config from the variant configuration in the parameters. + let variant_config = VariantConfig { + variants: input_variant_configuration + .unwrap_or_default() + .into_iter() + .map(|(key, values)| (key.into(), values)) + .collect(), + pin_run_as_build: None, + zip_keys: None, + }; + + // Determine the variant keys that are used in the recipe. + let used_variants = self + .package_manifest + .targets + .resolve(Some(host_platform)) + .flat_map(|dep| dep.dependencies.values().flatten()) + .filter(|(_, spec)| can_be_used_as_variant(spec)) + .map(|(name, _)| name.into()) + .collect(); + + // Determine the combinations of the used variants. + variant_config + .combinations(&used_variants, None) + .into_diagnostic() + } } fn input_globs() -> Vec { @@ -406,49 +431,38 @@ impl Protocol for CMakeBuildBackend { .finish(), ); - // Create a variant config from the variant configuration in the parameters. - let variant_config = VariantConfig { - variants: params - .variant_configuration - .unwrap_or_default() - .into_iter() - .map(|(key, values)| (key.into(), values)) - .collect(), - pin_run_as_build: None, - zip_keys: None, - }; + let package_name = PackageName::from_str(&self.package_manifest.package.name) + .into_diagnostic() + .context("`{name}` is not a valid package name")?; - // Determine the variant keys that are used in the recipe. - let used_variants = self - .package_manifest - .targets - .resolve(Some(host_platform)) - .flat_map(|dep| dep.dependencies.values().flatten()) - .filter(|(_, spec)| can_be_used_as_variant(spec)) - .map(|(name, _)| name.into()) - .collect(); + let directories = Directories::setup( + package_name.as_normalized(), + &self.manifest_path, + ¶ms.work_directory, + true, + &Utc::now(), + ) + .into_diagnostic() + .context("failed to setup build directories")?; - // Determine the combinations of the used variants. - let combinations = variant_config - .combinations(&used_variants, None) - .into_diagnostic()?; + // Create a variant config from the variant configuration in the parameters. + let variant_combinations = + self.compute_variants(params.variant_configuration, host_platform)?; // Construct the different outputs let mut packages = Vec::new(); - for variant in combinations { + for variant in variant_combinations { // TODO: Determine how and if we can determine this from the manifest. let recipe = self.recipe(host_platform, &channel_config, &variant)?; let output = Output { - build_configuration: self - .build_configuration( - &recipe, - channels.clone(), - params.build_platform.clone(), - params.host_platform.clone(), - ¶ms.work_directory, - variant, - ) - .await?, + build_configuration: self.build_configuration( + &recipe, + channels.clone(), + params.build_platform.clone(), + params.host_platform.clone(), + variant, + directories.clone(), + )?, recipe, finalized_dependencies: None, finalized_cache_dependencies: None, @@ -529,74 +543,135 @@ impl Protocol for CMakeBuildBackend { .map(|p| p.platform) .unwrap_or_else(Platform::current); - let variant = BTreeMap::new(); - - let recipe = self.recipe(host_platform, &channel_config, &variant)?; - let output = Output { - build_configuration: self - .build_configuration( - &recipe, - channels, - params.host_platform.clone(), - Some(PlatformAndVirtualPackages { - platform: host_platform, - virtual_packages: params.build_platform_virtual_packages, - }), - ¶ms.work_directory, - variant, - ) - .await?, - recipe, - finalized_dependencies: None, - finalized_cache_dependencies: None, - finalized_cache_sources: None, - finalized_sources: None, - build_summary: Arc::default(), - system_tools: Default::default(), - extra_meta: None, - }; - let tool_config = Configuration::builder() - .with_opt_cache_dir(self.cache_dir.clone()) - .with_logging_output_handler(self.logging_output_handler.clone()) - .with_channel_config(channel_config.clone()) - .with_testing(false) - .with_keep_build(true) - .finish(); + let package_name = PackageName::from_str(&self.package_manifest.package.name) + .into_diagnostic() + .context("`{name}` is not a valid package name")?; - let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; - let mut output_with_build_string = output.clone(); + let directories = Directories::setup( + package_name.as_normalized(), + &self.manifest_path, + ¶ms.work_directory, + true, + &Utc::now(), + ) + .into_diagnostic() + .context("failed to setup build directories")?; - let selector_config = output.build_configuration.selector_config(); + // Recompute all the variant combinations + let variant_combinations = + self.compute_variants(params.variant_configuration, host_platform)?; - let jinja = Jinja::new(selector_config.clone()).with_context(&output.recipe.context); + // Compute outputs for each variant + let mut outputs = Vec::with_capacity(variant_combinations.len()); + for variant in variant_combinations { + let recipe = self.recipe(host_platform, &channel_config, &variant)?; + let build_configuration = self.build_configuration( + &recipe, + channels.clone(), + params.host_platform.clone(), + Some(PlatformAndVirtualPackages { + platform: host_platform, + virtual_packages: params.build_platform_virtual_packages.clone(), + }), + variant, + directories.clone(), + )?; + + let mut output = Output { + build_configuration, + recipe, + finalized_dependencies: None, + finalized_cache_dependencies: None, + finalized_cache_sources: None, + finalized_sources: None, + build_summary: Arc::default(), + system_tools: Default::default(), + extra_meta: None, + }; - let hash = HashInfo::from_variant(output.variant(), output.recipe.build().noarch()); - let build_string = - output + // Resolve the build string + let selector_config = output.build_configuration.selector_config(); + let jinja = Jinja::new(selector_config.clone()).with_context(&output.recipe.context); + let hash = HashInfo::from_variant(output.variant(), output.recipe.build().noarch()); + let build_string = output .recipe .build() .string() - .resolve(&hash, output.recipe.build().number(), &jinja); - output_with_build_string.recipe.build.string = - BuildString::Resolved(build_string.to_string()); - - let tool_config = Arc::new(tool_config); - let (output, package) = temp_recipe - .within_context_async(move || async move { - run_build(output_with_build_string, &tool_config).await - }) - .await?; - - Ok(CondaBuildResult { - packages: vec![CondaBuiltPackage { + .resolve(&hash, output.recipe.build().number(), &jinja) + .into_owned(); + output.recipe.build.string = BuildString::Resolved(build_string); + + outputs.push(output); + } + + // Setup tool configuration + let tool_config = Arc::new( + Configuration::builder() + .with_opt_cache_dir(self.cache_dir.clone()) + .with_logging_output_handler(self.logging_output_handler.clone()) + .with_channel_config(channel_config.clone()) + .with_testing(false) + .with_keep_build(true) + .finish(), + ); + + // Determine the outputs to build + let selected_outputs = if let Some(output_identifiers) = params.outputs { + output_identifiers + .into_iter() + .filter_map(|iden| { + let pos = outputs.iter().position(|output| { + let CondaOutputIdentifier { + name, + version, + build, + subdir, + } = &iden; + name.as_ref() + .map_or(true, |n| output.name().as_normalized() == n) + && version + .as_ref() + .map_or(true, |v| output.version().to_string() == *v) + && build + .as_ref() + .map_or(true, |b| output.build_string() == b.as_str()) + && subdir + .as_ref() + .map_or(true, |s| output.target_platform().as_str() == s) + })?; + Some(outputs.remove(pos)) + }) + .collect() + } else { + outputs + }; + + let mut packages = Vec::with_capacity(selected_outputs.len()); + for output in selected_outputs { + let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; + let build_string = output + .recipe + .build + .string + .as_resolved() + .expect("build string must have already been resolved") + .to_string(); + let tool_config = tool_config.clone(); + let (output, package) = temp_recipe + .within_context_async(move || async move { run_build(output, &tool_config).await }) + .await?; + let built_package = CondaBuiltPackage { output_file: package, input_globs: input_globs(), name: output.name().as_normalized().to_string(), version: output.version().to_string(), build: build_string.to_string(), subdir: output.target_platform().to_string(), - }], - }) + }; + packages.push(built_package); + } + + Ok(CondaBuildResult { packages }) } } diff --git a/crates/pixi-build/src/bin/pixi-build-python/python.rs b/crates/pixi-build/src/bin/pixi-build-python/python.rs index f9a861a..69a9584 100644 --- a/crates/pixi-build/src/bin/pixi-build-python/python.rs +++ b/crates/pixi-build/src/bin/pixi-build-python/python.rs @@ -1,6 +1,6 @@ use std::{ borrow::Cow, - collections::BTreeMap, + collections::{BTreeMap, HashMap}, ffi::OsStr, path::{Path, PathBuf}, str::FromStr, @@ -18,7 +18,9 @@ use pixi_build_backend::{ }; use pixi_build_types::{ procedures::{ - conda_build::{CondaBuildParams, CondaBuildResult, CondaBuiltPackage}, + conda_build::{ + CondaBuildParams, CondaBuildResult, CondaBuiltPackage, CondaOutputIdentifier, + }, conda_metadata::{CondaMetadataParams, CondaMetadataResult}, initialize::{InitializeParams, InitializeResult}, negotiate_capabilities::{NegotiateCapabilitiesParams, NegotiateCapabilitiesResult}, @@ -210,7 +212,33 @@ impl PythonBuildBackend { Ok((requirements, installer)) } - /// Constructs a [`Recipe`] from the current manifest. + /// Read the entry points from the pyproject.toml and return them as a list. + /// + /// If the manifest is not a pyproject.toml file no entry-points are added. + fn entry_points(&self) -> Vec { + let scripts = self + .pyproject_manifest + .as_ref() + .and_then(|p| p.project.as_ref()) + .and_then(|p| p.scripts.as_ref()); + + scripts + .into_iter() + .flatten() + .flat_map(|(name, entry_point)| { + EntryPoint::from_str(&format!("{name} = {entry_point}")) + }) + .collect() + } + + /// Constructs a [`Recipe`] that will build the python package into a conda + /// package. + /// + /// If the package is editable, the recipe will not include the source but + /// only references to the original source files. + /// + /// Script entry points are read from the pyproject and added as entry + /// points in the conda package. fn recipe( &self, host_platform: Platform, @@ -218,40 +246,28 @@ impl PythonBuildBackend { editable: bool, variant: &BTreeMap, ) -> miette::Result { - // Parse the package name from the manifest - let package = &self.package_manifest.package; - - let name = PackageName::from_str(&package.name).into_diagnostic()?; + // Parse the package name and version from the manifest + let name = PackageName::from_str(&self.package_manifest.package.name).into_diagnostic()?; + let version = self.package_manifest.package.version.clone(); + // Determine whether the package should be built as a noarch package or as a + // generic package. let noarch_type = if self.config.noarch() { NoArchType::python() } else { NoArchType::none() }; - // Determine the entry points from the pyproject.toml - // which would be passed into recipe - let python = if let Some(pyproject_manifest) = &self.pyproject_manifest { - let mut python = Python::default(); - let scripts = pyproject_manifest - .project - .as_ref() - .and_then(|p| p.scripts.as_ref()); - if let Some(scripts) = scripts { - python.entry_points = scripts - .into_iter() - .flat_map(|(name, entry_point)| { - EntryPoint::from_str(&format!("{name} = {entry_point}")) - }) - .collect(); - } - python - } else { - Python::default() + // Construct python specific settings + let python = Python { + entry_points: self.entry_points(), + ..Python::default() }; let (requirements, installer) = self.requirements(host_platform, channel_config, variant)?; + + // Create a build script let build_platform = Platform::current(); let build_number = 0; @@ -270,7 +286,10 @@ impl PythonBuildBackend { } .render(); + // Define the sources of the package. let source = if editable { + // In editable mode we don't include the source in the package, the package will + // refer back to the original source. Vec::new() } else { Vec::from([Source::Path(PathSource { @@ -288,7 +307,7 @@ impl PythonBuildBackend { Ok(Recipe { schema_version: 1, package: Package { - version: package.version.clone().into(), + version: version.into(), name, }, context: Default::default(), @@ -313,7 +332,6 @@ impl PythonBuildBackend { // files: Default::default(), ..Build::default() }, - // TODO read from manifest requirements, tests: vec![], about: Default::default(), @@ -322,31 +340,15 @@ impl PythonBuildBackend { } /// Returns the build configuration for a recipe - pub async fn build_configuration( + pub fn build_configuration( &self, recipe: &Recipe, channels: Vec, build_platform: Option, host_platform: Option, - work_directory: &Path, + variant: BTreeMap, + directories: Directories, ) -> miette::Result { - // Parse the package name from the manifest - let name = self.package_manifest.package.name.clone(); - let name = PackageName::from_str(&name).into_diagnostic()?; - - std::fs::create_dir_all(work_directory) - .into_diagnostic() - .context("failed to create output directory")?; - let directories = Directories::setup( - name.as_normalized(), - &self.manifest_path, - work_directory, - true, - &Utc::now(), - ) - .into_diagnostic() - .context("failed to setup build directories")?; - let build_platform = build_platform.map(|p| PlatformWithVirtualPackages { platform: p.platform, virtual_packages: p.virtual_packages.unwrap_or_default(), @@ -372,7 +374,6 @@ impl PythonBuildBackend { } }; - let variant = BTreeMap::new(); let channels = channels.into_iter().map(Into::into).collect(); Ok(BuildConfiguration { @@ -397,6 +398,44 @@ impl PythonBuildBackend { sandbox_config: None, }) } + + /// Determine the all the variants that can be built for this package. + /// + /// The variants are computed based on the dependencies of the package and + /// the input variants. Each package that has a `*` as its version we + /// consider as a potential variant. If an input variant configuration for + /// it exists we add it. + pub fn compute_variants( + &self, + input_variant_configuration: Option>>, + host_platform: Platform, + ) -> miette::Result>> { + // Create a variant config from the variant configuration in the parameters. + let variant_config = VariantConfig { + variants: input_variant_configuration + .unwrap_or_default() + .into_iter() + .map(|(key, values)| (key.into(), values)) + .collect(), + pin_run_as_build: None, + zip_keys: None, + }; + + // Determine the variant keys that are used in the recipe. + let used_variants = self + .package_manifest + .targets + .resolve(Some(host_platform)) + .flat_map(|dep| dep.dependencies.values().flatten()) + .filter(|(_, spec)| can_be_used_as_variant(spec)) + .map(|(name, _)| name.into()) + .collect(); + + // Determine the combinations of the used variants. + variant_config + .combinations(&used_variants, None) + .into_diagnostic() + } } /// Determines the build input globs for given python package @@ -459,6 +498,20 @@ impl Protocol for PythonBuildBackend { .map(|p| p.platform) .unwrap_or(Platform::current()); + let package_name = PackageName::from_str(&self.package_manifest.package.name) + .into_diagnostic() + .context("`{name}` is not a valid package name")?; + + let directories = Directories::setup( + package_name.as_normalized(), + &self.manifest_path, + ¶ms.work_directory, + true, + &Utc::now(), + ) + .into_diagnostic() + .context("failed to setup build directories")?; + // Build the tool configuration let tool_config = Arc::new( Configuration::builder() @@ -503,15 +556,14 @@ impl Protocol for PythonBuildBackend { // TODO: Determine how and if we can determine this from the manifest. let recipe = self.recipe(host_platform, &channel_config, false, &variant)?; let output = Output { - build_configuration: self - .build_configuration( - &recipe, - channels.clone(), - params.build_platform.clone(), - params.host_platform.clone(), - ¶ms.work_directory.clone(), - ) - .await?, + build_configuration: self.build_configuration( + &recipe, + channels.clone(), + params.build_platform.clone(), + params.host_platform.clone(), + variant, + directories.clone(), + )?, recipe, finalized_dependencies: None, finalized_cache_dependencies: None, @@ -586,70 +638,141 @@ impl Protocol for PythonBuildBackend { root_dir: self.manifest_root.to_path_buf(), }; let channels = params.channel_base_urls.unwrap_or_default(); - let host_platform = params .host_platform .as_ref() .map(|p| p.platform) .unwrap_or_else(Platform::current); - let variant = BTreeMap::new(); - - let recipe = self.recipe(host_platform, &channel_config, params.editable, &variant)?; - let output = Output { - build_configuration: self - .build_configuration(&recipe, channels, None, None, ¶ms.work_directory) - .await?, - recipe, - finalized_dependencies: None, - finalized_cache_dependencies: None, - finalized_cache_sources: None, - finalized_sources: None, - build_summary: Arc::default(), - system_tools: Default::default(), - extra_meta: None, - }; - let tool_config = Configuration::builder() - .with_opt_cache_dir(self.cache_dir.clone()) - .with_logging_output_handler(self.logging_output_handler.clone()) - .with_channel_config(channel_config.clone()) - .with_testing(false) - .finish(); - - let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; - - let mut output_with_build_string = output.clone(); + let package_name = PackageName::from_str(&self.package_manifest.package.name) + .into_diagnostic() + .context("`{name}` is not a valid package name")?; - let selector_config = output.build_configuration.selector_config(); + let directories = Directories::setup( + package_name.as_normalized(), + &self.manifest_path, + ¶ms.work_directory, + true, + &Utc::now(), + ) + .into_diagnostic() + .context("failed to setup build directories")?; - let jinja = Jinja::new(selector_config.clone()).with_context(&output.recipe.context); + // Recompute all the variant combinations + let variant_combinations = + self.compute_variants(params.variant_configuration, host_platform)?; + + // Compute outputs for each variant + let mut outputs = Vec::with_capacity(variant_combinations.len()); + for variant in variant_combinations { + let recipe = self.recipe(host_platform, &channel_config, params.editable, &variant)?; + let build_configuration = self.build_configuration( + &recipe, + channels.clone(), + params.host_platform.clone(), + Some(PlatformAndVirtualPackages { + platform: host_platform, + virtual_packages: params.build_platform_virtual_packages.clone(), + }), + variant, + directories.clone(), + )?; + + let mut output = Output { + build_configuration, + recipe, + finalized_dependencies: None, + finalized_cache_dependencies: None, + finalized_cache_sources: None, + finalized_sources: None, + build_summary: Arc::default(), + system_tools: Default::default(), + extra_meta: None, + }; - let hash = HashInfo::from_variant(output.variant(), output.recipe.build().noarch()); - let build_string = - output + // Resolve the build string + let selector_config = output.build_configuration.selector_config(); + let jinja = Jinja::new(selector_config.clone()).with_context(&output.recipe.context); + let hash = HashInfo::from_variant(output.variant(), output.recipe.build().noarch()); + let build_string = output .recipe .build() .string() - .resolve(&hash, output.recipe.build().number(), &jinja); - output_with_build_string.recipe.build.string = - BuildString::Resolved(build_string.to_string()); + .resolve(&hash, output.recipe.build().number(), &jinja) + .into_owned(); + output.recipe.build.string = BuildString::Resolved(build_string); - let (output, package) = temp_recipe - .within_context_async(move || async move { - run_build(output_with_build_string, &tool_config).await - }) - .await?; + outputs.push(output); + } + + // Setup tool configuration + let tool_config = Arc::new( + Configuration::builder() + .with_opt_cache_dir(self.cache_dir.clone()) + .with_logging_output_handler(self.logging_output_handler.clone()) + .with_channel_config(channel_config.clone()) + .with_testing(false) + .with_keep_build(true) + .finish(), + ); - Ok(CondaBuildResult { - packages: vec![CondaBuiltPackage { + // Determine the outputs to build + let selected_outputs = if let Some(output_identifiers) = params.outputs { + output_identifiers + .into_iter() + .filter_map(|iden| { + let pos = outputs.iter().position(|output| { + let CondaOutputIdentifier { + name, + version, + build, + subdir, + } = &iden; + name.as_ref() + .map_or(true, |n| output.name().as_normalized() == n) + && version + .as_ref() + .map_or(true, |v| output.version().to_string() == *v) + && build + .as_ref() + .map_or(true, |b| output.build_string() == b.as_str()) + && subdir + .as_ref() + .map_or(true, |s| output.target_platform().as_str() == s) + })?; + Some(outputs.remove(pos)) + }) + .collect() + } else { + outputs + }; + + let mut packages = Vec::with_capacity(selected_outputs.len()); + for output in selected_outputs { + let temp_recipe = TemporaryRenderedRecipe::from_output(&output)?; + let build_string = output + .recipe + .build + .string + .as_resolved() + .expect("build string must have already been resolved") + .to_string(); + let tool_config = tool_config.clone(); + let (output, package) = temp_recipe + .within_context_async(move || async move { run_build(output, &tool_config).await }) + .await?; + let built_package = CondaBuiltPackage { output_file: package, input_globs: input_globs(), name: output.name().as_normalized().to_string(), version: output.version().to_string(), build: build_string.to_string(), subdir: output.target_platform().to_string(), - }], - }) + }; + packages.push(built_package); + } + + Ok(CondaBuildResult { packages }) } }