diff --git a/examples/pypi-find-links/pixi.lock b/examples/pypi-find-links/pixi.lock index cdcf005ea..266aa0ae2 100644 --- a/examples/pypi-find-links/pixi.lock +++ b/examples/pypi-find-links/pixi.lock @@ -33,8 +33,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/16/92/92a76dc2ff3a12e69ba94e7e05168d37d0345fa08c87e1fe24d0c2a42223/charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl - - pypi: file:///Users/tdejager/development/prefix/pixi/examples/pypi-find-links/links/requests-2.31.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl + - pypi: ./links/requests-2.31.0-py3-none-any.whl osx-64: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.8.30-h8857fd0_0.conda @@ -52,8 +52,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/50/89/354cc56cf4dd2449715bc9a0f54f3aef3dc700d2d62d1fa5bbea53b13426/charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl - - pypi: file:///Users/tdejager/development/prefix/pixi/examples/pypi-find-links/links/requests-2.31.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl + - pypi: ./links/requests-2.31.0-py3-none-any.whl osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.8.30-hf0a4a13_0.conda @@ -71,8 +71,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fa/44/b730e2a2580110ced837ac083d8ad222343c96bb6b66e9e4e706e4d0b6df/charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl - - pypi: file:///Users/tdejager/development/prefix/pixi/examples/pypi-find-links/links/requests-2.31.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl + - pypi: ./links/requests-2.31.0-py3-none-any.whl win-64: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h2466b09_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/ca-certificates-2024.8.30-h56e8100_0.conda @@ -92,8 +92,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/67/7b72b69d25b89c0b3cea583ee372c43aa24df15f0e0f8d3982c57804984b/charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl - - pypi: file:///Users/tdejager/development/prefix/pixi/examples/pypi-find-links/links/requests-2.31.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl + - pypi: ./links/requests-2.31.0-py3-none-any.whl packages: - kind: conda name: _libgcc_mutex @@ -929,7 +929,7 @@ packages: - kind: pypi name: requests version: 2.31.0 - url: file:///Users/tdejager/development/prefix/pixi/examples/pypi-find-links/links/requests-2.31.0-py3-none-any.whl + path: ./links/requests-2.31.0-py3-none-any.whl requires_dist: - charset-normalizer>=2,<4 - idna>=2.5,<4 diff --git a/examples/pypi-find-links/pixi.toml b/examples/pypi-find-links/pixi.toml index 36dcacef2..7d1bd85d6 100644 --- a/examples/pypi-find-links/pixi.toml +++ b/examples/pypi-find-links/pixi.toml @@ -3,6 +3,7 @@ authors = ["Tim de Jager "] channels = ["conda-forge"] name = "pypi-find-links" platforms = ["osx-arm64", "osx-64", "linux-64", "win-64"] + [project.pypi-options] # This is similar to the --find-links option in pip find-links = [{ path = "./links" }] diff --git a/src/lock_file/resolve/pypi.rs b/src/lock_file/resolve/pypi.rs index bb4e18691..d157bc1dc 100644 --- a/src/lock_file/resolve/pypi.rs +++ b/src/lock_file/resolve/pypi.rs @@ -35,7 +35,7 @@ use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; use uv_distribution_types::{ BuiltDist, DependencyMetadata, Diagnostic, Dist, FileLocation, HashPolicy, IndexCapabilities, - InstalledDist, InstalledRegistryDist, Name, Resolution, ResolvedDist, SourceDist, + IndexUrl, InstalledDist, InstalledRegistryDist, Name, Resolution, ResolvedDist, SourceDist, }; use uv_git::GitResolver; use uv_install_wheel::linker::LinkMode; @@ -56,7 +56,13 @@ use crate::{ uv_reporter::{UvReporter, UvReporterOptions}, }; -fn parse_hashes_from_hash_vec(hashes: &Vec) -> Option { +#[derive(Debug, thiserror::Error)] +#[error("Invalid hash: {0} type: {1}")] +struct InvalidHash(String, String); + +fn parse_hashes_from_hash_vec( + hashes: &Vec, +) -> Result, InvalidHash> { let mut sha256 = None; let mut md5 = None; @@ -75,20 +81,32 @@ fn parse_hashes_from_hash_vec(hashes: &Vec) -> Option } match (sha256, md5) { - (Some(sha256), None) => Some(PackageHashes::Sha256( - parse_digest_from_hex::(&sha256).expect("invalid sha256"), - )), - (None, Some(md5)) => Some(PackageHashes::Md5( - parse_digest_from_hex::(&md5).expect("invalid md5"), - )), - (Some(sha256), Some(md5)) => Some(PackageHashes::Md5Sha256( - parse_digest_from_hex::(&md5).expect("invalid md5"), - parse_digest_from_hex::(&sha256).expect("invalid sha256"), - )), - (None, None) => None, + (Some(sha256), None) => Ok(Some(PackageHashes::Sha256( + parse_digest_from_hex::(&sha256) + .ok_or_else(|| InvalidHash(sha256.clone(), "sha256".to_string()))?, + ))), + (None, Some(md5)) => Ok(Some(PackageHashes::Md5( + parse_digest_from_hex::(&md5) + .ok_or_else(|| InvalidHash(md5.clone(), "md5".to_string()))?, + ))), + (Some(sha256), Some(md5)) => Ok(Some(PackageHashes::Md5Sha256( + parse_digest_from_hex::(&md5) + .ok_or_else(|| InvalidHash(md5.clone(), "md5".to_string()))?, + parse_digest_from_hex::(&sha256) + .ok_or_else(|| InvalidHash(sha256.clone(), "sha256".to_string()))?, + ))), + (None, None) => Ok(None), } } +#[derive(Debug, thiserror::Error)] +enum ProcessPathUrlError { + #[error("expected given path for {0} but none found")] + NoGivenPath(String), + #[error("given path is an invalid file path")] + InvalidFilePath(String), +} + /// Given a pyproject.toml and either case: /// 1) dependencies = [ foo @ /home/foo ] /// 2) tool.pixi.pypi-dependencies.foo = { path = "/home/foo"} @@ -105,14 +123,16 @@ fn parse_hashes_from_hash_vec(hashes: &Vec) -> Option /// relative /// /// I think this has to do with the order of UV processing the requirements -fn process_uv_path_url(path_url: &uv_pep508::VerbatimUrl) -> PathBuf { - let given = path_url.given().expect("path should have a given url"); +fn process_uv_path_url(path_url: &uv_pep508::VerbatimUrl) -> Result { + let given = path_url + .given() + .ok_or_else(|| ProcessPathUrlError::NoGivenPath(path_url.to_string()))?; if given.starts_with("file://") { path_url .to_file_path() - .expect("path should be a valid file path") + .map_err(|_| ProcessPathUrlError::InvalidFilePath(path_url.to_string())) } else { - PathBuf::from(given) + Ok(PathBuf::from(given)) } } @@ -163,13 +183,13 @@ pub async fn resolve_pypi( ) }) .map_ok(|(record, p)| { - ( - uv_normalize::PackageName::new(p.name.as_normalized().to_string()) - .expect("cannot convert to package name"), + Ok(( + uv_normalize::PackageName::new(p.name.as_normalized().to_string())?, (record.clone(), p), - ) + )) }) - .collect::, _>>() + .collect::, uv_normalize::InvalidNameError>, _>>() + .into_diagnostic()? .into_diagnostic() .context("failed to extract python packages from conda metadata")?; @@ -424,7 +444,7 @@ pub async fn resolve_pypi( // one. As we have noted in the comment above. let resolver_in_memory_index = InMemoryIndex::default(); let python_version = PythonVersion::from_str(&interpreter_version.to_string()) - .expect("could not get version from interpreter"); + .map_err(|e| miette::miette!("{}", e))?; let resolution = Resolver::new_custom_io( manifest, options, @@ -465,10 +485,115 @@ pub async fn resolve_pypi( resolution, &context.capabilities, context.concurrency.downloads, + project_root, ) .await } +#[derive(Debug, thiserror::Error)] +enum GetUrlOrPathError { + #[error("expected absolute path found: {path}", path = .0.display())] + InvalidAbsolutePath(PathBuf), + #[error("invalid base url: {0}")] + InvalidBaseUrl(String), + #[error("cannot join these urls {0} + {1}")] + CannotJoin(String, String), + #[error("expected path found: {0}")] + ExpectedPath(String), +} + +/// Get the UrlOrPath from the index url and file location +/// This will be used to handle the case of a source or built distribution +/// coming from a registry index or a `--find-links` path +fn get_url_or_path( + index_url: &IndexUrl, + file_location: &FileLocation, + abs_project_root: &Path, +) -> Result { + const RELATIVE_BASE: &str = "./"; + match index_url { + // This is the case where the registry index is a PyPI index + // or an URL + IndexUrl::Pypi(_) | IndexUrl::Url(_) => { + let url = match file_location { + // Normal case, can be something like: + // https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl + FileLocation::AbsoluteUrl(url) => { + UrlOrPath::Url(Url::from_str(url.as_ref()).map_err(|_| { + GetUrlOrPathError::InvalidAbsolutePath(PathBuf::from(url.to_string())) + })?) + } + // This happens when it is relative to the non-standard index + // because we only lock absolute URLs, we need to join with the base + FileLocation::RelativeUrl(base, relative) => { + let base = Url::from_str(base) + .map_err(|_| GetUrlOrPathError::InvalidBaseUrl(base.clone()))?; + let url = base.join(relative).map_err(|_| { + GetUrlOrPathError::CannotJoin(base.to_string(), relative.clone()) + })?; + UrlOrPath::Url(url) + } + }; + Ok(url) + } + // From observation this is the case where the index is a `--find-links` path + // i.e a path to a directory. This is not a PyPI index, but a directory or a file with links to wheels + IndexUrl::Path(_) => { + let url = match file_location { + // Okay we would have something like: + // file:///home/user/project/dist/certifi-2024.8.30-py3-none-any.whl + FileLocation::AbsoluteUrl(url) => { + // Convert to a relative path from the base path + let absolute = url + .to_url() + .to_file_path() + .map_err(|_| GetUrlOrPathError::ExpectedPath(url.to_string()))?; + // !IMPORTANT! We need to strip the base path from the absolute path + // not the path returned by the uv solver. Why? Because we need the path relative + // to the project root, **not** the path relative to the --find-links path. + // This is because during installation we do something like: `project_root.join(relative_path)` + let relative = absolute.strip_prefix(abs_project_root); + let path = match relative { + // Apparently, we can make it relative to the project root + Ok(relative) => PathBuf::from_str(RELATIVE_BASE) + .map_err(|_| { + GetUrlOrPathError::ExpectedPath(RELATIVE_BASE.to_string()) + })? + .join(relative), + // We can't make it relative to the project root + // so we just return the absolute path + Err(_) => absolute, + }; + UrlOrPath::Path(path) + } + // This happens when it is relative to the non-standard index + // location on disk. + FileLocation::RelativeUrl(base, relative) => { + // This is the same logic as the `AbsoluteUrl` case + // basically but we just make an absolute path first + let absolute = PathBuf::from_str(base) + .map_err(|_| GetUrlOrPathError::ExpectedPath(base.clone()))?; + let relative = PathBuf::from_str(relative) + .map_err(|_| GetUrlOrPathError::ExpectedPath(relative.clone()))?; + let absolute = absolute.join(relative); + + let relative = absolute.strip_prefix(abs_project_root); + let path = match relative { + Ok(relative) => PathBuf::from_str(RELATIVE_BASE) + .map_err(|_| { + GetUrlOrPathError::ExpectedPath(RELATIVE_BASE.to_string()) + })? + .join(relative), + Err(_) => absolute, + }; + UrlOrPath::Path(path) + } + }; + Ok(url) + } + } +} + /// Create a vector of locked packages from a resolution async fn lock_pypi_packages<'a>( conda_python_packages: CondaPythonPackages, @@ -477,6 +602,7 @@ async fn lock_pypi_packages<'a>( resolution: Resolution, index_capabilities: &IndexCapabilities, concurrent_downloads: usize, + abs_project_root: &Path, ) -> miette::Result> { let mut locked_packages = LockedPypiPackages::with_capacity(resolution.len()); let database = DistributionDatabase::new(registry_client, build_dispatch, concurrent_downloads); @@ -487,50 +613,53 @@ async fn lock_pypi_packages<'a>( } let pypi_package_data = match dist { + // Ignore installed distributions ResolvedDist::Installed(_) => { - // TODO handle installed distributions continue; } + ResolvedDist::Installable(Dist::Built(dist)) => { let (url_or_path, hash) = match &dist { BuiltDist::Registry(dist) => { let best_wheel = dist.best_wheel(); - let url = match &best_wheel.file.url { - FileLocation::AbsoluteUrl(url) => UrlOrPath::Url( - Url::from_str(url.as_ref()).expect("invalid absolute url"), - ), - // This happens when it is relative to the non-standard index - FileLocation::RelativeUrl(base, relative) => { - let base = Url::from_str(base).expect("invalid base url"); - let url = base.join(relative).expect("could not join urls"); - UrlOrPath::Url(url) - } - }; - - let hash = parse_hashes_from_hash_vec(&dist.best_wheel().file.hashes); - (url, hash) + let hash = parse_hashes_from_hash_vec(&dist.best_wheel().file.hashes) + .into_diagnostic() + .context("cannot parse hashes for registry dist")?; + let url_or_path = get_url_or_path( + &best_wheel.index, + &best_wheel.file.url, + abs_project_root, + ) + .into_diagnostic() + .context("cannot convert registry dist")?; + (url_or_path, hash) } BuiltDist::DirectUrl(dist) => { let url = dist.url.to_url(); let direct_url = Url::parse(&format!("direct+{url}")) - .expect("could not create direct-url"); + .into_diagnostic() + .context("cannot create direct url")?; (UrlOrPath::Url(direct_url), None) } - BuiltDist::Path(dist) => { - (UrlOrPath::Path(process_uv_path_url(&dist.url)), None) - } + BuiltDist::Path(dist) => ( + UrlOrPath::Path(process_uv_path_url(&dist.url).into_diagnostic()?), + None, + ), }; let metadata = registry_client .wheel_metadata(dist, index_capabilities) .await - .expect("failed to get wheel metadata"); + .into_diagnostic() + .wrap_err("cannot get wheel metadata")?; PypiPackageData { name: pep508_rs::PackageName::new(metadata.name.to_string()) - .expect("cannot convert name"), + .into_diagnostic() + .context("cannot convert name")?, version: pep440_rs::Version::from_str(&metadata.version.to_string()) - .expect("cannot convert version"), + .into_diagnostic() + .context("cannot convert version")?, requires_python: metadata .requires_python .map(|r| to_version_specifiers(&r)) @@ -547,7 +676,13 @@ async fn lock_pypi_packages<'a>( // Handle new hash stuff let hash = source .file() - .and_then(|file| parse_hashes_from_hash_vec(&file.hashes)); + .and_then(|file| { + parse_hashes_from_hash_vec(&file.hashes) + .into_diagnostic() + .context("cannot parse hashes for sdist") + .transpose() + }) + .transpose()?; let metadata_response = database .get_or_build_wheel_metadata(&Dist::Source(source.clone()), HashPolicy::None) @@ -559,23 +694,17 @@ async fn lock_pypi_packages<'a>( // otherwise try to construct it from the source let (url_or_path, hash, editable) = match source { SourceDist::Registry(reg) => { - let url_or_path = match ®.file.url { - FileLocation::AbsoluteUrl(url) => UrlOrPath::Url( - Url::from_str(url.as_ref()).expect("invalid absolute url"), - ), - // This happens when it is relative to the non-standard index - FileLocation::RelativeUrl(base, relative) => { - let base = Url::from_str(base).expect("invalid base url"); - let url = base.join(relative).expect("could not join urls"); - UrlOrPath::Url(url) - } - }; + let url_or_path = + get_url_or_path(®.index, ®.file.url, abs_project_root) + .into_diagnostic() + .context("cannot convert registry sdist")?; (url_or_path, hash, false) } SourceDist::DirectUrl(direct) => { let url = direct.url.to_url(); let direct_url = Url::parse(&format!("direct+{url}")) - .expect("could not create direct-url"); + .into_diagnostic() + .context("could not create direct-url")?; (direct_url.into(), hash, false) } SourceDist::Git(git) => (git.url.to_url().into(), hash, false), @@ -593,7 +722,7 @@ async fn lock_pypi_packages<'a>( }; // process the path or url that we get back from uv - let given_path = process_uv_path_url(&path.url); + let given_path = process_uv_path_url(&path.url).into_diagnostic()?; // Create the url for the lock file. This is based on the passed in URL // instead of from the source path to copy the path that was passed in from @@ -602,7 +731,6 @@ async fn lock_pypi_packages<'a>( (url_or_path, hash, false) } SourceDist::Directory(dir) => { - // TODO: check that `install_path` is correct // Compute the hash of the package based on the source tree. let hash = if dir.install_path.is_dir() { Some( @@ -616,7 +744,7 @@ async fn lock_pypi_packages<'a>( }; // process the path or url that we get back from uv - let given_path = process_uv_path_url(&dir.url); + let given_path = process_uv_path_url(&dir.url).into_diagnostic()?; // Create the url for the lock file. This is based on the passed in URL // instead of from the source path to copy the path that was passed in from @@ -629,7 +757,7 @@ async fn lock_pypi_packages<'a>( PypiPackageData { name: to_normalize(&metadata.name).into_diagnostic()?, version: pep440_rs::Version::from_str(&metadata.version.to_string()) - .expect("cannot convert version"), + .into_diagnostic()?, requires_python: metadata .requires_python .map(|r| to_version_specifiers(&r))