diff --git a/docs/cli.md b/docs/cli.md index 225e530dd..9d2e093da 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -323,13 +323,11 @@ List project's packages. Highlighted packages are explicit dependencies. - `--json-pretty`: Whether to output in pretty json format - `--sort-by `: Sorting strategy [default: name] [possible values: size, name, type] - `--manifest-path `: The path to [manifest file](configuration.md), by default it searches for one in the parent directories. -- `--environment`(`-e`): The environment's packages to list, if non is provided the default environment's packages will be listed. +- `--environment (-e)`: The environment's packages to list, if non is provided the default environment's packages will be listed. - `--frozen`: Install the environment as defined in the lockfile. Without checking the status of the lockfile. It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). - `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. - `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`) -```shell - ```shell pixi list pixi list --json-pretty @@ -368,6 +366,124 @@ Output will look like this, where `python` will be green as it is the package th xz 5.2.6 h166bdaf_0 408.6 KiB conda xz-5.2.6-h166bdaf_0.tar.bz2 ``` +## `tree` + +Display the project's packages in a tree. Highlighted packages are those specified in the manifest. + +The package tree can also be inverted (`-i`), to see which packages require a specific dependencies. + +##### Arguments + +- `REGEX` optional regex of which direct dependencies to filter the tree to, or which dependencies to start with when inverting the tree. + +##### Options + +- `--invert (-i)`: Invert the dependency tree, that is given a `REGEX` pattern that matches some packages, show all the packages that depend on those. +- `--platform (-p)`: The platform to list packages for. Defaults to the current platform +- `--manifest-path `: The path to [manifest file](configuration.md), by default it searches for one in the parent directories. +- `--environment (-e)`: The environment's packages to list, if non is provided the default environment's packages will be listed. +- `--frozen`: Install the environment as defined in the lockfile. Without checking the status of the lockfile. It can also be controlled by the `PIXI_FROZEN` environment variable (example: `PIXI_FROZEN=true`). +- `--locked`: Only install if the `pixi.lock` is up-to-date with the [manifest file](configuration.md)[^1]. It can also be controlled by the `PIXI_LOCKED` environment variable (example: `PIXI_LOCKED=true`). Conflicts with `--frozen`. +- `--no-install`: Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. (Implied by `--frozen` and `--locked`) + +```shell +pixi tree +pixi tree pre-commit +pixi tree -i yaml +pixi tree --environment docs +pixi tree --platform win-64 +``` + +!!! warning + Use `-v` to show which `pypi` packages are not yet parsed correctly. The `extras` and `markers` parsing is still under development. + +Output will look like this, where direct packages in the [manifest file](configuration.md) will be green. +Once a package has been displayed once, the tree won't continue to recurse through its dependencies (compare the first time `python` appears, vs the rest), and it will instead be marked with a star `(*)`. + +Version numbers are colored by the package type, yellow for Conda packages and blue for PyPI. + +```shell +➜ pixi tree +├── pre-commit v3.3.3 +│ ├── cfgv v3.3.1 +│ │ └── python v3.12.2 +│ │ ├── bzip2 v1.0.8 +│ │ ├── libexpat v2.6.2 +│ │ ├── libffi v3.4.2 +│ │ ├── libsqlite v3.45.2 +│ │ │ └── libzlib v1.2.13 +│ │ ├── libzlib v1.2.13 (*) +│ │ ├── ncurses v6.4.20240210 +│ │ ├── openssl v3.2.1 +│ │ ├── readline v8.2 +│ │ │ └── ncurses v6.4.20240210 (*) +│ │ ├── tk v8.6.13 +│ │ │ └── libzlib v1.2.13 (*) +│ │ └── xz v5.2.6 +│ ├── identify v2.5.35 +│ │ └── python v3.12.2 (*) +... +└── tbump v6.9.0 +... + └── tomlkit v0.12.4 + └── python v3.12.2 (*) +``` + +A regex pattern can be specified to filter the tree to just those that show a specific direct dependency: + +```shell +➜ pixi tree pre-commit +└── pre-commit v3.3.3 + ├── virtualenv v20.25.1 + │ ├── filelock v3.13.1 + │ │ └── python v3.12.2 + │ │ ├── libexpat v2.6.2 + │ │ ├── readline v8.2 + │ │ │ └── ncurses v6.4.20240210 + │ │ ├── libsqlite v3.45.2 + │ │ │ └── libzlib v1.2.13 + │ │ ├── bzip2 v1.0.8 + │ │ ├── libzlib v1.2.13 (*) + │ │ ├── libffi v3.4.2 + │ │ ├── tk v8.6.13 + │ │ │ └── libzlib v1.2.13 (*) + │ │ ├── xz v5.2.6 + │ │ ├── ncurses v6.4.20240210 (*) + │ │ └── openssl v3.2.1 + │ ├── platformdirs v4.2.0 + │ │ └── python v3.12.2 (*) + │ ├── distlib v0.3.8 + │ │ └── python v3.12.2 (*) + │ └── python v3.12.2 (*) + ├── pyyaml v6.0.1 +... +``` + +Additionally, the tree can be inverted, and it can show which packages depend on a regex pattern. +The packages specified in the manifest will also be highlighted (in this case `cffconvert` and `pre-commit` would be). + +```shell +➜ pixi tree -i yaml + +ruamel.yaml v0.18.6 +├── pykwalify v1.8.0 +│ └── cffconvert v2.0.0 +└── cffconvert v2.0.0 + +pyyaml v6.0.1 +└── pre-commit v3.3.3 + +ruamel.yaml.clib v0.2.8 +└── ruamel.yaml v0.18.6 + ├── pykwalify v1.8.0 + │ └── cffconvert v2.0.0 + └── cffconvert v2.0.0 + +yaml v0.2.5 +└── pyyaml v6.0.1 + └── pre-commit v3.3.3 +``` + ## `shell` This command starts a new shell in the project's environment. diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 2e24fb514..06786a277 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -23,6 +23,7 @@ pub mod self_update; pub mod shell; pub mod shell_hook; pub mod task; +pub mod tree; pub mod upload; #[derive(Parser, Debug)] @@ -75,6 +76,7 @@ pub enum Command { Remove(remove::Args), SelfUpdate(self_update::Args), List(list::Args), + Tree(tree::Args), } #[derive(Parser, Debug, Default, Copy, Clone)] @@ -230,6 +232,7 @@ pub async fn execute_command(command: Command) -> miette::Result<()> { Command::Remove(cmd) => remove::execute(cmd).await, Command::SelfUpdate(cmd) => self_update::execute(cmd).await, Command::List(cmd) => list::execute(cmd).await, + Command::Tree(cmd) => tree::execute(cmd).await, } } diff --git a/src/cli/tree.rs b/src/cli/tree.rs new file mode 100644 index 000000000..7cbeb74e3 --- /dev/null +++ b/src/cli/tree.rs @@ -0,0 +1,425 @@ +use std::collections::HashMap; +use std::path::PathBuf; + +use clap::Parser; +use console::Color; +use itertools::Itertools; +use rattler_conda_types::Platform; + +use crate::lock_file::UpdateLockFileOptions; +use crate::project::manifest::EnvironmentName; +use crate::Project; + +/// Show a tree of project dependencies +#[derive(Debug, Parser)] +#[clap(arg_required_else_help = false, long_about=format!( + "\ + Show a tree of project dependencies\n\ + \n\ + Dependency names highlighted in {} are directly specified in the manifest. \ + {} version numbers are conda packages, PyPI version numbers are {}. + ", + console::style("green").fg(Color::Green).bold(), + console::style("Yellow").fg(Color::Yellow), + console::style("blue").fg(Color::Blue) +))] +pub struct Args { + /// List only packages matching a regular expression + #[arg()] + pub regex: Option, + + /// The platform to list packages for. Defaults to the current platform. + #[arg(long, short)] + pub platform: Option, + + /// The path to 'pixi.toml' + #[arg(long)] + pub manifest_path: Option, + + /// The environment to list packages for. Defaults to the default environment. + #[arg(short, long)] + pub environment: Option, + + #[clap(flatten)] + pub lock_file_usage: super::LockFileUsageArgs, + + /// Don't install the environment for pypi solving, only update the lock-file if it can solve without installing. + #[arg(long)] + pub no_install: bool, + + /// Invert tree and show what depends on given package in the regex argument + #[arg(short, long, requires = "regex")] + pub invert: bool, +} + +struct Symbols { + down: &'static str, + tee: &'static str, + ell: &'static str, + // right: &'static str, + empty: &'static str, +} + +static UTF8_SYMBOLS: Symbols = Symbols { + down: "│ ", + tee: "├──", + ell: "└──", + // right: "───", + empty: " ", +}; + +pub async fn execute(args: Args) -> miette::Result<()> { + let project = Project::load_or_else_discover(args.manifest_path.as_deref())?; + let environment_name = args + .environment + .map_or_else(|| EnvironmentName::Default, EnvironmentName::Named); + let environment = project + .environment(&environment_name) + .ok_or_else(|| miette::miette!("unknown environment '{environment_name}'"))?; + let lock_file = project + .up_to_date_lock_file(UpdateLockFileOptions { + lock_file_usage: args.lock_file_usage.into(), + no_install: args.no_install, + ..UpdateLockFileOptions::default() + }) + .await?; + let platform = args.platform.unwrap_or_else(Platform::current); + let locked_deps = lock_file + .lock_file + .environment(environment.name().as_str()) + .and_then(|env| env.packages(platform).map(Vec::from_iter)) + .unwrap_or_default(); + + let dep_map = generate_dependency_map(&locked_deps); + + let direct_deps = direct_dependencies(&environment, &platform); + + if args.invert { + print_inverted_dependency_tree(&invert_dep_map(&dep_map), &direct_deps, &args.regex)?; + } else { + print_dependency_tree(&dep_map, &direct_deps, &args.regex)?; + } + Ok(()) +} + +/// Filter and print an inverted dependency tree +fn print_inverted_dependency_tree( + inverted_dep_map: &HashMap, + direct_deps: &Vec, + regex: &Option, +) -> Result<(), miette::Error> { + let regex = regex + .as_ref() + .ok_or("") + .map_err(|_| miette::miette!("The -i flag requires a package name."))?; + let regex = regex::Regex::new(regex).map_err(|_| miette::miette!("Invalid regex pattern"))?; + + let mut root_pkg_names = inverted_dep_map.keys().collect_vec(); + root_pkg_names.retain(|p| regex.is_match(p)); + + if root_pkg_names.is_empty() { + Err(miette::miette!( + "Nothing depends on the given regular expression", + ))?; + } + + for pkg_name in root_pkg_names.iter() { + if let Some(pkg) = inverted_dep_map.get(*pkg_name) { + print_package( + "\n".to_string(), + pkg, + direct_deps.contains(&pkg.name), + false, + ); + + print_inverted_leaf(pkg, String::from(""), inverted_dep_map, direct_deps); + } + } + + Ok(()) +} + +/// Recursively print inverted dependency tree leaf nodes +fn print_inverted_leaf( + pkg: &Package, + prefix: String, + inverted_dep_map: &HashMap, + direct_deps: &Vec, +) { + let needed_count = pkg.needed_by.len(); + for (index, needed_name) in pkg.needed_by.iter().enumerate() { + let last = index == needed_count - 1; + let symbol = if last { + UTF8_SYMBOLS.ell + } else { + UTF8_SYMBOLS.tee + }; + + if let Some(needed_pkg) = inverted_dep_map.get(needed_name) { + print_package( + format!("{prefix}{symbol} "), + needed_pkg, + direct_deps.contains(&needed_pkg.name), + false, + ); + + let new_prefix = if index == needed_count - 1 { + format!("{}{} ", prefix, UTF8_SYMBOLS.empty) + } else { + format!("{}{} ", prefix, UTF8_SYMBOLS.down) + }; + + print_inverted_leaf(needed_pkg, new_prefix, inverted_dep_map, direct_deps) + } + } +} + +/// Filter and print a top down dependency tree +fn print_dependency_tree( + dep_map: &HashMap, + direct_deps: &Vec, + regex: &Option, +) -> Result<(), miette::Error> { + let mut filtered_deps = direct_deps.clone(); + + if let Some(regex) = regex { + let regex = regex::Regex::new(regex).map_err(|_| miette::miette!("Invalid regex"))?; + filtered_deps.retain(|p| regex.is_match(p)); + + if filtered_deps.is_empty() { + Err(miette::miette!( + "No top level dependencies matched the given regular expression" + ))?; + } + } + + let mut visited_pkgs = Vec::new(); + let direct_dep_count = filtered_deps.len(); + + for (index, pkg_name) in filtered_deps.iter().enumerate() { + visited_pkgs.push(pkg_name.to_owned()); + + let last = index == direct_dep_count - 1; + let symbol = if last { + UTF8_SYMBOLS.ell + } else { + UTF8_SYMBOLS.tee + }; + if let Some(pkg) = dep_map.get(pkg_name) { + print_package(format!("{symbol} "), pkg, true, false); + + let prefix = if last { + UTF8_SYMBOLS.empty + } else { + UTF8_SYMBOLS.down + }; + print_dependency_leaf( + pkg, + format!("{} ", prefix), + dep_map, + &mut visited_pkgs, + direct_deps, + ) + } + } + Ok(()) +} + +/// Recursively print top down dependency tree nodes +fn print_dependency_leaf( + pkg: &Package, + prefix: String, + dep_map: &HashMap, + visited_pkgs: &mut Vec, + direct_deps: &Vec, +) { + let dep_count = pkg.dependencies.len(); + for (index, dep_name) in pkg.dependencies.iter().enumerate() { + let last = index == dep_count - 1; + let symbol = if last { + UTF8_SYMBOLS.ell + } else { + UTF8_SYMBOLS.tee + }; + + if let Some(dep) = dep_map.get(dep_name) { + let visited = visited_pkgs.contains(&dep.name); + visited_pkgs.push(dep.name.to_owned()); + + print_package( + format!("{prefix}{symbol} "), + dep, + direct_deps.contains(&dep.name), + visited, + ); + + if visited { + continue; + } + + let new_prefix = if last { + format!("{}{} ", prefix, UTF8_SYMBOLS.empty) + } else { + format!("{}{} ", prefix, UTF8_SYMBOLS.down) + }; + print_dependency_leaf(dep, new_prefix, dep_map, visited_pkgs, direct_deps); + } else { + let visited = visited_pkgs.contains(dep_name); + visited_pkgs.push(dep_name.to_owned()); + + print_package( + format!("{prefix}{symbol} "), + &Package { + name: dep_name.to_owned(), + version: String::from(""), + dependencies: Vec::new(), + needed_by: Vec::new(), + source: PackageSource::Conda, + }, + false, + visited, + ) + } + } +} + +/// Print package and style by attributes, like if are a direct dependency (name is green and bold), +/// or by the source of the package (yellow version string for Conda, blue for PyPI). +/// Packages that have already been visited and will not be recursed into again are +/// marked with a star (*). +fn print_package(prefix: String, package: &Package, direct: bool, visited: bool) { + println!( + "{}{} {} {}", + prefix, + if direct { + console::style(&package.name).fg(Color::Green).bold() + } else { + console::style(&package.name) + }, + match package.source { + PackageSource::Conda => console::style(&package.version).fg(Color::Yellow), + PackageSource::Pypi => console::style(&package.version).fg(Color::Blue), + }, + if visited { "(*)" } else { "" } + ); +} + +/// Extract the direct Conda and PyPI dependencies from the environment +fn direct_dependencies( + environment: &crate::project::Environment<'_>, + platform: &Platform, +) -> Vec { + let mut project_dependency_names = environment + .dependencies(None, Some(*platform)) + .names() + .map(|p| p.as_source().to_string()) + .collect_vec(); + project_dependency_names.extend( + environment + .pypi_dependencies(Some(*platform)) + .into_iter() + .map(|(name, _)| name.as_normalized().as_dist_info_name().into_owned()), + ); + project_dependency_names +} + +#[derive(Debug, Copy, Clone)] +enum PackageSource { + Conda, + Pypi, +} + +#[derive(Debug, Clone)] +struct Package { + name: String, + version: String, + dependencies: Vec, + needed_by: Vec, + source: PackageSource, +} + +/// Builds a hashmap of dependencies, with names, versions, and what they depend on +fn generate_dependency_map(locked_deps: &Vec) -> HashMap { + let mut package_dependencies_map = HashMap::new(); + + for package in locked_deps { + let version = package.version().into_owned(); + + if let Some(conda_package) = package.as_conda() { + let name = conda_package + .package_record() + .name + .as_normalized() + .to_string(); + // Parse the dependencies of the package + let dependencies: Vec = conda_package + .package_record() + .depends + .iter() + .map(|d| { + d.split_once(' ') + .map_or_else(|| d.to_string(), |(dep_name, _)| dep_name.to_string()) + }) + .collect(); + + package_dependencies_map.insert( + name.clone(), + Package { + name: name.clone(), + version, + dependencies: dependencies.into_iter().unique().collect(), + needed_by: Vec::new(), + source: PackageSource::Conda, + }, + ); + } else if let Some(pypi_package) = package.as_pypi() { + let name = pypi_package + .data() + .package + .name + .as_dist_info_name() + .into_owned(); + + let mut dependencies = Vec::new(); + for p in pypi_package.data().package.requires_dist.iter() { + if let Some(markers) = &p.marker { + tracing::info!( + "Extra and environment markers currently cannot be parsed on {} which is specified by {}, skipping. {:?}", + p.name, + name, + markers + ); + } else { + dependencies.push(p.name.as_dist_info_name().into_owned()) + } + } + package_dependencies_map.insert( + name.clone(), + Package { + name: name.clone(), + version, + dependencies: dependencies.into_iter().unique().collect(), + needed_by: Vec::new(), + source: PackageSource::Pypi, + }, + ); + } + } + package_dependencies_map +} + +/// Given a map of dependencies, invert it so that it has what a package is needed by, +/// rather than what it depends on +fn invert_dep_map(dep_map: &HashMap) -> HashMap { + let mut inverted_deps = dep_map.clone(); + + for pkg in dep_map.values() { + for dep in pkg.dependencies.iter() { + if let Some(idep) = inverted_deps.get_mut(dep) { + idep.needed_by.push(pkg.name.clone()); + } + } + } + + inverted_deps +}