diff --git a/README.md b/README.md index 90f6b542..7a4b0ae8 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,9 @@ Usage: dust -v regex (Exclude files matching this regex (eg dust -v "\.png$" wou Usage: dust -L (dereference-links - Treat sym links as directories and go into them) Usage: dust -P (Disable the progress indicator) Usage: dust -R (For screen readers. Removes bars/symbols. Adds new column: depth level. (May want to use -p for full path too)) +Usage: dust -S (Custom Stack size - Use if you see: 'fatal runtime error: stack overflow' (default allocation: low memory=1048576, high memory=1073741824)"), Usage: dust --skip-total (No total row will be displayed) -Usage: dust -z 4000000 (Exclude files below size 4MB) +Usage: dust -z 4000000 (Exclude output files/directories below size 4MB) ``` ## Config file diff --git a/completions/_dust b/completions/_dust index 09c1ec9c..3674d226 100644 --- a/completions/_dust +++ b/completions/_dust @@ -29,6 +29,8 @@ _dust() { '(-t --file_types)*--filter=[Only include filepaths matching this regex. For png files type\: -e "\\.png\$" ]: : ' \ '-w+[Specify width of output overriding the auto detection of terminal width]: : ' \ '--terminal_width=[Specify width of output overriding the auto detection of terminal width]: : ' \ +'-S+[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \ +'--stack-size=[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \ '-p[Subdirectories will not have their path shortened]' \ '--full-paths[Subdirectories will not have their path shortened]' \ '-L[dereference sym links - Treat sym links as directories and go into them]' \ diff --git a/completions/_dust.ps1 b/completions/_dust.ps1 index 07b7d2b5..d51dec49 100644 --- a/completions/_dust.ps1 +++ b/completions/_dust.ps1 @@ -35,6 +35,8 @@ Register-ArgumentCompleter -Native -CommandName 'dust' -ScriptBlock { [CompletionResult]::new('--filter', 'filter', [CompletionResultType]::ParameterName, 'Only include filepaths matching this regex. For png files type: -e "\.png$" ') [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width') [CompletionResult]::new('--terminal_width', 'terminal_width', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width') + [CompletionResult]::new('-S', 'S ', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)') + [CompletionResult]::new('--stack-size', 'stack-size', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)') [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened') [CompletionResult]::new('--full-paths', 'full-paths', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened') [CompletionResult]::new('-L', 'L ', [CompletionResultType]::ParameterName, 'dereference sym links - Treat sym links as directories and go into them') diff --git a/completions/dust.bash b/completions/dust.bash index e427aff2..bbef4a14 100644 --- a/completions/dust.bash +++ b/completions/dust.bash @@ -19,7 +19,7 @@ _dust() { case "${cmd}" in dust) - opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --help --version [params]..." + opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -S -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --stack-size --help --version [params]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -81,6 +81,14 @@ _dust() { COMPREPLY=($(compgen -f "${cur}")) return 0 ;; + --stack-size) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -S) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; *) COMPREPLY=() ;; diff --git a/completions/dust.elv b/completions/dust.elv index 66f07330..036e0b8d 100644 --- a/completions/dust.elv +++ b/completions/dust.elv @@ -32,6 +32,8 @@ set edit:completion:arg-completer[dust] = {|@words| cand --filter 'Only include filepaths matching this regex. For png files type: -e "\.png$" ' cand -w 'Specify width of output overriding the auto detection of terminal width' cand --terminal_width 'Specify width of output overriding the auto detection of terminal width' + cand -S 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)' + cand --stack-size 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)' cand -p 'Subdirectories will not have their path shortened' cand --full-paths 'Subdirectories will not have their path shortened' cand -L 'dereference sym links - Treat sym links as directories and go into them' diff --git a/completions/dust.fish b/completions/dust.fish index 9778e1a9..30181cf4 100644 --- a/completions/dust.fish +++ b/completions/dust.fish @@ -5,6 +5,7 @@ complete -c dust -s z -l min-size -d 'Minimum size file to include in output' -r complete -c dust -s v -l invert-filter -d 'Exclude filepaths matching this regex. To ignore png files type: -v "\\.png$" ' -r complete -c dust -s e -l filter -d 'Only include filepaths matching this regex. For png files type: -e "\\.png$" ' -r complete -c dust -s w -l terminal_width -d 'Specify width of output overriding the auto detection of terminal width' -r +complete -c dust -s S -l stack-size -d 'Specify memory to use as stack size - use if you see: \'fatal runtime error: stack overflow\' (default low memory=1048576, high memory=1073741824)' -r complete -c dust -s p -l full-paths -d 'Subdirectories will not have their path shortened' complete -c dust -s L -l dereference-links -d 'dereference sym links - Treat sym links as directories and go into them' complete -c dust -s x -l limit-filesystem -d 'Only count the files and directories on the same filesystem as the supplied directory' diff --git a/man-page/dust.1 b/man-page/dust.1 index 2552ba34..288a663c 100644 --- a/man-page/dust.1 +++ b/man-page/dust.1 @@ -4,7 +4,7 @@ .SH NAME Dust \- Like du but more intuitive .SH SYNOPSIS -\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR] +\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-S\fR|\fB\-\-stack\-size\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR] .SH DESCRIPTION Like du but more intuitive .SH OPTIONS @@ -81,6 +81,9 @@ Only directories will be displayed. \fB\-F\fR, \fB\-\-only\-file\fR Only files will be displayed. (Finds your largest files) .TP +\fB\-S\fR, \fB\-\-stack\-size\fR +Specify memory to use as stack size \- use if you see: \*(Aqfatal runtime error: stack overflow\*(Aq (default low memory=1048576, high memory=1073741824) +.TP \fB\-h\fR, \fB\-\-help\fR Print help .TP diff --git a/src/cli.rs b/src/cli.rs index 1dcd3d45..20286712 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -186,6 +186,14 @@ pub fn build_cli() -> Command { .action(clap::ArgAction::SetTrue) .help("Only files will be displayed. (Finds your largest files)"), ) + .arg( + Arg::new("stack_size") + .short('S') + .long("stack-size") + .num_args(1) + .value_parser(value_parser!(usize)) + .help("Specify memory to use as stack size - use if you see: 'fatal runtime error: stack overflow' (default low memory=1048576, high memory=1073741824)"), + ) .arg(Arg::new("params").num_args(1..) .value_parser(value_parser!(String))) } diff --git a/src/config.rs b/src/config.rs index 99c92869..d74528ab 100644 --- a/src/config.rs +++ b/src/config.rs @@ -26,6 +26,7 @@ pub struct Config { pub disable_progress: Option, pub depth: Option, pub bars_on_right: Option, + pub stack_size: Option, } impl Config { @@ -95,6 +96,14 @@ impl Config { pub fn get_bars_on_right(&self, options: &ArgMatches) -> bool { Some(true) == self.bars_on_right || options.get_flag("bars_on_right") } + pub fn get_custom_stack_size(&self, options: &ArgMatches) -> Option { + let from_cmd_line = options.get_one::("stack_size"); + if from_cmd_line.is_none() { + self.stack_size + } else { + from_cmd_line.copied() + } + } } fn convert_min_size(input: &str, iso: bool) -> Option { diff --git a/src/main.rs b/src/main.rs index 236a2742..b31421c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,7 +27,6 @@ use config::get_config; use dir_walker::walk_it; use filter::get_biggest; use filter_type::get_all_file_types; -use rayon::ThreadPoolBuildError; use regex::Regex; use std::cmp::max; use std::path::PathBuf; @@ -177,11 +176,8 @@ fn main() { follow_links, progress_data: indicator.data.clone(), }; - - let result = panic::catch_unwind(|| init_rayon); - if result.is_err() { - eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1") - } + let stack_size = config.get_custom_stack_size(&options); + init_rayon(&stack_size); let top_level_nodes = walk_it(simplified_dirs, walk_data); @@ -227,18 +223,34 @@ fn main() { } } -fn init_rayon() -> Result<(), ThreadPoolBuildError> { - let large_stack = usize::pow(1024, 3); - let mut s = System::new(); - s.refresh_memory(); - let available = s.available_memory(); - - if available > large_stack.try_into().unwrap() { - // Larger stack size to handle cases with lots of nested directories - rayon::ThreadPoolBuilder::new() - .stack_size(large_stack) - .build_global() - } else { - rayon::ThreadPoolBuilder::new().build_global() +fn init_rayon(stack_size: &Option) { + // Rayon seems to raise this error on 32-bit builds + // The global thread pool has not been initialized.: ThreadPoolBuildError { kind: GlobalPoolAlreadyInitialized } + if cfg!(target_pointer_width = "64") { + let result = panic::catch_unwind(|| { + match stack_size { + Some(n) => rayon::ThreadPoolBuilder::new() + .stack_size(*n) + .build_global(), + None => { + let large_stack = usize::pow(1024, 3); + let mut s = System::new(); + s.refresh_memory(); + let available = s.available_memory(); + + if available > large_stack.try_into().unwrap() { + // Larger stack size to handle cases with lots of nested directories + rayon::ThreadPoolBuilder::new() + .stack_size(large_stack) + .build_global() + } else { + rayon::ThreadPoolBuilder::new().build_global() + } + } + } + }); + if result.is_err() { + eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1") + } } }