feat: Custom stack size

If a user reports a crash from stacksize - we can try different stack
memory sizes to try and figure out the cause.

Do not try and initialise rayon on 32 bit builds

Fix rayon initialisation, may not have been called before due to badly
formed closure

README.md

@@ -83,8 +83,9 @@ Usage: dust -v regex (Exclude files matching this regex (eg dust -v "\.png$" wou
 Usage: dust -L (dereference-links - Treat sym links as directories and go into them)
 Usage: dust -P (Disable the progress indicator)
 Usage: dust -R (For screen readers. Removes bars/symbols. Adds new column: depth level. (May want to use -p for full path too))
+Usage: dust -S (Custom Stack size - Use if you see: 'fatal runtime error: stack overflow' (default allocation: low memory=1048576, high memory=1073741824)"),
 Usage: dust --skip-total (No total row will be displayed)
-Usage: dust -z 4000000 (Exclude files below size 4MB)
+Usage: dust -z 4000000 (Exclude output files/directories below size 4MB)
 ```
 
 ## Config file

completions/_dust

@@ -29,6 +29,8 @@ _dust() {
 '(-t --file_types)*--filter=[Only include filepaths matching this regex. For png files type\: -e "\\.png\$" ]: : ' \
 '-w+[Specify width of output overriding the auto detection of terminal width]: : ' \
 '--terminal_width=[Specify width of output overriding the auto detection of terminal width]: : ' \
+'-S+[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \
+'--stack-size=[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \
 '-p[Subdirectories will not have their path shortened]' \
 '--full-paths[Subdirectories will not have their path shortened]' \
 '-L[dereference sym links - Treat sym links as directories and go into them]' \

completions/_dust.ps1

@@ -35,6 +35,8 @@ Register-ArgumentCompleter -Native -CommandName 'dust' -ScriptBlock {
             [CompletionResult]::new('--filter', 'filter', [CompletionResultType]::ParameterName, 'Only include filepaths matching this regex. For png files type: -e "\.png$" ')
             [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width')
             [CompletionResult]::new('--terminal_width', 'terminal_width', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width')
+            [CompletionResult]::new('-S', 'S ', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
+            [CompletionResult]::new('--stack-size', 'stack-size', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
             [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened')
             [CompletionResult]::new('--full-paths', 'full-paths', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened')
             [CompletionResult]::new('-L', 'L ', [CompletionResultType]::ParameterName, 'dereference sym links - Treat sym links as directories and go into them')

completions/dust.bash

@@ -19,7 +19,7 @@ _dust() {
 
     case "${cmd}" in
         dust)
-            opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --help --version [params]..."
+            opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -S -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --stack-size --help --version [params]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -81,6 +81,14 @@ _dust() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --stack-size)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -S)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 *)
                     COMPREPLY=()
                     ;;

completions/dust.elv

@@ -32,6 +32,8 @@ set edit:completion:arg-completer[dust] = {|@words|
             cand --filter 'Only include filepaths matching this regex. For png files type: -e "\.png$" '
             cand -w 'Specify width of output overriding the auto detection of terminal width'
             cand --terminal_width 'Specify width of output overriding the auto detection of terminal width'
+            cand -S 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
+            cand --stack-size 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
             cand -p 'Subdirectories will not have their path shortened'
             cand --full-paths 'Subdirectories will not have their path shortened'
             cand -L 'dereference sym links - Treat sym links as directories and go into them'

completions/dust.fish

@@ -5,6 +5,7 @@ complete -c dust -s z -l min-size -d 'Minimum size file to include in output' -r
 complete -c dust -s v -l invert-filter -d 'Exclude filepaths matching this regex. To ignore png files type: -v "\\.png$" ' -r
 complete -c dust -s e -l filter -d 'Only include filepaths matching this regex. For png files type: -e "\\.png$" ' -r
 complete -c dust -s w -l terminal_width -d 'Specify width of output overriding the auto detection of terminal width' -r
+complete -c dust -s S -l stack-size -d 'Specify memory to use as stack size - use if you see: \'fatal runtime error: stack overflow\' (default low memory=1048576, high memory=1073741824)' -r
 complete -c dust -s p -l full-paths -d 'Subdirectories will not have their path shortened'
 complete -c dust -s L -l dereference-links -d 'dereference sym links - Treat sym links as directories and go into them'
 complete -c dust -s x -l limit-filesystem -d 'Only count the files and directories on the same filesystem as the supplied directory'

man-page/dust.1

@@ -4,7 +4,7 @@
 .SH NAME
 Dust \- Like du but more intuitive
 .SH SYNOPSIS
-\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR] 
+\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-S\fR|\fB\-\-stack\-size\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR] 
 .SH DESCRIPTION
 Like du but more intuitive
 .SH OPTIONS
@@ -81,6 +81,9 @@ Only directories will be displayed.
 \fB\-F\fR, \fB\-\-only\-file\fR
 Only files will be displayed. (Finds your largest files)
 .TP
+\fB\-S\fR, \fB\-\-stack\-size\fR
+Specify memory to use as stack size \- use if you see: \*(Aqfatal runtime error: stack overflow\*(Aq (default low memory=1048576, high memory=1073741824)
+.TP
 \fB\-h\fR, \fB\-\-help\fR
 Print help
 .TP

src/cli.rs

@@ -186,6 +186,14 @@ pub fn build_cli() -> Command {
                 .action(clap::ArgAction::SetTrue)
                 .help("Only files will be displayed. (Finds your largest files)"),
         )
+        .arg(
+            Arg::new("stack_size")
+                .short('S')
+                .long("stack-size")
+                .num_args(1)
+                .value_parser(value_parser!(usize))
+                .help("Specify memory to use as stack size - use if you see: 'fatal runtime error: stack overflow' (default low memory=1048576, high memory=1073741824)"),
+        )
         .arg(Arg::new("params").num_args(1..)
             .value_parser(value_parser!(String)))
 }

src/config.rs

@@ -26,6 +26,7 @@ pub struct Config {
     pub disable_progress: Option<bool>,
     pub depth: Option<usize>,
     pub bars_on_right: Option<bool>,
+    pub stack_size: Option<usize>,
 }
 
 impl Config {
@@ -95,6 +96,14 @@ impl Config {
     pub fn get_bars_on_right(&self, options: &ArgMatches) -> bool {
         Some(true) == self.bars_on_right || options.get_flag("bars_on_right")
     }
+    pub fn get_custom_stack_size(&self, options: &ArgMatches) -> Option<usize> {
+        let from_cmd_line = options.get_one::<usize>("stack_size");
+        if from_cmd_line.is_none() {
+            self.stack_size
+        } else {
+            from_cmd_line.copied()
+        }
+    }
 }
 
 fn convert_min_size(input: &str, iso: bool) -> Option<usize> {

src/main.rs

@@ -27,7 +27,6 @@ use config::get_config;
 use dir_walker::walk_it;
 use filter::get_biggest;
 use filter_type::get_all_file_types;
-use rayon::ThreadPoolBuildError;
 use regex::Regex;
 use std::cmp::max;
 use std::path::PathBuf;
@@ -177,11 +176,8 @@ fn main() {
         follow_links,
         progress_data: indicator.data.clone(),
     };
-
-    let result = panic::catch_unwind(|| init_rayon);
-    if result.is_err() {
-        eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
-    }
+    let stack_size = config.get_custom_stack_size(&options);
+    init_rayon(&stack_size);
 
     let top_level_nodes = walk_it(simplified_dirs, walk_data);
 
@@ -227,18 +223,34 @@ fn main() {
     }
 }
 
-fn init_rayon() -> Result<(), ThreadPoolBuildError> {
-    let large_stack = usize::pow(1024, 3);
-    let mut s = System::new();
-    s.refresh_memory();
-    let available = s.available_memory();
-
-    if available > large_stack.try_into().unwrap() {
-        // Larger stack size to handle cases with lots of nested directories
-        rayon::ThreadPoolBuilder::new()
-            .stack_size(large_stack)
-            .build_global()
-    } else {
-        rayon::ThreadPoolBuilder::new().build_global()
+fn init_rayon(stack_size: &Option<usize>) {
+    // Rayon seems to raise this error on 32-bit builds
+    // The global thread pool has not been initialized.: ThreadPoolBuildError { kind: GlobalPoolAlreadyInitialized }
+    if cfg!(target_pointer_width = "64") {
+        let result = panic::catch_unwind(|| {
+            match stack_size {
+                Some(n) => rayon::ThreadPoolBuilder::new()
+                    .stack_size(*n)
+                    .build_global(),
+                None => {
+                    let large_stack = usize::pow(1024, 3);
+                    let mut s = System::new();
+                    s.refresh_memory();
+                    let available = s.available_memory();
+
+                    if available > large_stack.try_into().unwrap() {
+                        // Larger stack size to handle cases with lots of nested directories
+                        rayon::ThreadPoolBuilder::new()
+                            .stack_size(large_stack)
+                            .build_global()
+                    } else {
+                        rayon::ThreadPoolBuilder::new().build_global()
+                    }
+                }
+            }
+        });
+        if result.is_err() {
+            eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
+        }
     }
 }