diff --git a/carbonserver/carbonserver.go b/carbonserver/carbonserver.go index c68b84d35..e2fc7e1dc 100644 --- a/carbonserver/carbonserver.go +++ b/carbonserver/carbonserver.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "io/fs" "math" "net" "net/http" @@ -49,6 +50,7 @@ import ( "go.uber.org/zap" "github.com/NYTimes/gziphandler" + "github.com/charlievieth/fastwalk" "github.com/dgryski/go-expirecache" "github.com/dgryski/go-trigram" "github.com/dgryski/httputil" @@ -900,10 +902,11 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName var t0 = time.Now() var fidx = listener.CurrentFileIndex() var files []string - var filesLen int + var filesMutex sync.Mutex + var filesLen atomic.Uint64 var details = make(map[string]*protov3.MetricDetails) var trieIdx *trieIndex - var metricsKnown uint64 + var metricsKnown atomic.Uint64 var infos []zap.Field if listener.trieIndex { if fidx == nil || !listener.concurrentIndex { @@ -927,7 +930,7 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName files = append(files, fileName) } if strings.HasSuffix(fileName, ".wsp") { - metricsKnown++ + metricsKnown.Add(1) } } cacheIndexRuntime := time.Since(tcache) @@ -972,9 +975,9 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName break } - filesLen++ + filesLen.Add(1) if strings.HasSuffix(entry.Path, ".wsp") { - metricsKnown++ + metricsKnown.Add(1) } } if err := flc.Close(); err != nil { @@ -1012,12 +1015,25 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName logger.Error("can't index symlink data dir", zap.String("path", dir)) } - err := filepath.Walk(dir, func(p string, info os.FileInfo, err error) error { + fastwalkConf := fastwalk.Config{ + Follow: false, // do not follow symlinks + // numWorkers default is sane here (>=4, but <=32) + } + // please note that fastwalk.Walk function should be concurrently safe + // TODO: refactor this + // e.g. we can construct filesList concurrently and update index from fileList cache single threaded later + err := fastwalk.Walk(&fastwalkConf, dir, func(p string, d fs.DirEntry, err error) error { if err != nil { logger.Info("error processing", zap.String("path", p), zap.Error(err)) return nil } + // getting file info from direntry + info, ierr := d.Info() + if ierr != nil { + logger.Info("error processing", zap.String("path", p), zap.Error(err)) + return nil + } // WHY: as filepath.walk could potentially taking a long // time to complete (>= 5 minutes or more), depending // on how many files are there on disk. It's nice to @@ -1039,7 +1055,8 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName // // TODO: only trigger enter the loop when it's half full? // len(listener.newMetricsChan) >= cap(listener.newMetricsChan)/2 - if listener.trieIndex && listener.concurrentIndex && listener.newMetricsChan != nil { + if listener.trieIndex && listener.concurrentIndex && listener.newMetricsChan != nil && len(listener.newMetricsChan) >= cap(listener.newMetricsChan)/2 { + logger.Info("trying to flush incoming metrics into index") newMetricsLoop: for { select { @@ -1054,10 +1071,11 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName } } + logger.Info("flush of incoming metrics done, proceeding with file list populating") isFullMetric := strings.HasSuffix(info.Name(), ".wsp") if info.IsDir() || isFullMetric { // both dir and metric file is needed for supporting trigram index. trimmedName := strings.TrimPrefix(p, listener.whisperData) - filesLen++ + filesLen.Add(1) var dataPoints, logicalSize, physicalSize int64 if isFullMetric { @@ -1093,11 +1111,14 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName } } } else { + // we're in fastwalk goroutine + filesMutex.Lock() + defer filesMutex.Unlock() files = append(files, trimmedName) } if isFullMetric { - metricsKnown++ + metricsKnown.Add(1) } } @@ -1138,6 +1159,7 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName ) } } + logger.Info("filewalk is done, proceeding with index update") if listener.concurrentIndex && trieIdx != nil { trieIdx.prune() @@ -1160,7 +1182,7 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName totalSpace := stat.Blocks * uint64(stat.Bsize) fileScanRuntime := time.Since(t0) - atomic.StoreUint64(&listener.metrics.MetricsKnown, metricsKnown) + atomic.StoreUint64(&listener.metrics.MetricsKnown, metricsKnown.Load()) atomic.AddUint64(&listener.metrics.FileScanTimeNS, uint64(fileScanRuntime.Nanoseconds())) nfidx := &fileIndex{ @@ -1226,12 +1248,12 @@ func (listener *CarbonserverListener) updateFileList(dir string, cacheMetricName zap.Duration("rdtime_update_runtime", rdTimeUpdateRuntime), zap.Duration("cache_index_runtime", cacheIndexRuntime), zap.Duration("total_runtime", time.Since(t0)), - zap.Int("Files", filesLen), + zap.Uint64("Files", filesLen.Load()), zap.Int("index_size", indexSize), zap.Int("pruned_trigrams", pruned), zap.Int("cache_metric_len_before", cacheMetricLen), zap.Int("cache_metric_len_after", len(cacheMetricNames)), - zap.Uint64("metrics_known", metricsKnown), + zap.Uint64("metrics_known", metricsKnown.Load()), zap.String("index_type", indexType), zap.Bool("read_from_cache", readFromCache), ) diff --git a/carbonserver/flc.go b/carbonserver/flc.go index 2941115c6..f92d878e5 100644 --- a/carbonserver/flc.go +++ b/carbonserver/flc.go @@ -10,6 +10,7 @@ import ( "io" "os" "strings" + "sync" ) // file list cache @@ -82,6 +83,7 @@ type fileListCacheCommon struct { path string mode byte file *os.File + mutex sync.Mutex reader *gzip.Reader writer *gzip.Writer } @@ -237,6 +239,8 @@ func newFileListCacheV1ReadOnly(flcc *fileListCacheCommon) *fileListCacheV1 { } func (flc *fileListCacheV1) Write(entry *FLCEntry) error { + flc.mutex.Lock() + defer flc.mutex.Unlock() _, err := flc.writer.Write([]byte(entry.Path + "\n")) return err } @@ -300,6 +304,8 @@ func (flc *fileListCacheV2) Write(entry *FLCEntry) error { buf[offset] = '\n' + flc.mutex.Lock() + defer flc.mutex.Unlock() _, err := flc.writer.Write(buf) return err diff --git a/carbonserver/trie.go b/carbonserver/trie.go index 8bfad2bed..07c49c178 100644 --- a/carbonserver/trie.go +++ b/carbonserver/trie.go @@ -292,6 +292,7 @@ type trieIndex struct { fileCount int depth uint64 longestMetric string + mutex sync.Mutex // qau: Quota And Usage qauMetrics []points.Points @@ -492,6 +493,8 @@ func (t *trieInsertError) Error() string { return t.typ } // // insert returns either a file node or dir node, after inserted. func (ti *trieIndex) insert(path string, logicalSize, physicalSize, dataPoints, firstSeenAt int64) (*trieNode, error) { + ti.mutex.Lock() + defer ti.mutex.Unlock() path = filepath.Clean(path) if len(path) > 0 && path[0] == '/' { // skipcq: GO-S1005 path = path[1:] diff --git a/go.mod b/go.mod index a237dd20d..394f2c5e1 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( ) require ( + github.com/charlievieth/fastwalk v1.0.3 golang.org/x/net v0.25.0 google.golang.org/protobuf v1.34.1 ) diff --git a/go.sum b/go.sum index 664f41e60..cec261188 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charlievieth/fastwalk v1.0.3 h1:eNWFaNPe5srPqQ5yyDbhAf11paeZaHWcihRhpuYFfSg= +github.com/charlievieth/fastwalk v1.0.3/go.mod h1:JSfglY/gmL/rqsUS1NCsJTocB5n6sSl9ApAqif4CUbs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= diff --git a/vendor/github.com/charlievieth/fastwalk/.gitignore b/vendor/github.com/charlievieth/fastwalk/.gitignore new file mode 100644 index 000000000..21efe58d4 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/.gitignore @@ -0,0 +1,2 @@ +/vendor +*.test diff --git a/vendor/github.com/charlievieth/fastwalk/LICENSE b/vendor/github.com/charlievieth/fastwalk/LICENSE new file mode 100644 index 000000000..6d18063cd --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2022 Charlie Vieth + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/charlievieth/fastwalk/Makefile b/vendor/github.com/charlievieth/fastwalk/Makefile new file mode 100644 index 000000000..9fc53ac7a --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/Makefile @@ -0,0 +1,60 @@ +.PHONY: test_build_darwin_arm64 +test_build_darwin_arm64: + GOOS=darwin GOARCH=arm64 go test -c -o /dev/null + +.PHONY: test_build_darwin_amd64 +test_build_darwin_amd64: + GOOS=darwin GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_linux_arm64 +test_build_linux_arm64: + GOOS=linux GOARCH=arm64 go test -c -o /dev/null + +.PHONY: test_build_linux_amd64 +test_build_linux_amd64: + GOOS=linux GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_windows_amd64 +test_build_windows_amd64: + GOOS=windows GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_freebsd_amd64 +test_build_freebsd_amd64: + GOOS=freebsd GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_openbsd_amd64 +test_build_openbsd_amd64: + GOOS=openbsd GOARCH=amd64 go test -c -o /dev/null + +.PHONY: test_build_netbsd_amd64 +test_build_netbsd_amd64: + GOOS=netbsd GOARCH=amd64 go test -c -o /dev/null + +# Test that we can build fastwalk on multiple platforms +.PHONY: test_build +test_build: test_build_darwin_arm64 test_build_darwin_amd64 \ + test_build_linux_arm64 test_build_linux_amd64 \ + test_build_windows_amd64 test_build_freebsd_amd64 \ + test_build_openbsd_amd64 test_build_netbsd_amd64 + +.PHONY: test +test: # runs all tests against the package with race detection and coverage percentage + @go test -race -cover ./... +ifeq "$(shell go env GOOS)" "darwin" + @go test -tags nogetdirentries -race -cover ./... +endif + +.PHONY: quick +quick: # runs all tests without coverage or the race detector + @go test ./... + +.PHONY: bench +bench: + @go test -run '^$' -bench . -benchmem ./... + +.PHONY: bench_comp +bench_comp: + @go run ./scripts/bench_comp.go + +.PHONY: all +all: test test_build diff --git a/vendor/github.com/charlievieth/fastwalk/README.md b/vendor/github.com/charlievieth/fastwalk/README.md new file mode 100644 index 000000000..7b8faa05f --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/README.md @@ -0,0 +1,218 @@ +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](https://pkg.go.dev/github.com/charlievieth/fastwalk) +[![Test fastwalk on macOS](https://github.com/charlievieth/fastwalk/actions/workflows/macos.yml/badge.svg)](https://github.com/charlievieth/fastwalk/actions/workflows/macos.yml) +[![Test fastwalk on Linux](https://github.com/charlievieth/fastwalk/actions/workflows/linux.yml/badge.svg)](https://github.com/charlievieth/fastwalk/actions/workflows/linux.yml) +[![Test fastwalk on Windows](https://github.com/charlievieth/fastwalk/actions/workflows/windows.yml/badge.svg)](https://github.com/charlievieth/fastwalk/actions/workflows/windows.yml) + +# fastwalk + +Fast parallel directory traversal for Golang. + +Package fastwalk provides a fast parallel version of [`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc) +that is \~2x faster on macOS, \~4x faster on Linux, \~6x faster on Windows, +allocates 50% less memory, and requires 25% fewer memory allocations. +Additionally, it is \~4-5x faster than [godirwalk](https://github.com/karrick/godirwalk) +across OSes. + +Inspired by and based off of [golang.org/x/tools/internal/fastwalk](https://pkg.go.dev/golang.org/x/tools@v0.1.9/internal/fastwalk). + +## Features + +* Fast: multiple goroutines stat the filesystem and call the + [`filepath.WalkDirFunc`](https://pkg.go.dev/io/fs#WalkDirFunc) callback concurrently +* Safe symbolic link traversal ([`Config.Follow`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Config)) +* Same behavior and callback signature as [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) +* Wrapper functions are provided to ignore duplicate files and directories: + [`IgnoreDuplicateFiles()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#IgnoreDuplicateFiles) + and + [`IgnoreDuplicateDirs()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#IgnoreDuplicateDirs) +* Extensively tested on macOS, Linux, and Windows + +## Usage + +Usage is the same as [`filepath.WalkDir`](https://pkg.go.dev/io/fs#WalkDirFunc), +but the [`walkFn`](https://pkg.go.dev/path/filepath@go1.17.7#WalkFunc) +argument to [`fastwalk.Walk`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk) +must be safe for concurrent use. + +Examples can be found in the [examples](./examples) directory. + + + +The below example is a very simple version of the POSIX +[find](https://pubs.opengroup.org/onlinepubs/007904975/utilities/find.html) utility: +```go +// fwfind is a an example program that is similar to POSIX find, +// but faster and worse (it's an example). +package main + +import ( + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/charlievieth/fastwalk" +) + +const usageMsg = `Usage: %[1]s [-L] [-name] [PATH...]: + +%[1]s is a poor replacement for the POSIX find utility + +` + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stdout, usageMsg, filepath.Base(os.Args[0])) + flag.PrintDefaults() + } + pattern := flag.String("name", "", "Pattern to match file names against.") + followLinks := flag.Bool("L", false, "Follow symbolic links") + flag.Parse() + + // If no paths are provided default to the current directory: "." + args := flag.Args() + if len(args) == 0 { + args = append(args, ".") + } + + // Follow links if the "-L" flag is provided + conf := fastwalk.Config{ + Follow: *followLinks, + } + + walkFn := func(path string, d fs.DirEntry, err error) error { + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", path, err) + return nil // returning the error stops iteration + } + if *pattern != "" { + if ok, err := filepath.Match(*pattern, d.Name()); !ok { + // invalid pattern (err != nil) or name does not match + return err + } + } + _, err = fmt.Println(path) + return err + } + for _, root := range args { + if err := fastwalk.Walk(&conf, root, walkFn); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", root, err) + os.Exit(1) + } + } +} +``` + +## Benchmarks + +Benchmarks were created using `go1.17.6` and can be generated with the `bench_comp` make target: +```sh +$ make bench_comp +``` + +### Darwin + +**Hardware:** +``` +goos: darwin +goarch: arm64 +cpu: Apple M1 Max +``` + +#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): +``` + filepath fastwalk delta +time/op 27.9ms ± 1% 13.0ms ± 1% -53.33% +alloc/op 4.33MB ± 0% 2.14MB ± 0% -50.55% +allocs/op 50.9k ± 0% 37.7k ± 0% -26.01% +``` + +#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): +``` + godirwalk fastwalk delta +time/op 58.5ms ± 3% 18.0ms ± 2% -69.30% +alloc/op 25.3MB ± 0% 2.1MB ± 0% -91.55% +allocs/op 57.6k ± 0% 37.7k ± 0% -34.59% +``` + +### Linux + +**Hardware:** +``` +goos: linux +goarch: amd64 +cpu: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz +drive: Samsung SSD 970 PRO 1TB +``` + +#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 10.1ms ± 2% 2.8ms ± 2% -72.83% +alloc/op 2.44MB ± 0% 1.70MB ± 0% -30.46% +allocs/op 47.2k ± 0% 36.9k ± 0% -21.80% +``` + +#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 13.7ms ±16% 2.8ms ± 2% -79.88% +alloc/op 7.48MB ± 0% 1.70MB ± 0% -77.34% +allocs/op 53.8k ± 0% 36.9k ± 0% -31.38% +``` + +### Windows + +**Hardware:** +``` +goos: windows +goarch: amd64 +pkg: github.com/charlievieth/fastwalk +cpu: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz +``` + +#### [`filepath.WalkDir`](https://pkg.go.dev/path/filepath@go1.17.7#WalkDir) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 88.0ms ± 1% 14.6ms ± 1% -83.47% +alloc/op 5.68MB ± 0% 6.76MB ± 0% +19.01% +allocs/op 69.6k ± 0% 90.4k ± 0% +29.87% +``` + +#### [`godirwalk.Walk()`](https://pkg.go.dev/github.com/karrick/godirwalk@v1.16.1#Walk) vs. [`fastwalk.Walk()`](https://pkg.go.dev/github.com/charlievieth/fastwalk#Walk): + +``` + filepath fastwalk delta +time/op 87.4ms ± 1% 14.6ms ± 1% -83.34% +alloc/op 6.14MB ± 0% 6.76MB ± 0% +10.24% +allocs/op 100k ± 0% 90k ± 0% -9.59% +``` + +## Darwin: getdirentries64 + +The `nogetdirentries` build tag can be used to prevent `fastwalk` from using +and linking to the non-public `__getdirentries64` syscall. This is required +if an app using `fastwalk` is to be distributed via Apple's App Store (see +https://github.com/golang/go/issues/30933 for more details). When using +`__getdirentries64` is disabled, `fastwalk` will use `readdir_r` instead, +which is what the Go standard library uses for +[`os.ReadDir`](https://pkg.go.dev/os#ReadDir) and is about \~10% slower than +`__getdirentries64` +([benchmarks](https://github.com/charlievieth/fastwalk/blob/2e6a1b8a1ce88e578279e6e631b2129f7144ec87/fastwalk_darwin_test.go#L19-L57)). + +Example of how to build and test that your program is not linked to `__getdirentries64`: +```sh +# NOTE: the following only applies to darwin (aka macOS) + +# Build binary that imports fastwalk without linking to __getdirentries64. +$ go build -tags nogetdirentries -o YOUR_BINARY +# Test that __getdirentries64 is not linked (this should print no output). +$ ! otool -dyld_info YOUR_BINARY | grep -F getdirentries64 +``` + +There is a also a script [scripts/links2getdirentries.bash](scripts/links2getdirentries.bash) +that can be used to check if a program binary links to getdirentries. diff --git a/vendor/github.com/charlievieth/fastwalk/adapters.go b/vendor/github.com/charlievieth/fastwalk/adapters.go new file mode 100644 index 000000000..c7d02e103 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/adapters.go @@ -0,0 +1,104 @@ +package fastwalk + +import ( + "io/fs" + "os" + "path/filepath" +) + +func isDir(path string, d fs.DirEntry) bool { + if d.IsDir() { + return true + } + if d.Type()&os.ModeSymlink != 0 { + if fi, err := StatDirEntry(path, d); err == nil { + return fi.IsDir() + } + } + return false +} + +// IgnoreDuplicateDirs wraps fs.WalkDirFunc walkFn to make it follow symbolic +// links and ignore duplicate directories (if a symlink points to a directory +// that has already been traversed it is skipped). The walkFn is called for +// for skipped directories, but the directory is not traversed (this is +// required for error handling). +// +// The Config.Follow setting has no effect on the behavior of Walk when +// this wrapper is used. +// +// In most use cases, the returned fs.WalkDirFunc should not be reused between +// in another call to Walk. If it is reused, any previously visited file will +// be skipped. +// +// NOTE: The order of traversal is undefined. Given an "example" directory +// like the one below where "dir" is a directory and "smydir1" and "smydir2" +// are links to it, only one of "dir", "smydir1", or "smydir2" will be +// traversed, but which one is undefined. +// +// example +// ├── dir +// ├── smydir1 -> dir +// └── smydir2 -> dir +func IgnoreDuplicateDirs(walkFn fs.WalkDirFunc) fs.WalkDirFunc { + filter := NewEntryFilter() + return func(path string, d fs.DirEntry, err error) error { + // Call walkFn before checking the entry filter so that we + // don't record directories that are skipped with SkipDir. + err = walkFn(path, d, err) + if err != nil { + if err != filepath.SkipDir && isDir(path, d) { + filter.Entry(path, d) + } + return err + } + if isDir(path, d) { + if filter.Entry(path, d) { + return filepath.SkipDir + } + if d.Type() == os.ModeSymlink { + return ErrTraverseLink + } + } + return nil + } +} + +// IgnoreDuplicateFiles wraps walkFn so that symlinks are followed and duplicate +// files are ignored. If a symlink resolves to a file that has already been +// visited it will be skipped. +// +// In most use cases, the returned fs.WalkDirFunc should not be reused between +// in another call to Walk. If it is reused, any previously visited file will +// be skipped. +// +// This can significantly slow Walk as os.Stat() is called for each path +// (on Windows, os.Stat() is only needed for symlinks). +func IgnoreDuplicateFiles(walkFn fs.WalkDirFunc) fs.WalkDirFunc { + filter := NewEntryFilter() + return func(path string, d fs.DirEntry, err error) error { + // Skip all duplicate files, directories, and links + if filter.Entry(path, d) { + if isDir(path, d) { + return filepath.SkipDir + } + return nil + } + err = walkFn(path, d, err) + if err == nil && d.Type() == os.ModeSymlink && isDir(path, d) { + err = ErrTraverseLink + } + return err + } +} + +// IgnorePermissionErrors wraps walkFn so that permission errors are ignored. +// The returned fs.WalkDirFunc may be reused. +func IgnorePermissionErrors(walkFn fs.WalkDirFunc) fs.WalkDirFunc { + return func(path string, d fs.DirEntry, err error) error { + if err != nil && os.IsPermission(err) { + return nil + } + return walkFn(path, d, err) + } +} diff --git a/vendor/github.com/charlievieth/fastwalk/dirent.go b/vendor/github.com/charlievieth/fastwalk/dirent.go new file mode 100644 index 000000000..2f259e2ea --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/dirent.go @@ -0,0 +1,46 @@ +package fastwalk + +import ( + "io/fs" + "os" + "sync" + "sync/atomic" + "unsafe" +) + +type fileInfo struct { + once sync.Once + fs.FileInfo + err error +} + +func loadFileInfo(pinfo **fileInfo) *fileInfo { + ptr := (*unsafe.Pointer)(unsafe.Pointer(pinfo)) + fi := (*fileInfo)(atomic.LoadPointer(ptr)) + if fi == nil { + fi = &fileInfo{} + if !atomic.CompareAndSwapPointer( + (*unsafe.Pointer)(unsafe.Pointer(pinfo)), // adrr + nil, // old + unsafe.Pointer(fi), // new + ) { + fi = (*fileInfo)(atomic.LoadPointer(ptr)) + } + } + return fi +} + +// StatDirEntry returns the fs.FileInfo for the file or subdirectory described +// by the entry. If the entry is a symbolic link, StatDirEntry returns the +// fs.FileInfo for the file the line references (os.Stat). +// If fs.DirEntry de is a fastwalk.DirEntry it's Stat() method is used and the +// returned fs.FileInfo may be a previously cached result. +func StatDirEntry(path string, de fs.DirEntry) (fs.FileInfo, error) { + if de.Type()&os.ModeSymlink == 0 { + return de.Info() + } + if d, ok := de.(DirEntry); ok { + return d.Stat() + } + return os.Stat(path) +} diff --git a/vendor/github.com/charlievieth/fastwalk/dirent_portable.go b/vendor/github.com/charlievieth/fastwalk/dirent_portable.go new file mode 100644 index 000000000..d1ce4d9ff --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/dirent_portable.go @@ -0,0 +1,38 @@ +//go:build appengine || solaris || (!linux && !darwin && !freebsd && !openbsd && !netbsd) +// +build appengine solaris !linux,!darwin,!freebsd,!openbsd,!netbsd + +package fastwalk + +import ( + "io/fs" + "os" +) + +type portableDirent struct { + fs.DirEntry + path string + stat *fileInfo +} + +// TODO: cache the result of Stat +func (d *portableDirent) Stat() (fs.FileInfo, error) { + if d.DirEntry.Type()&os.ModeSymlink == 0 { + return d.DirEntry.Info() + } + stat := loadFileInfo(&d.stat) + stat.once.Do(func() { + stat.FileInfo, stat.err = os.Stat(d.path) + }) + return stat.FileInfo, stat.err +} + +func newDirEntry(dirName string, info fs.DirEntry) fs.DirEntry { + return &portableDirent{ + DirEntry: info, + path: dirName + string(os.PathSeparator) + info.Name(), + } +} + +func fileInfoToDirEntry(dirname string, fi fs.FileInfo) fs.DirEntry { + return newDirEntry(dirname, fs.FileInfoToDirEntry(fi)) +} diff --git a/vendor/github.com/charlievieth/fastwalk/dirent_unix.go b/vendor/github.com/charlievieth/fastwalk/dirent_unix.go new file mode 100644 index 000000000..f9fd879ff --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/dirent_unix.go @@ -0,0 +1,63 @@ +//go:build (aix || darwin || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd) && !appengine && !solaris +// +build aix darwin dragonfly freebsd js,wasm linux netbsd openbsd +// +build !appengine +// +build !solaris + +package fastwalk + +import ( + "io/fs" + "os" +) + +type unixDirent struct { + parent string + name string + typ os.FileMode + info *fileInfo + stat *fileInfo +} + +func (d *unixDirent) Name() string { return d.name } +func (d *unixDirent) IsDir() bool { return d.typ.IsDir() } +func (d *unixDirent) Type() os.FileMode { return d.typ } + +func (d *unixDirent) Info() (fs.FileInfo, error) { + info := loadFileInfo(&d.info) + info.once.Do(func() { + info.FileInfo, info.err = os.Lstat(d.parent + "/" + d.name) + }) + return info.FileInfo, info.err +} + +func (d *unixDirent) Stat() (fs.FileInfo, error) { + if d.typ&os.ModeSymlink == 0 { + return d.Info() + } + stat := loadFileInfo(&d.stat) + stat.once.Do(func() { + stat.FileInfo, stat.err = os.Stat(d.parent + "/" + d.name) + }) + return stat.FileInfo, stat.err +} + +func newUnixDirent(parent, name string, typ os.FileMode) *unixDirent { + return &unixDirent{ + parent: parent, + name: name, + typ: typ, + } +} + +func fileInfoToDirEntry(dirname string, fi fs.FileInfo) fs.DirEntry { + info := &fileInfo{ + FileInfo: fi, + } + info.once.Do(func() {}) + return &unixDirent{ + parent: dirname, + name: fi.Name(), + typ: fi.Mode().Type(), + info: info, + } +} diff --git a/vendor/github.com/charlievieth/fastwalk/entry_filter_portable.go b/vendor/github.com/charlievieth/fastwalk/entry_filter_portable.go new file mode 100644 index 000000000..9ea1c6a23 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/entry_filter_portable.go @@ -0,0 +1,38 @@ +//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd && !windows && !solaris) +// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd,!windows,!solaris + +package fastwalk + +import ( + "io/fs" + "path/filepath" + "sync" +) + +type EntryFilter struct { + // we assume most files have not been seen so + // no need for a RWMutex + mu sync.Mutex + seen map[string]struct{} +} + +func (e *EntryFilter) Entry(path string, _ fs.DirEntry) bool { + name, err := filepath.EvalSymlinks(path) + if err != nil { + return false + } + e.mu.Lock() + if e.seen == nil { + e.seen = make(map[string]struct{}, 128) + } + _, ok := e.seen[name] + if !ok { + e.seen[name] = struct{}{} + } + e.mu.Unlock() + return ok +} + +func NewEntryFilter() *EntryFilter { + return &EntryFilter{seen: make(map[string]struct{}, 128)} +} diff --git a/vendor/github.com/charlievieth/fastwalk/entry_filter_unix.go b/vendor/github.com/charlievieth/fastwalk/entry_filter_unix.go new file mode 100644 index 000000000..277ee2d8b --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/entry_filter_unix.go @@ -0,0 +1,61 @@ +//go:build (linux || darwin || freebsd || openbsd || netbsd || !windows) && !appengine +// +build linux darwin freebsd openbsd netbsd !windows +// +build !appengine + +package fastwalk + +import ( + "io/fs" + "sync" + "syscall" +) + +type fileKey struct { + Dev uint64 + Ino uint64 +} + +type entryMap struct { + mu sync.Mutex + keys map[fileKey]struct{} +} + +// An EntryFilter keeps track of visited directory entries and can be used to +// detect and avoid symlink loops or processing the same file twice. +type EntryFilter struct { + // Use an array of 8 to reduce lock contention. The entryMap is + // picked via the inode number. We don't take the device number + // into account because: we don't expect to see many of them and + // uniformly spreading the load isn't terribly beneficial here. + ents [8]entryMap +} + +// NewEntryFilter returns a new EntryFilter +func NewEntryFilter() *EntryFilter { + return new(EntryFilter) +} + +func (e *EntryFilter) seen(dev, ino uint64) (seen bool) { + m := &e.ents[ino%uint64(len(e.ents))] + m.mu.Lock() + if _, seen = m.keys[fileKey{dev, ino}]; !seen { + if m.keys == nil { + m.keys = make(map[fileKey]struct{}) + } + m.keys[fileKey{dev, ino}] = struct{}{} + } + m.mu.Unlock() + return seen +} + +// TODO: this name is confusing and should be fixed + +// Entry returns if path and fs.DirEntry have been seen before. +func (e *EntryFilter) Entry(path string, de fs.DirEntry) (seen bool) { + fi, err := StatDirEntry(path, de) + if err != nil { + return true // treat errors as duplicate files + } + stat := fi.Sys().(*syscall.Stat_t) + return e.seen(uint64(stat.Dev), uint64(stat.Ino)) +} diff --git a/vendor/github.com/charlievieth/fastwalk/entry_filter_windows.go b/vendor/github.com/charlievieth/fastwalk/entry_filter_windows.go new file mode 100644 index 000000000..e60f78243 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/entry_filter_windows.go @@ -0,0 +1,157 @@ +//go:build windows && !appengine +// +build windows,!appengine + +package fastwalk + +import ( + "io/fs" + "os" + "path/filepath" + "sync" + "syscall" +) + +type fileKey struct { + VolumeSerialNumber uint32 + FileIndexHigh uint32 + FileIndexLow uint32 +} + +type EntryFilter struct { + mu sync.Mutex + seen map[fileKey]struct{} +} + +func NewEntryFilter() *EntryFilter { + return &EntryFilter{seen: make(map[fileKey]struct{}, 128)} +} + +func (e *EntryFilter) Entry(path string, _ fs.DirEntry) bool { + namep, err := syscall.UTF16PtrFromString(fixLongPath(path)) + if err != nil { + return false + } + + h, err := syscall.CreateFile(namep, 0, 0, nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return false + } + + var d syscall.ByHandleFileInformation + err = syscall.GetFileInformationByHandle(h, &d) + syscall.CloseHandle(h) + if err != nil { + return false + } + + key := fileKey{ + VolumeSerialNumber: d.VolumeSerialNumber, + FileIndexHigh: d.FileIndexHigh, + FileIndexLow: d.FileIndexLow, + } + + e.mu.Lock() + if e.seen == nil { + e.seen = make(map[fileKey]struct{}) + } + _, ok := e.seen[key] + if !ok { + e.seen[key] = struct{}{} + } + e.mu.Unlock() + + return ok +} + +func isAbs(path string) (b bool) { + v := filepath.VolumeName(path) + if v == "" { + return false + } + path = path[len(v):] + if path == "" { + return false + } + return os.IsPathSeparator(path[0]) +} + +// fixLongPath returns the extended-length (\\?\-prefixed) form of +// path when needed, in order to avoid the default 260 character file +// path limit imposed by Windows. If path is not easily converted to +// the extended-length form (for example, if path is a relative path +// or contains .. elements), or is short enough, fixLongPath returns +// path unmodified. +// +// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath +func fixLongPath(path string) string { + // Do nothing (and don't allocate) if the path is "short". + // Empirically (at least on the Windows Server 2013 builder), + // the kernel is arbitrarily okay with < 248 bytes. That + // matches what the docs above say: + // "When using an API to create a directory, the specified + // path cannot be so long that you cannot append an 8.3 file + // name (that is, the directory name cannot exceed MAX_PATH + // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. + // + // The MSDN docs appear to say that a normal path that is 248 bytes long + // will work; empirically the path must be less then 248 bytes long. + if len(path) < 248 { + // Don't fix. (This is how Go 1.7 and earlier worked, + // not automatically generating the \\?\ form) + return path + } + + // The extended form begins with \\?\, as in + // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt. + // The extended form disables evaluation of . and .. path + // elements and disables the interpretation of / as equivalent + // to \. The conversion here rewrites / to \ and elides + // . elements as well as trailing or duplicate separators. For + // simplicity it avoids the conversion entirely for relative + // paths or paths containing .. elements. For now, + // \\server\share paths are not converted to + // \\?\UNC\server\share paths because the rules for doing so + // are less well-specified. + if len(path) >= 2 && path[:2] == `\\` { + // Don't canonicalize UNC paths. + return path + } + if !isAbs(path) { + // Relative path + return path + } + + const prefix = `\\?` + + pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) + copy(pathbuf, prefix) + n := len(path) + r, w := 0, len(prefix) + for r < n { + switch { + case os.IsPathSeparator(path[r]): + // empty block + r++ + case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): + // /./ + r++ + case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): + // /../ is currently unhandled + return path + default: + pathbuf[w] = '\\' + w++ + for ; r < n && !os.IsPathSeparator(path[r]); r++ { + pathbuf[w] = path[r] + w++ + } + } + } + // A drive's root directory needs a trailing \ + if w == len(`\\?\c:`) { + pathbuf[w] = '\\' + w++ + } + return string(pathbuf[:w]) +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk.go b/vendor/github.com/charlievieth/fastwalk/fastwalk.go new file mode 100644 index 000000000..4e74acfed --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk.go @@ -0,0 +1,394 @@ +// Package fastwalk provides a faster version of filepath.Walk for file system +// scanning tools. +package fastwalk + +/* + * This code borrows heavily from golang.org/x/tools/internal/fastwalk + * and as such the Go license can be found in the go.LICENSE file and + * is reproduced below: + * + * Copyright (c) 2009 The Go Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "sync" +) + +// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the +// symlink named in the call may be traversed. +var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory") + +// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the +// callback should not be called for any other files in the current directory. +// Child directories will still be traversed. +var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory") + +// SkipDir is used as a return value from WalkDirFuncs to indicate that +// the directory named in the call is to be skipped. It is not returned +// as an error by any function. +var SkipDir = fs.SkipDir + +// DefaultNumWorkers returns the default number of worker goroutines to use in +// fastwalk.Walk and is the value of runtime.GOMAXPROCS(-1) clamped to a range +// of 4 to 32 except on Darwin where it is either 4 (8 cores or less) or 6 +// (more than 8 cores). This is because Walk / IO performance on Darwin +// degrades with more concurrency. +// +// The optimal number for your workload may be lower or higher. The results +// of BenchmarkFastWalkNumWorkers benchmark may be informative. +func DefaultNumWorkers() int { + numCPU := runtime.GOMAXPROCS(-1) + if numCPU < 4 { + return 4 + } + // Darwin IO performance on APFS slows with more workers. + // Stat performance is best around 2-4 and file IO is best + // around 4-6. More workers only benefit CPU intensive tasks. + if runtime.GOOS == "darwin" { + if numCPU <= 8 { + return 4 + } + return 6 + } + if numCPU > 32 { + return 32 + } + return numCPU +} + +// DefaultConfig is the default Config used when none is supplied. +var DefaultConfig = Config{ + Follow: false, + NumWorkers: DefaultNumWorkers(), +} + +type Config struct { + // TODO: do we want to pass a sentinel error to WalkFunc if + // a symlink loop is detected? + + // Follow symbolic links ignoring directories that would lead + // to infinite loops; that is, entering a previously visited + // directory that is an ancestor of the last file encountered. + // + // The sentinel error ErrTraverseLink is ignored when Follow + // is true (this to prevent users from defeating the loop + // detection logic), but SkipDir and ErrSkipFiles are still + // respected. + Follow bool + + // Number of parallel workers to use. If NumWorkers if ≤ 0 then + // the greater of runtime.NumCPU() or 4 is used. + NumWorkers int +} + +// A DirEntry extends the fs.DirEntry interface to add a Stat() method +// that returns the result of calling os.Stat() on the underlying file. +// The results of Info() and Stat() are cached. +// +// The fs.DirEntry argument passed to the fs.WalkDirFunc by Walk is +// always a DirEntry. The only exception is the root directory with +// with Walk is called. +type DirEntry interface { + fs.DirEntry + + // Stat returns the FileInfo for the file or subdirectory described + // by the entry. The returned FileInfo may be from the time of the + // original directory read or from the time of the call to Stat. + // If the entry denotes a symbolic link, Stat reports the information + // about the target itself, not the link. + Stat() (fs.FileInfo, error) +} + +// Walk is a faster implementation of filepath.Walk. +// +// filepath.Walk's design necessarily calls os.Lstat on each file, even if +// the caller needs less info. Many tools need only the type of each file. +// On some platforms, this information is provided directly by the readdir +// system call, avoiding the need to stat each file individually. +// fastwalk_unix.go contains a fork of the syscall routines. +// +// See golang.org/issue/16399 +// +// Walk walks the file tree rooted at root, calling walkFn for each file or +// directory in the tree, including root. +// +// If walkFn returns filepath.SkipDir, the directory is skipped. +// +// Unlike filepath.WalkDir: +// - File stat calls must be done by the user and should be done via +// the DirEntry argument to walkFn since it caches the results of +// Stat and Lstat. +// - The fs.DirEntry argument is always a fastwalk.DirEntry, which has +// a Stat() method that returns the result of calling os.Stat() on the +// file. The result of Stat() may be cached. +// - Multiple goroutines stat the filesystem concurrently. The provided +// walkFn must be safe for concurrent use. +// - Walk can follow symlinks if walkFn returns the ErrTraverseLink +// sentinel error. It is the walkFn's responsibility to prevent +// Walk from going into symlink cycles. +func Walk(conf *Config, root string, walkFn fs.WalkDirFunc) error { + if conf == nil { + dupe := DefaultConfig + conf = &dupe + } + fi, err := os.Lstat(root) + if err != nil { + return err + } + + // Make sure to wait for all workers to finish, otherwise + // walkFn could still be called after returning. This Wait call + // runs after close(e.donec) below. + var wg sync.WaitGroup + defer wg.Wait() + + numWorkers := conf.NumWorkers + if numWorkers <= 0 { + numWorkers = DefaultNumWorkers() + } + + w := &walker{ + fn: walkFn, + enqueuec: make(chan walkItem, numWorkers), // buffered for performance + workc: make(chan walkItem, numWorkers), // buffered for performance + donec: make(chan struct{}), + + // buffered for correctness & not leaking goroutines: + resc: make(chan error, numWorkers), + + follow: conf.Follow, + } + if w.follow { + if fi, err := os.Stat(root); err == nil { + w.ignoredDirs = append(w.ignoredDirs, fi) + } + } + + defer close(w.donec) + + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go w.doWork(&wg) + } + + root = cleanRootPath(root) + todo := []walkItem{{dir: root, info: fileInfoToDirEntry(filepath.Dir(root), fi)}} + out := 0 + for { + workc := w.workc + var workItem walkItem + if len(todo) == 0 { + workc = nil + } else { + workItem = todo[len(todo)-1] + } + select { + case workc <- workItem: + todo = todo[:len(todo)-1] + out++ + case it := <-w.enqueuec: + todo = append(todo, it) + case err := <-w.resc: + out-- + if err != nil { + return err + } + if out == 0 && len(todo) == 0 { + // It's safe to quit here, as long as the buffered + // enqueue channel isn't also readable, which might + // happen if the worker sends both another unit of + // work and its result before the other select was + // scheduled and both w.resc and w.enqueuec were + // readable. + select { + case it := <-w.enqueuec: + todo = append(todo, it) + default: + return nil + } + } + } + } +} + +// doWork reads directories as instructed (via workc) and runs the +// user's callback function. +func (w *walker) doWork(wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case <-w.donec: + return + case it := <-w.workc: + select { + case <-w.donec: + return + case w.resc <- w.walk(it.dir, it.info, !it.callbackDone): + } + } + } +} + +type walker struct { + fn fs.WalkDirFunc + + donec chan struct{} // closed on fastWalk's return + workc chan walkItem // to workers + enqueuec chan walkItem // from workers + resc chan error // from workers + + ignoredDirs []os.FileInfo + follow bool +} + +type walkItem struct { + dir string + info fs.DirEntry + callbackDone bool // callback already called; don't do it again +} + +func (w *walker) enqueue(it walkItem) { + select { + case w.enqueuec <- it: + case <-w.donec: + } +} + +func (w *walker) shouldSkipDir(fi os.FileInfo) bool { + for _, ignored := range w.ignoredDirs { + if os.SameFile(ignored, fi) { + return true + } + } + return false +} + +func (w *walker) shouldTraverse(path string, de fs.DirEntry) bool { + // TODO: do we need to use filepath.EvalSymlinks() here? + ts, err := StatDirEntry(path, de) + if err != nil { + return false + } + if !ts.IsDir() { + return false + } + if w.shouldSkipDir(ts) { + return false + } + for { + parent := filepath.Dir(path) + if parent == path { + return true + } + parentInfo, err := os.Stat(parent) + if err != nil { + return false + } + if os.SameFile(ts, parentInfo) { + return false + } + path = parent + } +} + +func joinPaths(dir, base string) string { + // Handle the case where the root path argument to Walk is "/" + // without this the returned path is prefixed with "//". + if os.PathSeparator == '/' && dir == "/" { + return dir + base + } + return dir + string(os.PathSeparator) + base +} + +func (w *walker) onDirEnt(dirName, baseName string, de fs.DirEntry) error { + joined := joinPaths(dirName, baseName) + typ := de.Type() + if typ == os.ModeDir { + w.enqueue(walkItem{dir: joined, info: de}) + return nil + } + + err := w.fn(joined, de, nil) + if typ == os.ModeSymlink { + if err == ErrTraverseLink { + if !w.follow { + // Set callbackDone so we don't call it twice for both the + // symlink-as-symlink and the symlink-as-directory later: + w.enqueue(walkItem{dir: joined, info: de, callbackDone: true}) + return nil + } + err = nil // Ignore ErrTraverseLink when Follow is true. + } + if err == filepath.SkipDir { + // Permit SkipDir on symlinks too. + return nil + } + if err == nil && w.follow && w.shouldTraverse(joined, de) { + // Traverse symlink + w.enqueue(walkItem{dir: joined, info: de, callbackDone: true}) + } + } + return err +} + +func (w *walker) walk(root string, info fs.DirEntry, runUserCallback bool) error { + if runUserCallback { + err := w.fn(root, info, nil) + if err == filepath.SkipDir { + return nil + } + if err != nil { + return err + } + } + + err := readDir(root, w.onDirEnt) + if err != nil { + // Second call, to report ReadDir error. + return w.fn(root, info, err) + } + return nil +} + +func cleanRootPath(root string) string { + for i := len(root) - 1; i >= 0; i-- { + if !os.IsPathSeparator(root[i]) { + return root[:i+1] + } + } + if root != "" { + return root[0:1] // root is all path separators ("//") + } + return root +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_getdirentries_darwin.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_getdirentries_darwin.go new file mode 100644 index 000000000..d1cdc6cd2 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_getdirentries_darwin.go @@ -0,0 +1,130 @@ +//go:build darwin && go1.13 && !appengine && !nogetdirentries +// +build darwin,go1.13,!appengine,!nogetdirentries + +package fastwalk + +import ( + "io/fs" + "os" + "sync" + "syscall" + "unsafe" +) + +const direntBufSize = 32 * 1024 + +var direntBufPool = sync.Pool{ + New: func() interface{} { + b := make([]byte, direntBufSize) + return &b + }, +} + +func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + fd, err := syscall.Open(dirName, syscall.O_RDONLY, 0) + if err != nil { + return &os.PathError{Op: "open", Path: dirName, Err: err} + } + defer syscall.Close(fd) + + p := direntBufPool.Get().(*[]byte) + defer direntBufPool.Put(p) + dbuf := *p + + var skipFiles bool + var basep uintptr + for { + length, err := getdirentries(fd, dbuf, &basep) + if err != nil { + return &os.PathError{Op: "getdirentries64", Path: dirName, Err: err} + } + if length == 0 { + break + } + buf := dbuf[:length] + + for i := 0; len(buf) > 0; i++ { + reclen, ok := direntReclen(buf) + if !ok || reclen > uint64(len(buf)) { + break + } + rec := buf[:reclen] + buf = buf[reclen:] + typ := direntType(rec) + if skipFiles && typ.IsRegular() { + continue + } + const namoff = uint64(unsafe.Offsetof(syscall.Dirent{}.Name)) + namlen, ok := direntNamlen(rec) + if !ok || namoff+namlen > uint64(len(rec)) { + break + } + name := rec[namoff : namoff+namlen] + for i, c := range name { + if c == 0 { + name = name[:i] + break + } + } + if string(name) == "." || string(name) == ".." { + continue + } + nm := string(name) + if err := fn(dirName, nm, newUnixDirent(dirName, nm, typ)); err != nil { + if err != ErrSkipFiles { + return err + } + skipFiles = true + } + } + } + + return nil +} + +// readInt returns the size-bytes unsigned integer in native byte order at offset off. +func readInt(b []byte, off, size uintptr) (uint64, bool) { + if len(b) >= int(off+size) { + p := b[off:] + _ = p[1] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(p[0]) | uint64(p[1])<<8, true + } + return 0, false +} + +// Statically assert that the size of Reclen and Namlen is 2. +var _ = ([2]int{})[unsafe.Sizeof(syscall.Dirent{}.Reclen)-1] +var _ = ([2]int{})[unsafe.Sizeof(syscall.Dirent{}.Namlen)-1] + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Namlen), unsafe.Sizeof(syscall.Dirent{}.Namlen)) +} + +func direntType(buf []byte) os.FileMode { + off := unsafe.Offsetof(syscall.Dirent{}.Type) + if off >= uintptr(len(buf)) { + return ^os.FileMode(0) // unknown + } + typ := buf[off] + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_nogetdirentries_darwin.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_nogetdirentries_darwin.go new file mode 100644 index 000000000..cbd24dd9c --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_nogetdirentries_darwin.go @@ -0,0 +1,84 @@ +//go:build darwin && go1.13 && !appengine && nogetdirentries +// +build darwin,go1.13,!appengine,nogetdirentries + +package fastwalk + +import ( + "io/fs" + "os" + "syscall" + "unsafe" +) + +//sys closedir(dir uintptr) (err error) +//sys readdir_r(dir uintptr, entry *Dirent, result **Dirent) (res Errno) + +func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + fd, err := opendir(dirName) + if err != nil { + return &os.PathError{Op: "opendir", Path: dirName, Err: err} + } + defer closedir(fd) //nolint:errcheck + + skipFiles := false + var dirent syscall.Dirent + var entptr *syscall.Dirent + for { + if errno := readdir_r(fd, &dirent, &entptr); errno != 0 { + if errno == syscall.EINTR { + continue + } + return &os.PathError{Op: "readdir", Path: dirName, Err: errno} + } + if entptr == nil { // EOF + break + } + if dirent.Ino == 0 { + continue + } + typ := dtToType(dirent.Type) + if skipFiles && typ.IsRegular() { + continue + } + name := (*[len(syscall.Dirent{}.Name)]byte)(unsafe.Pointer(&dirent.Name))[:] + for i, c := range name { + if c == 0 { + name = name[:i] + break + } + } + // Check for useless names before allocating a string. + if string(name) == "." || string(name) == ".." { + continue + } + nm := string(name) + if err := fn(dirName, nm, newUnixDirent(dirName, nm, typ)); err != nil { + if err != ErrSkipFiles { + return err + } + skipFiles = true + } + } + + return nil +} + +func dtToType(typ uint8) os.FileMode { + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_portable.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_portable.go new file mode 100644 index 000000000..dee35f6f4 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_portable.go @@ -0,0 +1,42 @@ +//go:build appengine || solaris || (!linux && !darwin && !freebsd && !openbsd && !netbsd) +// +build appengine solaris !linux,!darwin,!freebsd,!openbsd,!netbsd + +package fastwalk + +import ( + "io/fs" + "os" +) + +// readDir calls fn for each directory entry in dirName. +// It does not descend into directories or follow symlinks. +// If fn returns a non-nil error, readDir returns with that error +// immediately. +func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + f, err := os.Open(dirName) + if err != nil { + return err + } + des, readErr := f.ReadDir(-1) + f.Close() + if readErr != nil && len(des) == 0 { + return readErr + } + + var skipFiles bool + for _, d := range des { + if skipFiles && d.Type().IsRegular() { + continue + } + // Need to use FileMode.Type().Type() for fs.DirEntry + e := newDirEntry(dirName, d) + if err := fn(dirName, d.Name(), e); err != nil { + if err != ErrSkipFiles { + return err + } + skipFiles = true + } + } + + return readErr +} diff --git a/vendor/github.com/charlievieth/fastwalk/fastwalk_unix.go b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix.go new file mode 100644 index 000000000..2ef216453 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/fastwalk_unix.go @@ -0,0 +1,105 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (linux || freebsd || openbsd || netbsd) && !appengine +// +build linux freebsd openbsd netbsd +// +build !appengine + +package fastwalk + +import ( + "io/fs" + "os" + "syscall" + + "github.com/charlievieth/fastwalk/internal/dirent" +) + +// More than 5760 to work around https://golang.org/issue/24015. +const blockSize = 8192 + +// unknownFileMode is a sentinel (and bogus) os.FileMode +// value used to represent a syscall.DT_UNKNOWN Dirent.Type. +const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice + +func readDir(dirName string, fn func(dirName, entName string, de fs.DirEntry) error) error { + fd, err := open(dirName, 0, 0) + if err != nil { + return &os.PathError{Op: "open", Path: dirName, Err: err} + } + defer syscall.Close(fd) + + // The buffer must be at least a block long. + buf := make([]byte, blockSize) // stack-allocated; doesn't escape + bufp := 0 // starting read position in buf + nbuf := 0 // end valid data in buf + skipFiles := false + for { + if bufp >= nbuf { + bufp = 0 + nbuf, err = readDirent(fd, buf) + if err != nil { + return os.NewSyscallError("readdirent", err) + } + if nbuf <= 0 { + return nil + } + } + consumed, name, typ := dirent.Parse(buf[bufp:nbuf]) + bufp += consumed + + if name == "" || name == "." || name == ".." { + continue + } + // Fallback for filesystems (like old XFS) that don't + // support Dirent.Type and have DT_UNKNOWN (0) there + // instead. + if typ == unknownFileMode { + fi, err := os.Lstat(dirName + "/" + name) + if err != nil { + // It got deleted in the meantime. + if os.IsNotExist(err) { + continue + } + return err + } + typ = fi.Mode() & os.ModeType + } + if skipFiles && typ.IsRegular() { + continue + } + de := newUnixDirent(dirName, name, typ) + if err := fn(dirName, name, de); err != nil { + if err == ErrSkipFiles { + skipFiles = true + continue + } + return err + } + } +} + +// According to https://golang.org/doc/go1.14#runtime +// A consequence of the implementation of preemption is that on Unix systems, including Linux and macOS +// systems, programs built with Go 1.14 will receive more signals than programs built with earlier releases. +// +// This causes syscall.Open and syscall.ReadDirent sometimes fail with EINTR errors. +// We need to retry in this case. +func open(path string, mode int, perm uint32) (fd int, err error) { + for { + fd, err := syscall.Open(path, mode, perm) + if err != syscall.EINTR { + return fd, err + } + } +} + +func readDirent(fd int, buf []byte) (n int, err error) { + for { + nbuf, err := syscall.ReadDirent(fd, buf) + if err != syscall.EINTR { + return nbuf, err + } + } +} diff --git a/vendor/github.com/charlievieth/fastwalk/go.LICENSE b/vendor/github.com/charlievieth/fastwalk/go.LICENSE new file mode 100644 index 000000000..6a66aea5e --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/go.LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent.go new file mode 100644 index 000000000..394a91da0 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent.go @@ -0,0 +1,109 @@ +//go:build aix || dragonfly || freebsd || (js && wasm) || wasip1 || linux || netbsd || openbsd || solaris + +package dirent + +import ( + "os" + "runtime" + "syscall" + "unsafe" +) + +// readInt returns the size-bytes unsigned integer in native byte order at offset off. +func readInt(b []byte, off, size uintptr) (u uint64, ok bool) { + if len(b) < int(off+size) { + return 0, false + } + if isBigEndian { + return readIntBE(b[off:], size), true + } + return readIntLE(b[off:], size), true +} + +func readIntBE(b []byte, size uintptr) uint64 { + switch size { + case 1: + return uint64(b[0]) + case 2: + _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[1]) | uint64(b[0])<<8 + case 4: + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24 + case 8: + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 + default: + panic("syscall: readInt with unsupported size") + } +} + +func readIntLE(b []byte, size uintptr) uint64 { + switch size { + case 1: + return uint64(b[0]) + case 2: + _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[0]) | uint64(b[1])<<8 + case 4: + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 + case 8: + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + default: + panic("syscall: readInt with unsupported size") + } +} + +const InvalidMode = os.FileMode(1<<32 - 1) + +func Parse(buf []byte) (consumed int, name string, typ os.FileMode) { + + reclen, ok := direntReclen(buf) + if !ok || reclen > uint64(len(buf)) { + // WARN: this is a hard error because we consumed 0 bytes + // and not stopping here could lead to an infinite loop. + return 0, "", InvalidMode + } + consumed = int(reclen) + rec := buf[:reclen] + + ino, ok := direntIno(rec) + if !ok { + return consumed, "", InvalidMode + } + // When building to wasip1, the host runtime might be running on Windows + // or might expose a remote file system which does not have the concept + // of inodes. Therefore, we cannot make the assumption that it is safe + // to skip entries with zero inodes. + if ino == 0 && runtime.GOOS != "wasip1" { + return consumed, "", InvalidMode + } + + typ = direntType(buf) + + const namoff = uint64(unsafe.Offsetof(syscall.Dirent{}.Name)) + namlen, ok := direntNamlen(rec) + if !ok || namoff+namlen > uint64(len(rec)) { + return consumed, "", InvalidMode + } + namebuf := rec[namoff : namoff+namlen] + for i, c := range namebuf { + if c == 0 { + namebuf = namebuf[:i] + break + } + } + // Check for useless names before allocating a string. + if string(namebuf) == "." { + name = "." + } else if string(namebuf) == ".." { + name = ".." + } else { + name = string(namebuf) + } + return consumed, name, typ +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_aix.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_aix.go new file mode 100644 index 000000000..15d84355c --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_aix.go @@ -0,0 +1,29 @@ +//go:build aix + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Ino), unsafe.Sizeof(syscall.Dirent{}.Ino)) +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + reclen, ok := direntReclen(buf) + if !ok { + return 0, false + } + return reclen - uint64(unsafe.Offsetof(syscall.Dirent{}.Name)), true +} + +func direntType(buf []byte) os.FileMode { + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_dragonfly.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_dragonfly.go new file mode 100644 index 000000000..f9065f57b --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_dragonfly.go @@ -0,0 +1,54 @@ +//go:build dragonfly + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Fileno), unsafe.Sizeof(syscall.Dirent{}.Fileno)) +} + +func direntReclen(buf []byte) (uint64, bool) { + namlen, ok := direntNamlen(buf) + if !ok { + return 0, false + } + return (16 + namlen + 1 + 7) &^ 7, true +} + +func direntNamlen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Namlen), unsafe.Sizeof(syscall.Dirent{}.Namlen)) +} + +func direntType(buf []byte) os.FileMode { + off := unsafe.Offsetof(syscall.Dirent{}.Type) + if off >= uintptr(len(buf)) { + return ^os.FileMode(0) // unknown + } + typ := buf[off] + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DBF: + // DT_DBF is "database record file". + // fillFileStatFromSys treats as regular file. + return 0 + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_freebsd.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_freebsd.go new file mode 100644 index 000000000..215251820 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_freebsd.go @@ -0,0 +1,46 @@ +//go:build freebsd + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Fileno), unsafe.Sizeof(syscall.Dirent{}.Fileno)) +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Namlen), unsafe.Sizeof(syscall.Dirent{}.Namlen)) +} + +func direntType(buf []byte) os.FileMode { + off := unsafe.Offsetof(syscall.Dirent{}.Type) + if off >= uintptr(len(buf)) { + return ^os.FileMode(0) // unknown + } + typ := buf[off] + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_js.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_js.go new file mode 100644 index 000000000..18bef93dd --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_js.go @@ -0,0 +1,27 @@ +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return 1, true +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + reclen, ok := direntReclen(buf) + if !ok { + return 0, false + } + return reclen - uint64(unsafe.Offsetof(syscall.Dirent{}.Name)), true +} + +func direntType(buf []byte) os.FileMode { + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_linux.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_linux.go new file mode 100644 index 000000000..c2f6865bc --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_linux.go @@ -0,0 +1,50 @@ +//go:build linux + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Ino), unsafe.Sizeof(syscall.Dirent{}.Ino)) +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + reclen, ok := direntReclen(buf) + if !ok { + return 0, false + } + return reclen - uint64(unsafe.Offsetof(syscall.Dirent{}.Name)), true +} + +func direntType(buf []byte) os.FileMode { + off := unsafe.Offsetof(syscall.Dirent{}.Type) + if off >= uintptr(len(buf)) { + return ^os.FileMode(0) // unknown + } + typ := buf[off] + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_netbsd.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_netbsd.go new file mode 100644 index 000000000..99dbe7bab --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_netbsd.go @@ -0,0 +1,46 @@ +//go:build netbsd + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Fileno), unsafe.Sizeof(syscall.Dirent{}.Fileno)) +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Namlen), unsafe.Sizeof(syscall.Dirent{}.Namlen)) +} + +func direntType(buf []byte) os.FileMode { + off := unsafe.Offsetof(syscall.Dirent{}.Type) + if off >= uintptr(len(buf)) { + return ^os.FileMode(0) // unknown + } + typ := buf[off] + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_openbsd.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_openbsd.go new file mode 100644 index 000000000..ffd48b5a6 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_openbsd.go @@ -0,0 +1,46 @@ +//go:build openbsd + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Fileno), unsafe.Sizeof(syscall.Dirent{}.Fileno)) +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Namlen), unsafe.Sizeof(syscall.Dirent{}.Namlen)) +} + +func direntType(buf []byte) os.FileMode { + off := unsafe.Offsetof(syscall.Dirent{}.Type) + if off >= uintptr(len(buf)) { + return ^os.FileMode(0) // unknown + } + typ := buf[off] + switch typ { + case syscall.DT_BLK: + return os.ModeDevice + case syscall.DT_CHR: + return os.ModeDevice | os.ModeCharDevice + case syscall.DT_DIR: + return os.ModeDir + case syscall.DT_FIFO: + return os.ModeNamedPipe + case syscall.DT_LNK: + return os.ModeSymlink + case syscall.DT_REG: + return 0 + case syscall.DT_SOCK: + return os.ModeSocket + } + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_solaris.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_solaris.go new file mode 100644 index 000000000..8ef23c1ad --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_solaris.go @@ -0,0 +1,29 @@ +//go:build solaris + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Ino), unsafe.Sizeof(syscall.Dirent{}.Ino)) +} + +func direntReclen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Reclen), unsafe.Sizeof(syscall.Dirent{}.Reclen)) +} + +func direntNamlen(buf []byte) (uint64, bool) { + reclen, ok := direntReclen(buf) + if !ok { + return 0, false + } + return reclen - uint64(unsafe.Offsetof(syscall.Dirent{}.Name)), true +} + +func direntType(buf []byte) os.FileMode { + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_wasip1.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_wasip1.go new file mode 100644 index 000000000..5e6c4db3a --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/dirent_wasip1.go @@ -0,0 +1,53 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build wasip1 + +package dirent + +import ( + "os" + "syscall" + "unsafe" +) + +// https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md#-dirent-record +const sizeOfDirent = 24 + +func direntIno(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Ino), unsafe.Sizeof(syscall.Dirent{}.Ino)) +} + +func direntReclen(buf []byte) (uint64, bool) { + namelen, ok := direntNamlen(buf) + return sizeOfDirent + namelen, ok +} + +func direntNamlen(buf []byte) (uint64, bool) { + return readInt(buf, unsafe.Offsetof(syscall.Dirent{}.Namlen), unsafe.Sizeof(syscall.Dirent{}.Namlen)) +} + +func direntType(buf []byte) os.FileMode { + off := unsafe.Offsetof(syscall.Dirent{}.Type) + if off >= uintptr(len(buf)) { + return ^os.FileMode(0) // unknown + } + switch syscall.Filetype(buf[off]) { + case syscall.FILETYPE_BLOCK_DEVICE: + return os.ModeDevice + case syscall.FILETYPE_CHARACTER_DEVICE: + return os.ModeDevice | os.ModeCharDevice + case syscall.FILETYPE_DIRECTORY: + return os.ModeDir + case syscall.FILETYPE_REGULAR_FILE: + return 0 + case syscall.FILETYPE_SOCKET_DGRAM: + return os.ModeSocket + case syscall.FILETYPE_SOCKET_STREAM: + return os.ModeSocket + case syscall.FILETYPE_SYMBOLIC_LINK: + return os.ModeSymlink + } + return ^os.FileMode(0) // unknown +} diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/doc.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/doc.go new file mode 100644 index 000000000..4ff436d6d --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/doc.go @@ -0,0 +1,2 @@ +// Package dirent parses raw syscall dirents +package dirent diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/endian_big.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/endian_big.go new file mode 100644 index 000000000..c64da9b65 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/endian_big.go @@ -0,0 +1,9 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +//go:build armbe || arm64be || m68k || mips || mips64 || mips64p32 || ppc || ppc64 || s390 || s390x || shbe || sparc || sparc64 + +package dirent + +const isBigEndian = true diff --git a/vendor/github.com/charlievieth/fastwalk/internal/dirent/endian_little.go b/vendor/github.com/charlievieth/fastwalk/internal/dirent/endian_little.go new file mode 100644 index 000000000..1d4e69a1a --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/internal/dirent/endian_little.go @@ -0,0 +1,9 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +//go:build 386 || amd64 || amd64p32 || alpha || arm || arm64 || loong64 || mipsle || mips64le || mips64p32le || nios2 || ppc64le || riscv || riscv64 || sh || wasm + +package dirent + +const isBigEndian = false diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin.go b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin.go new file mode 100644 index 000000000..29be85f99 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin.go @@ -0,0 +1,32 @@ +//go:build darwin && go1.12 +// +build darwin,go1.12 + +package fastwalk + +import "syscall" + +// Copied from syscall/syscall_unix.go + +// Do the interface allocations only once for common +// Errno values. +var ( + errEAGAIN error = syscall.EAGAIN + errEINVAL error = syscall.EINVAL + errENOENT error = syscall.ENOENT +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case syscall.EAGAIN: + return errEAGAIN + case syscall.EINVAL: + return errEINVAL + case syscall.ENOENT: + return errENOENT + } + return e +} diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s new file mode 100644 index 000000000..ccf2ff6f8 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_amd64.1_13.s @@ -0,0 +1,28 @@ +//go:build go1.13 +// +build go1.13 + +#include "textflag.h" + +TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_closedir(SB) + +GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB) + +TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readdir_r(SB) + +GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB) + +TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_opendir(SB) + +GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB) + +TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc___getdirentries64(SB) + +GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8 +DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB) diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s new file mode 100644 index 000000000..ccf2ff6f8 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_darwin_arm64.1_13.s @@ -0,0 +1,28 @@ +//go:build go1.13 +// +build go1.13 + +#include "textflag.h" + +TEXT libc_closedir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_closedir(SB) + +GLOBL ·libc_closedir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_closedir_trampoline_addr(SB)/8, $libc_closedir_trampoline<>(SB) + +TEXT libc_readdir_r_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readdir_r(SB) + +GLOBL ·libc_readdir_r_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readdir_r_trampoline_addr(SB)/8, $libc_readdir_r_trampoline<>(SB) + +TEXT libc_opendir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_opendir(SB) + +GLOBL ·libc_opendir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_opendir_trampoline_addr(SB)/8, $libc_opendir_trampoline<>(SB) + +TEXT libc___getdirentries64_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc___getdirentries64(SB) + +GLOBL ·libc___getdirentries64_trampoline_addr(SB), RODATA, $8 +DATA ·libc___getdirentries64_trampoline_addr(SB)/8, $libc___getdirentries64_trampoline<>(SB) diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go b/vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go new file mode 100644 index 000000000..9e5b0cd8f --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_getdirentries_darwin.go @@ -0,0 +1,40 @@ +//go:build !nogetdirentries && darwin && go1.12 +// +build !nogetdirentries,darwin,go1.12 + +package fastwalk + +import ( + "syscall" + "unsafe" +) + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:linkname syscall_syscall6 syscall.syscall6 + +// Single-word zero for use when we need a valid pointer to 0 bytes. +var _zero uintptr + +func getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) { + var _p0 unsafe.Pointer + if len(buf) > 0 { + _p0 = unsafe.Pointer(&buf[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + r0, _, e1 := syscall_syscall6(libc___getdirentries64_trampoline_addr, + uintptr(fd), uintptr(_p0), uintptr(len(buf)), uintptr(unsafe.Pointer(basep)), + 0, 0) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } else if n < 0 { + err = errnoErr(syscall.EINVAL) + } + return +} + +var libc___getdirentries64_trampoline_addr uintptr + +//go:cgo_import_dynamic libc___getdirentries64 __getdirentries64 "/usr/lib/libSystem.B.dylib" diff --git a/vendor/github.com/charlievieth/fastwalk/zsyscall_nogetdirentries_darwin.go b/vendor/github.com/charlievieth/fastwalk/zsyscall_nogetdirentries_darwin.go new file mode 100644 index 000000000..289486375 --- /dev/null +++ b/vendor/github.com/charlievieth/fastwalk/zsyscall_nogetdirentries_darwin.go @@ -0,0 +1,55 @@ +//go:build nogetdirentries && darwin && go1.12 +// +build nogetdirentries,darwin,go1.12 + +package fastwalk + +import ( + "syscall" + "unsafe" +) + +// Implemented in the runtime package (runtime/sys_darwin.go) +func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) +func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) + +//go:linkname syscall_syscall syscall.syscall +//go:linkname syscall_syscallPtr syscall.syscallPtr + +func closedir(dir uintptr) (err error) { + _, _, e1 := syscall_syscall(libc_closedir_trampoline_addr, dir, 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_closedir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_closedir closedir "/usr/lib/libSystem.B.dylib" + +func readdir_r(dir uintptr, entry *syscall.Dirent, result **syscall.Dirent) syscall.Errno { + res, _, _ := syscall_syscall(libc_readdir_r_trampoline_addr, dir, uintptr(unsafe.Pointer(entry)), uintptr(unsafe.Pointer(result))) + return syscall.Errno(res) +} + +var libc_readdir_r_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readdir_r readdir_r "/usr/lib/libSystem.B.dylib" + +func opendir(path string) (dir uintptr, err error) { + // We implent opendir so that we don't have to open a file, duplicate + // it's FD, then call fdopendir with it. + p, err := syscall.BytePtrFromString(path) + if err != nil { + return 0, err + } + r0, _, e1 := syscall_syscallPtr(libc_opendir_trampoline_addr, uintptr(unsafe.Pointer(p)), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return r0, err +} + +var libc_opendir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_opendir opendir "/usr/lib/libSystem.B.dylib" diff --git a/vendor/modules.txt b/vendor/modules.txt index 5a9152ff4..0d246af5b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -58,6 +58,10 @@ github.com/beorn7/perks/quantile # github.com/cespare/xxhash/v2 v2.3.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 +# github.com/charlievieth/fastwalk v1.0.3 +## explicit; go 1.18 +github.com/charlievieth/fastwalk +github.com/charlievieth/fastwalk/internal/dirent # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew