Use the fastwalk passage to do a parallel walk of the directory

structure.
gadget-inc · Jan 8, 2025 · 60963c1 · 60963c1
1 parent 771bcb1
commit 60963c1
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 2 deletions.
diff --git a/go.mod b/go.mod
@@ -40,6 +40,7 @@ require (
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/charlievieth/fastwalk v1.0.9 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect

diff --git a/go.sum b/go.sum
@@ -55,6 +55,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charlievieth/fastwalk v1.0.9 h1:Odb92AfoReO3oFBfDGT5J+nwgzQPF/gWAw6E6/lkor0=
+github.com/charlievieth/fastwalk v1.0.9/go.mod h1:yGy1zbxog41ZVMcKA/i8ojXLFsuayX5VvwhQVoj9PBI=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=

diff --git a/internal/files/writer.go b/internal/files/writer.go
@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/charlievieth/fastwalk"
 	"github.com/gadget-inc/dateilager/internal/db"
 	"github.com/gadget-inc/dateilager/internal/pb"
 	"github.com/gobwas/glob"
@@ -211,7 +212,10 @@ func HardlinkDir(olddir, newdir string) error {
 		return fmt.Errorf("cannot create new root dir %v: %w", olddir, err)
 	}
 
-	return filepath.WalkDir(olddir, func(oldpath string, d os.DirEntry, err error) error {
+	fastwalkConf := fastwalk.DefaultConfig.Copy()
+	//fastwalkConf.Sort = fastwalk.SortDirsFirst
+
+	return fastwalk.Walk(fastwalkConf, olddir, func(oldpath string, d os.DirEntry, err error) error {
 		if err != nil {
 			return fmt.Errorf("failed to walk dir: %v, %w", oldpath, err)
 		}

diff --git a/test/hardlink_dir_benchmark_test.go b/test/hardlink_dir_benchmark_test.go
@@ -27,6 +27,7 @@ func TestHardlinkDir(t *testing.T) {
 }
 
 func BenchmarkHardlinkDir(b *testing.B) {
+
 	wd, err := os.Getwd()
 	if err != nil {
 		b.Error(err)
@@ -36,10 +37,19 @@ func BenchmarkHardlinkDir(b *testing.B) {
 	tmpDir := emptyTmpDir(b)
 	defer os.RemoveAll(tmpDir)
 
+	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		err := files.HardlinkDir(bigDir, path.Join(tmpDir, "node_modules", fmt.Sprintf("%d", n)))
+		copyDir := path.Join(tmpDir, "node_modules", fmt.Sprintf("%d", n))
+		err := files.HardlinkDir(bigDir, copyDir)
+		b.StopTimer()
+		if err != nil {
+			b.Error(err)
+		}
+
+		err = CompareDirectories(bigDir, copyDir)
 		if err != nil {
 			b.Error(err)
 		}
+		b.StartTimer()
 	}
 }