diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go
index ef38e92c9e..b4f55a79bc 100644
--- a/pkg/rke2/rke2.go
+++ b/pkg/rke2/rke2.go
@@ -140,6 +140,10 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error {
 	}
 	executor.Set(ex)
 
+	// Clear data directories that are no longer in use
+	// Errors from this should not prevent the cluster from starting
+	_ = cleanupDataDir(dataDir)
+
 	// check for force restart file
 	var forceRestart bool
 	if _, err := os.Stat(ForceRestartFile(dataDir)); err != nil {
diff --git a/pkg/rke2/rke2_linux.go b/pkg/rke2/rke2_linux.go
index 1b7b59c41c..f238675690 100644
--- a/pkg/rke2/rke2_linux.go
+++ b/pkg/rke2/rke2_linux.go
@@ -7,10 +7,13 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/fs"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -507,3 +510,52 @@ func hostnameFromMetadataEndpoint(ctx context.Context) string {
 
 	return strings.TrimSpace(string(b))
 }
+
+func cleanupDataDir(dataDir string) error {
+	paths, err := getProcessExecutablePaths()
+	activePaths := []string{}
+	if err != nil {
+		return err
+	}
+	// Ensures only unique path within the data directory are
+	// captured within the array
+	for _, path := range paths {
+		if !strings.HasPrefix(path, dataDir) {
+			continue
+		}
+		arr := strings.Split(path, "/")
+		rkePath := strings.Join(arr[0:7], "/")
+		if !slices.Contains(activePaths, rkePath) {
+			activePaths = append(activePaths, rkePath)
+		}
+	}
+
+	// Ensures file is a directory, has rke2 within the name, and is not an active path
+	// if the function is unable to clear out the diretory, the error is ignored.
+	err = filepath.WalkDir(path, func(path string, info fs.DirEntry, err error) error {
+		if !info.IsDir() || !string.Contains(path, "rke2") || slices.Contains(activePaths, path) {
+			continue
+		}
+		_ := os.RemoveAll(path)
+	})
+
+	return err
+}
+
+func getProcessExecutablePaths() ([]string, error) {
+	path := "/proc"
+	executables := []string{}
+	err := filepath.WalkDir(path, func(path string, dir fs.DirEntry, err error) error {
+		if !dir.IsDir() {
+			return nil
+		}
+		exePath := filepath.Join(path, "exe")
+		exeLinkPath, err := os.Readlink(exePath)
+		if err != nil {
+			return nil
+		}
+		executables = append(executables, exeLinkPath)
+		return nil
+	})
+	return executables, err
+}
diff --git a/pkg/rke2/rke2_windows.go b/pkg/rke2/rke2_windows.go
index 5b9d095e6c..ec78d54600 100644
--- a/pkg/rke2/rke2_windows.go
+++ b/pkg/rke2/rke2_windows.go
@@ -72,3 +72,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*pebinaryexecuto
 		CNI:             "",
 	}, nil
 }
+
+func cleanupDataDir() error {
+	return nil
+}