[release-1.32] Backports for 2025-01

go.mod

@@ -17,7 +17,7 @@ replace (
 	github.com/google/cadvisor => github.com/k3s-io/cadvisor v0.51.0-k3s1
 	github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.12.0
 	github.com/open-policy-agent/opa => github.com/open-policy-agent/opa v0.59.0 // github.com/Microsoft/hcsshim using bad version v0.42.2
-	github.com/opencontainers/runc => github.com/k3s-io/runc v1.2.1-k3s1
+	github.com/opencontainers/runc => github.com/k3s-io/runc v1.2.4-k3s1
 	github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.11.0
 	github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.19.1
 	github.com/prometheus/common => github.com/prometheus/common v0.55.0
@@ -98,6 +98,7 @@ require (
 	github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5
 	github.com/go-test/deep v1.0.7
 	github.com/google/cadvisor v0.51.0
+	github.com/google/go-containerregistry v0.20.2
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.1
@@ -147,7 +148,7 @@ require (
 	golang.org/x/mod v0.22.0
 	golang.org/x/net v0.31.0
 	golang.org/x/sync v0.9.0
-	golang.org/x/sys v0.27.0
+	golang.org/x/sys v0.28.0
 	google.golang.org/grpc v1.68.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.32.0
@@ -234,7 +235,7 @@ require (
 	github.com/coreos/go-oidc v2.2.1+incompatible // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/cyphar/filepath-securejoin v0.3.4 // indirect
+	github.com/cyphar/filepath-securejoin v0.3.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect

go.sum

@@ -468,8 +468,8 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8=
-github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM=
+github.com/cyphar/filepath-securejoin v0.3.5 h1:L81NHjquoQmcPgXcttUS9qTSR/+bXry6pbSINQGpjj4=
+github.com/cyphar/filepath-securejoin v0.3.5/go.mod h1:edhVd3c6OXKjUmSrVa/tGJRS9joFTxlslFCAyaxigkE=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -1021,8 +1021,8 @@ github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.32.0-k3s1 h1:cr0h
 github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.32.0-k3s1/go.mod h1:Kun5c2svjAPx0nnvJKYQWhfeNW+O0EpzHgRhDcYoSY0=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.32.0-k3s1 h1:jx/EPIAzm6iPCNjXp6rbcv3L+ElzNMVl1oj85iaRWyM=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.32.0-k3s1/go.mod h1:/vuNyuDhhSkWaqPq1t/ncDtqvlOftYgosR747XXodNM=
-github.com/k3s-io/runc v1.2.1-k3s1 h1:74ZffmoNVdX+jO+XYLv0iU/A9Yse1DZ2slZog7gNDgE=
-github.com/k3s-io/runc v1.2.1-k3s1/go.mod h1:/PXzF0h531HTMsYQnmxXkBD7YaGShm/2zcRB79dksUc=
+github.com/k3s-io/runc v1.2.4-k3s1 h1:wCqlmxJiVUsZvdNsSUUm7Ng0kCK2wAgyBt4lcp2lLP8=
+github.com/k3s-io/runc v1.2.4-k3s1/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM=
 github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314 h1:TrZb/yM0OtBuifPXlKaOfcxpJqzakA8+KsoO4c69ZLM=
 github.com/k3s-io/spegel v0.0.23-0.20240516234953-f3d2c4072314/go.mod h1:bMHfSjj1+Zf5VITCZe/wLjuni6rYAj/DjPU/kIVnhfA=
 github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=

manifests/traefik.yaml

@@ -28,7 +28,7 @@ spec:
     priorityClassName: "system-cluster-critical"
     image:
       repository: "rancher/mirrored-library-traefik"
-      tag: "2.11.10"
+      tag: "2.11.18"
     tolerations:
     - key: "CriticalAddonsOnly"
       operator: "Exists"

pkg/agent/config/config.go

@@ -2,9 +2,11 @@ package config
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	cryptorand "crypto/rand"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/hex"
 	"encoding/pem"
 	"fmt"
@@ -32,6 +34,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/pkg/errors"
+	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/rancher/wrangler/v3/pkg/slice"
 	"github.com/sirupsen/logrus"
@@ -133,9 +136,9 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b
 	return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token())
 }
 
-func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester {
+func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string, csr []byte) HTTPRequester {
 	return func(u string, client *http.Client, username, password, token string) ([]byte, error) {
-		req, err := http.NewRequest(http.MethodGet, u, nil)
+		req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(csr))
 		if err != nil {
 			return nil, err
 		}
@@ -238,51 +241,93 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string)
 	}
 }
 
-func getServingCert(nodeName string, nodeIPs []net.IP, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
-	servingCert, err := Request("/v1-"+version.Program+"/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
+// getKubeletServingCert fills the kubelet server certificate with content returned
+// from the server.  We attempt to POST a CSR to the server, in hopes that it will
+// sign the cert using our locally generated key. If the server does not support CSR
+// signing, the key generated by the server is used instead.
+func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
 	if err != nil {
-		return nil, err
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
 	}
 
-	servingCert, servingKey := splitCertKeyPEM(servingCert)
-
-	if err := os.WriteFile(servingCertFile, servingCert, 0600); err != nil {
-		return nil, errors.Wrapf(err, "failed to write node cert")
+	basename := filepath.Base(certFile)
+	body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
+	if err != nil {
+		return err
 	}
 
-	if err := os.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
-		return nil, errors.Wrapf(err, "failed to write node key")
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(body)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
 	}
-
-	cert, err := tls.X509KeyPair(servingCert, servingKey)
-	if err != nil {
-		return nil, err
+	if len(keyBytes) > 0 {
+		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
+		}
 	}
-	return &cert, nil
+	return nil
 }
 
-func getHostFile(filename, keyFile string, info *clientaccess.Info) error {
+// getHostFile fills a file with content returned from the server.
+func getHostFile(filename string, info *clientaccess.Info) error {
 	basename := filepath.Base(filename)
 	fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename)
 	if err != nil {
 		return err
 	}
-	if keyFile == "" {
-		if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write cert %s", filename)
-		}
-	} else {
-		fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
-		if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write cert %s", filename)
-		}
+	if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", filename)
+	}
+	return nil
+}
+
+// getClientCert fills a client certificate with content returned from the server.
+// We attempt to POST a CSR to the server, in hopes that it will sign the cert using
+// our locally generated key. If the server does not support CSR signing, the key
+// generated by the server is used instead.
+func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+	}
+
+	basename := filepath.Base(certFile)
+	fileBytes, err := info.Post("/v1-"+version.Program+"/"+basename, csr)
+	if err != nil {
+		return err
+	}
+
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(fileBytes)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
+	}
+	if len(keyBytes) > 0 {
 		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write key %s", filename)
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
 		}
 	}
 	return nil
 }
 
+func getCSRBytes(keyFile string) ([]byte, error) {
+	keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
+	if err != nil {
+		return nil, err
+	}
+	key, err := certutil.ParsePrivateKeyPEM(keyBytes)
+	if err != nil {
+		return nil, err
+	}
+	return x509.CreateCertificateRequest(cryptorand.Reader, &x509.CertificateRequest{}, key)
+}
+
 func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
 	for {
 		b, rest := pem.Decode(bytes)
@@ -301,19 +346,33 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
 	return
 }
 
-func getNodeNamedHostFile(filename, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
-	basename := filepath.Base(filename)
-	fileBytes, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
+// getKubeletClientCert fills the kubelet client certificate with content returned
+// from the server.  We attempt to POST a CSR to the server, in hopes that it will
+// sign the cert using our locally generated key. If the server does not support CSR
+// signing, the key generated by the server is used instead.
+func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+	}
+
+	basename := filepath.Base(certFile)
+	body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
 	if err != nil {
 		return err
 	}
-	fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
 
-	if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write cert %s", filename)
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(body)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
 	}
-	if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write key %s", filename)
+	if len(keyBytes) > 0 {
+		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
+		}
 	}
 	return nil
 }
@@ -399,12 +458,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 
 	clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt")
-	if err := getHostFile(clientCAFile, "", info); err != nil {
+	if err := getHostFile(clientCAFile, info); err != nil {
 		return nil, err
 	}
 
 	serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt")
-	if err := getHostFile(serverCAFile, "", info); err != nil {
+	if err := getHostFile(serverCAFile, info); err != nil {
 		return nil, err
 	}
 
@@ -498,14 +557,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	// that the cert will not be valid for, as they are not present in the list collected here.
 	nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)
 
-	// Ask the server to generate a kubelet server cert+key. These files are unique to this node.
-	servingCert, err := getServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info)
-	if err != nil {
+	// Ask the server to sign our kubelet server cert.
+	if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
 		return nil, errors.Wrap(err, servingKubeletCert)
 	}
 
-	// Ask the server to genrate a kubelet client cert+key. These files are unique to this node.
-	if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
+	// Ask the server to sign our kubelet client cert.
+	if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
 		return nil, errors.Wrap(err, clientKubeletCert)
 	}
 
@@ -518,8 +576,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
 	clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")
 
-	// Ask the server to send us its kube-proxy client cert+key. These files are not unique to this node.
-	if err := getHostFile(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
+	// Ask the server to sign our kube-proxy client cert.
+	if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
 		return nil, errors.Wrap(err, clientKubeProxyCert)
 	}
 
@@ -532,8 +590,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
 	clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")
 
-	// Ask the server to send us its agent controller client cert+key. These files are not unique to this node.
-	if err := getHostFile(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
+	// Ask the server to sign our agent controller client cert.
+	if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
 		return nil, errors.Wrap(err, clientK3sControllerCert)
 	}
 
@@ -625,7 +683,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	applyCRIDockerdAddress(nodeConfig)
 	applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd)
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
-	nodeConfig.Certificate = servingCert
 
 	if envInfo.BindAddress != "" {
 		nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress

pkg/agent/flannel/setup.go

@@ -69,14 +69,21 @@ func Prepare(ctx context.Context, nodeConfig *config.Node) error {
 func Run(ctx context.Context, nodeConfig *config.Node) error {
 	logrus.Infof("Starting flannel with backend %s", nodeConfig.FlannelBackend)
 
-	if err := util.WaitForRBACReady(ctx, nodeConfig.AgentConfig.KubeConfigK3sController, util.DefaultAPIServerReadyTimeout, authorizationv1.ResourceAttributes{
-		Verb:     "list",
-		Resource: "nodes",
-	}, ""); err != nil {
-		return errors.Wrap(err, "flannel failed to wait for RBAC")
+	kubeConfig := nodeConfig.AgentConfig.KubeConfigKubelet
+	resourceAttrs := authorizationv1.ResourceAttributes{Verb: "list", Resource: "nodes"}
+
+	// Compatibility code for AuthorizeNodeWithSelectors feature-gate.
+	// If the kubelet cannot list nodes, then wait for the k3s-controller RBAC to become ready, and use that kubeconfig instead.
+	if canListNodes, err := util.CheckRBAC(ctx, kubeConfig, resourceAttrs, ""); err != nil {
+		return errors.Wrap(err, "failed to check if RBAC allows node list")
+	} else if !canListNodes {
+		kubeConfig = nodeConfig.AgentConfig.KubeConfigK3sController
+		if err := util.WaitForRBACReady(ctx, kubeConfig, util.DefaultAPIServerReadyTimeout, resourceAttrs, ""); err != nil {
+			return errors.Wrap(err, "flannel failed to wait for RBAC")
+		}
 	}
 
-	coreClient, err := util.GetClientSet(nodeConfig.AgentConfig.KubeConfigK3sController)
+	coreClient, err := util.GetClientSet(kubeConfig)
 	if err != nil {
 		return err
 	}
@@ -90,7 +97,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 		return errors.Wrap(err, "failed to check netMode for flannel")
 	}
 	go func() {
-		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, nodeConfig.AgentConfig.KubeConfigK3sController, nodeConfig.FlannelIPv6Masq, netMode)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, kubeConfig, nodeConfig.FlannelIPv6Masq, netMode)
 		if err != nil && !errors.Is(err, context.Canceled) {
 			logrus.Errorf("flannel exited: %v", err)
 			os.Exit(1)

pkg/agent/flannel/setup_test.go

@@ -62,7 +62,7 @@ func Test_createFlannelConf(t *testing.T) {
 		var agent = config.Agent{}
 		agent.ClusterCIDR = stringToCIDR(tt.args)[0]
 		agent.ClusterCIDRs = stringToCIDR(tt.args)
-		var nodeConfig = &config.Node{Docker: false, ContainerRuntimeEndpoint: "", SELinux: false, FlannelBackend: "vxlan", FlannelConfFile: "test_file", FlannelConfOverride: false, FlannelIface: nil, Containerd: containerd, Images: "", AgentConfig: agent, Token: "", Certificate: nil, ServerHTTPSPort: 0}
+		var nodeConfig = &config.Node{Docker: false, ContainerRuntimeEndpoint: "", SELinux: false, FlannelBackend: "vxlan", FlannelConfFile: "test_file", FlannelConfOverride: false, FlannelIface: nil, Containerd: containerd, Images: "", AgentConfig: agent, Token: "", ServerHTTPSPort: 0}
 
 		t.Run(tt.name, func(t *testing.T) {
 			if err := createFlannelConf(nodeConfig); (err != nil) != tt.wantErr {