Use recert for proxy seed reconfiguration

# Overview

LCA will now use recert's new proxy feature. This will allow us to
support environments where the seed proxy is different than the upgraded
cluster's proxy. It will also allow us to perform a rollout-free seed
proxy reconfiguration.

Clusters with a proxy can only be upgraded using seeds that have a
proxy. Clusters without a proxy can only be upgraded using seeds that
don't have a proxy.

# Notes

Two new fields have been added to `SeedReconfiguration`, `Proxy` and
`StatusProxy` - see commit diff for an explanation of why we need both
and how they're determined in every scenario.

Added a new `HasProxy` field to the seed cluster info struct.

Seed images will now contain a `com.openshift.lifecycle-agent.seed_cluster_info`
OCI label which will contain a JSON serialized seed cluster info struct.
This label will allow LCA (and maybe IBIO in the future?) to make
informed decisions about the compatibility of a particular seed image
with the cluster being upgraded. For example, LCA will refuse to upgrade
a cluster that has a proxy with a seed that doesn't. This label helps
LCA knows whether the seed has a proxy or not (through the new
`HasProxy` field), and block the upgrade appropriately before the image
even gets pulled.

# install-config

On top of proxy, we're also going to be using recert's new
install-config option. See comment about the new InstallConfig field in
the seed reconfiguration struct for more information about why.

api/seedreconfig/seedreconfig.go

@@ -97,6 +97,41 @@ type SeedReconfiguration struct {
 	// This will be used to create the node network and choose ip address for the node.
 	// Equivalent to install-config.yaml's machineNetwork.
 	MachineNetwork string `json:"machine_network,omitempty"`
+
+	// Proxy is the proxy settings for the cluster. Equivalent to
+	// install-config.yaml's proxy. This will replace the proxy settings of the
+	// seed cluster. During IBI, the HTTP and HTTPS and NO proxy settings are
+	// given by the user, and IBIO will also calculate a final NO_PROXY
+	// configuration based on the user provided NO_PROXY and the cluster's
+	// configuration. During an IBU, this is simply taken from the cluster's
+	// Proxy CR .spec field.
+	Proxy *Proxy `json:"proxy,omitempty"`
+
+	// StatusProxy is the Proxy configuration from the Proxy CR's status field.
+	// The Cluster Network Operator calculates a proxy configuration based on
+	// the user provided proxy configuration (taken from the Proxy CR's spec)
+	// and puts that proxy configuration in the Proxy CR's status. The status
+	// proxy configuration is the one that's actually used by the operators in
+	// the cluster. During a seed reconfiguration, we need to know both the
+	// status proxy configuration and the user's spec proxy configuration, as
+	// both are required to perform a rollout free seed reconfiguration.
+	// Otherwise we would have to teach recert how to calculate the status
+	// proxy from the spec proxy which is non-trivial. During an IBU, we simply
+	// take this status from the cluster's Proxy CR .status field. During an
+	// IBI, IBIO should do its best to emulate the behavior of the CNO so that
+	// the status proxy configuration is identical to what CNO would have
+	// calculated, so that rollouts are avoided.
+	StatusProxy *Proxy `json:"status_proxy,omitempty"`
+
+	// The install-config.yaml used to generate the cluster. This is used to
+	// populate the cluster-config-v1 configmaps in the cluster, which usually
+	// hold a slightly modified version of the user's installation
+	// install-config.yaml. Changing this is important because the Cluster
+	// Network Operator uses the information in this configmap to set the Proxy
+	// CR's no_proxy status field. In IBU, LCA simply copies it from the
+	// upgraded cluster. In IBI, a fake install-config should be generated by
+	// the IBIO. This parameter is required when the proxy parameters are set.
+	InstallConfig string `json:"install_config,omitempty"`
 }
 
 type KubeConfigCryptoRetention struct {
@@ -122,3 +157,20 @@ type ClientAuthCrypto struct {
 type IngresssCrypto struct {
 	IngressCA PEM `json:"ingress_ca,omitempty"`
 }
+
+// Proxy defines the proxy settings for the cluster.
+// At least one of HTTPProxy or HTTPSProxy is required.
+// Aims to be the same as https://github.com/openshift/installer/blob/ad59622147974f2d2d62bcdeaf342ae4f87ed84f/pkg/types/installconfig.go#L454-L468
+type Proxy struct {
+	// HTTPProxy is the URL of the proxy for HTTP requests.
+	// +optional
+	HTTPProxy string `json:"httpProxy,omitempty"`
+
+	// HTTPSProxy is the URL of the proxy for HTTPS requests.
+	// +optional
+	HTTPSProxy string `json:"httpsProxy,omitempty"`
+
+	// NoProxy is a comma-separated list of domains and CIDRs for which the proxy should not be used.
+	// +optional
+	NoProxy string `json:"noProxy,omitempty"`
+}

controllers/prep_handlers.go

@@ -69,22 +69,57 @@ func (r *ImageBasedUpgradeReconciler) getSeedImage(
 		return fmt.Errorf("failed to pull image: %w", err)
 	}
 
-	r.Log.Info("Checking seed image compatibility")
-	if err := r.checkSeedImageCompatibility(ctx, ibu.Spec.SeedImageRef.Image); err != nil {
+	labels, err := r.getLabelsForSeedImage(ibu.Spec.SeedImageRef.Image)
+	if err != nil {
+		return fmt.Errorf("failed to get seed image labels: %w", err)
+	}
+
+	r.Log.Info("Checking seed image version compatibility")
+	if err := checkSeedImageVersionCompatibility(labels); err != nil {
+		return fmt.Errorf("checking seed image compatibility: %w", err)
+	}
+
+	seedInfo, err := getSeedConfigFromLabel(labels)
+	if err != nil {
+		return fmt.Errorf("failed to get seed cluster info from label: %w", err)
+	}
+
+	seedHasProxy := false
+	if seedInfo == nil {
+		// Older images may not have the seed cluster info label, in which case
+		// we assume no proxy so that if the current cluster has proxy, it will
+		// fail the compatibility check.
+		seedHasProxy = seedInfo.HasProxy
+	}
+
+	clusterHasProxy, err := lcautils.HasProxy(ctx, r.Client)
+	if err != nil {
+		return fmt.Errorf("failed to check if cluster has proxy: %w", err)
+	}
+
+	r.Log.Info("Checking seed image proxy compatibility")
+	if err := checkSeedImageProxyCompatibility(seedHasProxy, clusterHasProxy); err != nil {
 		return fmt.Errorf("checking seed image compatibility: %w", err)
 	}
 
 	return nil
 }
 
-// checkSeedImageCompatibility checks if the seed image is compatible with the
-// current version of the lifecycle-agent by inspecting the OCI image's labels
-// and checking if the specified format version equals the hard-coded one that
-// this version of the lifecycle agent expects. That format version is set by
-// the lca-cli during the image build process, and is only manually bumped by
-// developers when the image format changes in a way that is incompatible with
-// previous versions of the lifecycle-agent.
-func (r *ImageBasedUpgradeReconciler) checkSeedImageCompatibility(_ context.Context, seedImageRef string) error {
+func getSeedConfigFromLabel(labels map[string]string) (*seedclusterinfo.SeedClusterInfo, error) {
+	seedFormatLabelValue, ok := labels[common.SeedClusterInfoOCILabel]
+	if !ok {
+		return nil, nil
+	}
+
+	var seedInfo seedclusterinfo.SeedClusterInfo
+	if err := json.Unmarshal([]byte(seedFormatLabelValue), &seedInfo); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal seed cluster info: %w", err)
+	}
+
+	return &seedInfo, nil
+}
+
+func (r *ImageBasedUpgradeReconciler) getLabelsForSeedImage(seedImageRef string) (map[string]string, error) {
 	inspectArgs := []string{
 		"inspect",
 		"--format", "json",
@@ -97,28 +132,40 @@ func (r *ImageBasedUpgradeReconciler) checkSeedImageCompatibility(_ context.Cont
 
 	// TODO: use the context when execute supports it
 	if inspectRaw, err := r.Executor.Execute("podman", inspectArgs...); err != nil || inspectRaw == "" {
-		return fmt.Errorf("failed to inspect image: %w", err)
+		return nil, fmt.Errorf("failed to inspect image: %w", err)
 	} else {
 		if err := json.Unmarshal([]byte(inspectRaw), &inspect); err != nil {
-			return fmt.Errorf("failed to unmarshal image inspect output: %w", err)
+			return nil, fmt.Errorf("failed to unmarshal image inspect output: %w", err)
 		}
 	}
 
 	if len(inspect) != 1 {
-		return fmt.Errorf("expected 1 image inspect result, got %d", len(inspect))
+		return nil, fmt.Errorf("expected 1 image inspect result, got %d", len(inspect))
 	}
 
-	seedFormatLabelValue, ok := inspect[0].Labels[common.SeedFormatOCILabel]
+	return inspect[0].Labels, nil
+}
+
+// checkSeedImageVersionCompatibility checks if the seed image is compatible with the
+// current version of the lifecycle-agent by inspecting the OCI image's labels
+// and checking if the specified format version equals the hard-coded one that
+// this version of the lifecycle agent expects. That format version is set by
+// the LCA during the image build process to the value of the code constant,
+// and the code constant is only manually bumped by developers when the image
+// format changes in a way that is incompatible with previous versions of the
+// lifecycle-agent.
+func checkSeedImageVersionCompatibility(labels map[string]string) error {
+	seedFormatLabelValue, ok := labels[common.SeedFormatOCILabel]
 	if !ok {
 		return fmt.Errorf(
-			"seed image %s is missing the %s label, please build a new image using the latest version of the lca-cli",
-			seedImageRef, common.SeedFormatOCILabel)
+			"seed image is missing the %s label, please build a new image using the latest version of the lca-cli",
+			common.SeedFormatOCILabel)
 	}
 
 	// Hard equal since we don't have backwards compatibility guarantees yet.
 	// In the future we might want to have backwards compatibility code to
 	// handle older seed formats and in that case we'll look at the version
-	// number and do the right thing.
+	// number and do the right thing accordingly.
 	if seedFormatLabelValue != fmt.Sprintf("%d", common.SeedFormatVersion) {
 		return fmt.Errorf("seed image format version mismatch: expected %d, got %s",
 			common.SeedFormatVersion, seedFormatLabelValue)
@@ -127,6 +174,24 @@ func (r *ImageBasedUpgradeReconciler) checkSeedImageCompatibility(_ context.Cont
 	return nil
 }
 
+// checkSeedImageProxyCompatibility checks for proxy configuration
+// compatibility of the seed image vs the current cluster. If the seed image
+// has a proxy and the cluster being upgraded doesn't, we cannot proceed as
+// recert does not support proxy rename under those conditions. Similarly, we
+// cannot proceed if the cluster being upgraded has a proxy but the seed image
+// doesn't.
+func checkSeedImageProxyCompatibility(seedHasProxy, hasProxy bool) error {
+	if seedHasProxy && !hasProxy {
+		return fmt.Errorf("seed image has a proxy but the cluster being upgraded does not, this combination is not supported")
+	}
+
+	if !seedHasProxy && hasProxy {
+		return fmt.Errorf("seed image does not have a proxy but the cluster being upgraded does, this combination is not supported")
+	}
+
+	return nil
+}
+
 // validateSeedOcpVersion rejects upgrade request if seed image version is not higher than current cluster (target) OCP version
 func (r *ImageBasedUpgradeReconciler) validateSeedOcpVersion(seedOcpVersion string) error {
 	// get target OCP version

internal/clusterconfig/clusterconfig.go

@@ -33,8 +33,7 @@ import (
 const (
 	manifestDir = "manifests"
 
-	proxyName     = "cluster"
-	proxyFileName = "proxy.json"
+	proxyName = "cluster"
 
 	pullSecretName = "pull-secret"
 
@@ -78,9 +77,6 @@ func (r *UpgradeClusterConfigGather) FetchClusterConfig(ctx context.Context, ost
 	}
 	manifestsDir := filepath.Join(clusterConfigPath, manifestDir)
 
-	if err := r.fetchProxy(ctx, manifestsDir); err != nil {
-		return err
-	}
 	if err := r.fetchIDMS(ctx, manifestsDir); err != nil {
 		return err
 	}
@@ -102,7 +98,7 @@ func (r *UpgradeClusterConfigGather) FetchClusterConfig(ctx context.Context, ost
 	return nil
 }
 
-func (r *UpgradeClusterConfigGather) fetchPullSecret(ctx context.Context) (string, error) {
+func (r *UpgradeClusterConfigGather) getPullSecret(ctx context.Context) (string, error) {
 	r.Log.Info("Fetching pull-secret")
 	sd, err := utils.GetSecretData(ctx, common.PullSecretName, common.OpenshiftConfigNamespace, corev1.DockerConfigJsonKey, r.Client)
 	if err != nil {
@@ -111,44 +107,15 @@ func (r *UpgradeClusterConfigGather) fetchPullSecret(ctx context.Context) (strin
 	return sd, nil
 }
 
-func (r *UpgradeClusterConfigGather) fetchProxy(ctx context.Context, manifestsDir string) error {
-	r.Log.Info("Fetching cluster-wide proxy", "name", proxyName)
-
-	proxy := v1.Proxy{}
-	if err := r.Client.Get(ctx, types.NamespacedName{Name: proxyName}, &proxy); err != nil {
-		return fmt.Errorf("failed to get proxy: %w", err)
-	}
-
-	p := v1.Proxy{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: proxy.Name,
-		},
-		Spec: proxy.Spec,
-	}
-	typeMeta, err := r.typeMetaForObject(&p)
-	if err != nil {
-		return err
-	}
-	p.TypeMeta = *typeMeta
-
-	filePath := filepath.Join(manifestsDir, proxyFileName)
-	r.Log.Info("Writing proxy to file", "path", filePath)
-	err = utils.MarshalToFile(p, filePath)
-	if err != nil {
-		return fmt.Errorf("filed to write proxy to file to path %s :%w", filePath, err)
-	}
-	return nil
-}
-
-func (r *UpgradeClusterConfigGather) fetchSSHPublicKey() (string, error) {
+func (r *UpgradeClusterConfigGather) getSSHPublicKey() (string, error) {
 	sshKey, err := os.ReadFile(filepath.Join(hostPath, sshKeyFile))
 	if err != nil {
 		return "", fmt.Errorf("failed to read sshKey: %w", err)
 	}
 	return string(sshKey), err
 }
 
-func (r *UpgradeClusterConfigGather) fetchInfraID(ctx context.Context) (string, error) {
+func (r *UpgradeClusterConfigGather) getInfraID(ctx context.Context) (string, error) {
 	infra, err := utils.GetInfrastructure(ctx, r.Client)
 	if err != nil {
 		return "", fmt.Errorf("failed to get infrastructure: %w", err)
@@ -177,8 +144,50 @@ func (r *UpgradeClusterConfigGather) GetKubeadminPasswordHash(ctx context.Contex
 	return kubeadminPasswordHash, nil
 }
 
-func SeedReconfigurationFromClusterInfo(clusterInfo *utils.ClusterInfo,
-	kubeconfigCryptoRetention *seedreconfig.KubeConfigCryptoRetention, sshKey, infraID, pullSecret, kubeadminPasswordHash string) *seedreconfig.SeedReconfiguration {
+func (r *UpgradeClusterConfigGather) GetProxy(ctx context.Context) (*seedreconfig.Proxy, *seedreconfig.Proxy, error) {
+	proxy := v1.Proxy{}
+	if err := r.Client.Get(ctx, types.NamespacedName{Name: proxyName}, &proxy); err != nil {
+		return nil, nil, fmt.Errorf("failed to get proxy: %w", err)
+	}
+
+	if proxy.Spec.HTTPProxy == "" && proxy.Spec.HTTPSProxy == "" && proxy.Spec.NoProxy == "" {
+		return nil, nil, nil
+	}
+
+	return &seedreconfig.Proxy{
+			HTTPProxy:  proxy.Spec.HTTPProxy,
+			HTTPSProxy: proxy.Spec.HTTPSProxy,
+			NoProxy:    proxy.Spec.NoProxy,
+		},
+		&seedreconfig.Proxy{
+			HTTPProxy:  proxy.Status.HTTPProxy,
+			HTTPSProxy: proxy.Status.HTTPSProxy,
+			NoProxy:    proxy.Status.NoProxy,
+		},
+		nil
+
+}
+
+func (r *UpgradeClusterConfigGather) GetInstallConfig(ctx context.Context) (string, error) {
+	configmap := corev1.ConfigMap{}
+	if err := r.Client.Get(ctx, types.NamespacedName{Namespace: common.InstallConfigCMNamespace, Name: common.InstallConfigCM}, &configmap); err != nil {
+		return "", fmt.Errorf("failed to get install-config configmap: %w", err)
+	}
+	return configmap.Data[common.InstallConfigCMInstallConfigDataKey], nil
+
+}
+
+func SeedReconfigurationFromClusterInfo(
+	clusterInfo *utils.ClusterInfo,
+	kubeconfigCryptoRetention *seedreconfig.KubeConfigCryptoRetention,
+	sshKey string,
+	infraID string,
+	pullSecret string,
+	kubeadminPasswordHash string,
+	proxy,
+	statusProxy *seedreconfig.Proxy,
+	installConfig string,
+) *seedreconfig.SeedReconfiguration {
 	return &seedreconfig.SeedReconfiguration{
 		APIVersion:                seedreconfig.SeedReconfigurationVersion,
 		BaseDomain:                clusterInfo.BaseDomain,
@@ -192,6 +201,9 @@ func SeedReconfigurationFromClusterInfo(clusterInfo *utils.ClusterInfo,
 		SSHKey:                    sshKey,
 		PullSecret:                pullSecret,
 		KubeadminPasswordHash:     kubeadminPasswordHash,
+		Proxy:                     proxy,
+		StatusProxy:               statusProxy,
+		InstallConfig:             installConfig,
 	}
 }
 
@@ -208,17 +220,17 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust
 		return fmt.Errorf("failed to get kubeconfig retention from crypto dir: %w", err)
 	}
 
-	sshKey, err := r.fetchSSHPublicKey()
+	sshKey, err := r.getSSHPublicKey()
 	if err != nil {
 		return err
 	}
 
-	infraID, err := r.fetchInfraID(ctx)
+	infraID, err := r.getInfraID(ctx)
 	if err != nil {
 		return err
 	}
 
-	pullSecret, err := r.fetchPullSecret(ctx)
+	pullSecret, err := r.getPullSecret(ctx)
 	if err != nil {
 		return err
 	}
@@ -228,11 +240,24 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust
 		return err
 	}
 
+	proxy, statusProxy, err := r.GetProxy(ctx)
+	if err != nil {
+		return err
+	}
+
+	installConfig, err := r.GetInstallConfig(ctx)
+	if err != nil {
+		return err
+	}
+
 	seedReconfiguration := SeedReconfigurationFromClusterInfo(clusterInfo, seedReconfigurationKubeconfigRetention,
 		sshKey,
 		infraID,
 		pullSecret,
 		kubeadminPasswordHash,
+		proxy,
+		statusProxy,
+		installConfig,
 	)
 
 	filePath := filepath.Join(clusterConfigPath, common.SeedReconfigurationFileName)