diff --git a/api/seedreconfig/seedreconfig.go b/api/seedreconfig/seedreconfig.go index 9f8eac79c..a3a8f2fea 100644 --- a/api/seedreconfig/seedreconfig.go +++ b/api/seedreconfig/seedreconfig.go @@ -97,6 +97,41 @@ type SeedReconfiguration struct { // This will be used to create the node network and choose ip address for the node. // Equivalent to install-config.yaml's machineNetwork. MachineNetwork string `json:"machine_network,omitempty"` + + // Proxy is the proxy settings for the cluster. Equivalent to + // install-config.yaml's proxy. This will replace the proxy settings of the + // seed cluster. During IBI, the HTTP and HTTPS and NO proxy settings are + // given by the user, and IBIO will also calculate a final NO_PROXY + // configuration based on the user provided NO_PROXY and the cluster's + // configuration. During an IBU, this is simply taken from the cluster's + // Proxy CR .spec field. + Proxy *Proxy `json:"proxy,omitempty"` + + // StatusProxy is the Proxy configuration from the Proxy CR's status field. + // The Cluster Network Operator calculates a proxy configuration based on + // the user provided proxy configuration (taken from the Proxy CR's spec) + // and puts that proxy configuration in the Proxy CR's status. The status + // proxy configuration is the one that's actually used by the operators in + // the cluster. During a seed reconfiguration, we need to know both the + // status proxy configuration and the user's spec proxy configuration, as + // both are required to perform a rollout free seed reconfiguration. + // Otherwise we would have to teach recert how to calculate the status + // proxy from the spec proxy which is non-trivial. During an IBU, we simply + // take this status from the cluster's Proxy CR .status field. During an + // IBI, IBIO should do its best to emulate the behavior of the CNO so that + // the status proxy configuration is identical to what CNO would have + // calculated, so that rollouts are avoided. + StatusProxy *Proxy `json:"status_proxy,omitempty"` + + // The install-config.yaml used to generate the cluster. This is used to + // populate the cluster-config-v1 configmaps in the cluster, which usually + // hold a slightly modified version of the user's installation + // install-config.yaml. Changing this is important because the Cluster + // Network Operator uses the information in this configmap to set the Proxy + // CR's no_proxy status field. In IBU, LCA simply copies it from the + // upgraded cluster. In IBI, a fake install-config should be generated by + // the IBIO. This parameter is required when the proxy parameters are set. + InstallConfig string `json:"install_config,omitempty"` } type KubeConfigCryptoRetention struct { @@ -122,3 +157,20 @@ type ClientAuthCrypto struct { type IngresssCrypto struct { IngressCA PEM `json:"ingress_ca,omitempty"` } + +// Proxy defines the proxy settings for the cluster. +// At least one of HTTPProxy or HTTPSProxy is required. +// Aims to be the same as https://github.com/openshift/installer/blob/ad59622147974f2d2d62bcdeaf342ae4f87ed84f/pkg/types/installconfig.go#L454-L468 +type Proxy struct { + // HTTPProxy is the URL of the proxy for HTTP requests. + // +optional + HTTPProxy string `json:"httpProxy,omitempty"` + + // HTTPSProxy is the URL of the proxy for HTTPS requests. + // +optional + HTTPSProxy string `json:"httpsProxy,omitempty"` + + // NoProxy is a comma-separated list of domains and CIDRs for which the proxy should not be used. + // +optional + NoProxy string `json:"noProxy,omitempty"` +} diff --git a/controllers/prep_handlers.go b/controllers/prep_handlers.go index 55b005b97..4df5fd15e 100644 --- a/controllers/prep_handlers.go +++ b/controllers/prep_handlers.go @@ -69,22 +69,49 @@ func (r *ImageBasedUpgradeReconciler) getSeedImage( return fmt.Errorf("failed to pull image: %w", err) } - r.Log.Info("Checking seed image compatibility") - if err := r.checkSeedImageCompatibility(ctx, ibu.Spec.SeedImageRef.Image); err != nil { + labels, err := r.getLabelsForSeedImage(ibu.Spec.SeedImageRef.Image) + if err != nil { + return fmt.Errorf("failed to get seed image labels: %w", err) + } + + r.Log.Info("Checking seed image version compatibility") + if err := checkSeedImageVersionCompatibility(labels); err != nil { + return fmt.Errorf("checking seed image compatibility: %w", err) + } + + seedInfo, err := getSeedConfigFromLabel(labels) + if err != nil { + return fmt.Errorf("failed to get seed cluster info from label: %w", err) + } + + hasProxy, err := lcautils.HasProxy(ctx, r.Client) + if err != nil { + return fmt.Errorf("failed to check if cluster has proxy: %w", err) + } + + r.Log.Info("Checking seed image proxy compatibility") + if err := checkSeedImageProxyCompatibility(seedInfo.HasProxy, hasProxy); err != nil { return fmt.Errorf("checking seed image compatibility: %w", err) } return nil } -// checkSeedImageCompatibility checks if the seed image is compatible with the -// current version of the lifecycle-agent by inspecting the OCI image's labels -// and checking if the specified format version equals the hard-coded one that -// this version of the lifecycle agent expects. That format version is set by -// the lca-cli during the image build process, and is only manually bumped by -// developers when the image format changes in a way that is incompatible with -// previous versions of the lifecycle-agent. -func (r *ImageBasedUpgradeReconciler) checkSeedImageCompatibility(_ context.Context, seedImageRef string) error { +func getSeedConfigFromLabel(labels map[string]string) (*seedclusterinfo.SeedClusterInfo, error) { + seedFormatLabelValue, ok := labels[common.SeedClusterInfoOCILabel] + if !ok { + return nil, fmt.Errorf("seed image is missing the %s label, please build a new image using the latest version of LCA", common.SeedFormatOCILabel) + } + + var seedInfo seedclusterinfo.SeedClusterInfo + if err := json.Unmarshal([]byte(seedFormatLabelValue), &seedInfo); err != nil { + return nil, fmt.Errorf("failed to unmarshal seed cluster info: %w", err) + } + + return &seedInfo, nil +} + +func (r *ImageBasedUpgradeReconciler) getLabelsForSeedImage(seedImageRef string) (map[string]string, error) { inspectArgs := []string{ "inspect", "--format", "json", @@ -97,28 +124,40 @@ func (r *ImageBasedUpgradeReconciler) checkSeedImageCompatibility(_ context.Cont // TODO: use the context when execute supports it if inspectRaw, err := r.Executor.Execute("podman", inspectArgs...); err != nil || inspectRaw == "" { - return fmt.Errorf("failed to inspect image: %w", err) + return nil, fmt.Errorf("failed to inspect image: %w", err) } else { if err := json.Unmarshal([]byte(inspectRaw), &inspect); err != nil { - return fmt.Errorf("failed to unmarshal image inspect output: %w", err) + return nil, fmt.Errorf("failed to unmarshal image inspect output: %w", err) } } if len(inspect) != 1 { - return fmt.Errorf("expected 1 image inspect result, got %d", len(inspect)) + return nil, fmt.Errorf("expected 1 image inspect result, got %d", len(inspect)) } - seedFormatLabelValue, ok := inspect[0].Labels[common.SeedFormatOCILabel] + return inspect[0].Labels, nil +} + +// checkSeedImageVersionCompatibility checks if the seed image is compatible with the +// current version of the lifecycle-agent by inspecting the OCI image's labels +// and checking if the specified format version equals the hard-coded one that +// this version of the lifecycle agent expects. That format version is set by +// the LCA during the image build process to the value of the code constant, +// and the code constant is only manually bumped by developers when the image +// format changes in a way that is incompatible with previous versions of the +// lifecycle-agent. +func checkSeedImageVersionCompatibility(labels map[string]string) error { + seedFormatLabelValue, ok := labels[common.SeedFormatOCILabel] if !ok { return fmt.Errorf( - "seed image %s is missing the %s label, please build a new image using the latest version of the lca-cli", - seedImageRef, common.SeedFormatOCILabel) + "seed image is missing the %s label, please build a new image using the latest version of the lca-cli", + common.SeedFormatOCILabel) } // Hard equal since we don't have backwards compatibility guarantees yet. // In the future we might want to have backwards compatibility code to // handle older seed formats and in that case we'll look at the version - // number and do the right thing. + // number and do the right thing accordingly. if seedFormatLabelValue != fmt.Sprintf("%d", common.SeedFormatVersion) { return fmt.Errorf("seed image format version mismatch: expected %d, got %s", common.SeedFormatVersion, seedFormatLabelValue) @@ -127,6 +166,24 @@ func (r *ImageBasedUpgradeReconciler) checkSeedImageCompatibility(_ context.Cont return nil } +// checkSeedImageProxyCompatibility checks for proxy configuration +// compatibility of the seed image vs the current cluster. If the seed image +// has a proxy and the cluster being upgraded doesn't, we cannot proceed as +// recert does not support proxy rename under those conditions. Similarly, we +// cannot proceed if the cluster being upgraded has a proxy but the seed image +// doesn't. +func checkSeedImageProxyCompatibility(seedHasProxy, hasProxy bool) error { + if seedHasProxy && !hasProxy { + return fmt.Errorf("seed image has a proxy but the cluster being upgraded does not, this combination is not supported") + } + + if !seedHasProxy && hasProxy { + return fmt.Errorf("seed image does not have a proxy but the cluster being upgraded does, this combination is not supported") + } + + return nil +} + // validateSeedOcpVersion rejects upgrade request if seed image version is not higher than current cluster (target) OCP version func (r *ImageBasedUpgradeReconciler) validateSeedOcpVersion(seedOcpVersion string) error { // get target OCP version diff --git a/internal/clusterconfig/clusterconfig.go b/internal/clusterconfig/clusterconfig.go index cc70b7f48..f7e889dac 100644 --- a/internal/clusterconfig/clusterconfig.go +++ b/internal/clusterconfig/clusterconfig.go @@ -33,8 +33,7 @@ import ( const ( manifestDir = "manifests" - proxyName = "cluster" - proxyFileName = "proxy.json" + proxyName = "cluster" pullSecretName = "pull-secret" @@ -78,9 +77,6 @@ func (r *UpgradeClusterConfigGather) FetchClusterConfig(ctx context.Context, ost } manifestsDir := filepath.Join(clusterConfigPath, manifestDir) - if err := r.fetchProxy(ctx, manifestsDir); err != nil { - return err - } if err := r.fetchIDMS(ctx, manifestsDir); err != nil { return err } @@ -102,7 +98,7 @@ func (r *UpgradeClusterConfigGather) FetchClusterConfig(ctx context.Context, ost return nil } -func (r *UpgradeClusterConfigGather) fetchPullSecret(ctx context.Context) (string, error) { +func (r *UpgradeClusterConfigGather) getPullSecret(ctx context.Context) (string, error) { r.Log.Info("Fetching pull-secret") sd, err := utils.GetSecretData(ctx, common.PullSecretName, common.OpenshiftConfigNamespace, corev1.DockerConfigJsonKey, r.Client) if err != nil { @@ -111,36 +107,7 @@ func (r *UpgradeClusterConfigGather) fetchPullSecret(ctx context.Context) (strin return sd, nil } -func (r *UpgradeClusterConfigGather) fetchProxy(ctx context.Context, manifestsDir string) error { - r.Log.Info("Fetching cluster-wide proxy", "name", proxyName) - - proxy := v1.Proxy{} - if err := r.Client.Get(ctx, types.NamespacedName{Name: proxyName}, &proxy); err != nil { - return fmt.Errorf("failed to get proxy: %w", err) - } - - p := v1.Proxy{ - ObjectMeta: metav1.ObjectMeta{ - Name: proxy.Name, - }, - Spec: proxy.Spec, - } - typeMeta, err := r.typeMetaForObject(&p) - if err != nil { - return err - } - p.TypeMeta = *typeMeta - - filePath := filepath.Join(manifestsDir, proxyFileName) - r.Log.Info("Writing proxy to file", "path", filePath) - err = utils.MarshalToFile(p, filePath) - if err != nil { - return fmt.Errorf("filed to write proxy to file to path %s :%w", filePath, err) - } - return nil -} - -func (r *UpgradeClusterConfigGather) fetchSSHPublicKey() (string, error) { +func (r *UpgradeClusterConfigGather) getSSHPublicKey() (string, error) { sshKey, err := os.ReadFile(filepath.Join(hostPath, sshKeyFile)) if err != nil { return "", fmt.Errorf("failed to read sshKey: %w", err) @@ -148,7 +115,7 @@ func (r *UpgradeClusterConfigGather) fetchSSHPublicKey() (string, error) { return string(sshKey), err } -func (r *UpgradeClusterConfigGather) fetchInfraID(ctx context.Context) (string, error) { +func (r *UpgradeClusterConfigGather) getInfraID(ctx context.Context) (string, error) { infra, err := utils.GetInfrastructure(ctx, r.Client) if err != nil { return "", fmt.Errorf("failed to get infrastructure: %w", err) @@ -177,8 +144,50 @@ func (r *UpgradeClusterConfigGather) GetKubeadminPasswordHash(ctx context.Contex return kubeadminPasswordHash, nil } -func SeedReconfigurationFromClusterInfo(clusterInfo *utils.ClusterInfo, - kubeconfigCryptoRetention *seedreconfig.KubeConfigCryptoRetention, sshKey, infraID, pullSecret, kubeadminPasswordHash string) *seedreconfig.SeedReconfiguration { +func (r *UpgradeClusterConfigGather) GetProxy(ctx context.Context) (*seedreconfig.Proxy, *seedreconfig.Proxy, error) { + proxy := v1.Proxy{} + if err := r.Client.Get(ctx, types.NamespacedName{Name: proxyName}, &proxy); err != nil { + return nil, nil, fmt.Errorf("failed to get proxy: %w", err) + } + + if proxy.Spec.HTTPProxy == "" && proxy.Spec.HTTPSProxy == "" && proxy.Spec.NoProxy == "" { + return nil, nil, nil + } + + return &seedreconfig.Proxy{ + HTTPProxy: proxy.Spec.HTTPProxy, + HTTPSProxy: proxy.Spec.HTTPSProxy, + NoProxy: proxy.Spec.NoProxy, + }, + &seedreconfig.Proxy{ + HTTPProxy: proxy.Status.HTTPProxy, + HTTPSProxy: proxy.Status.HTTPSProxy, + NoProxy: proxy.Status.NoProxy, + }, + nil + +} + +func (r *UpgradeClusterConfigGather) GetInstallConfig(ctx context.Context) (string, error) { + configmap := corev1.ConfigMap{} + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: common.InstallConfigCMNamespace, Name: common.InstallConfigCM}, &configmap); err != nil { + return "", fmt.Errorf("failed to get install-config configmap: %w", err) + } + return configmap.Data[common.InstallConfigCMInstallConfigDataKey], nil + +} + +func SeedReconfigurationFromClusterInfo( + clusterInfo *utils.ClusterInfo, + kubeconfigCryptoRetention *seedreconfig.KubeConfigCryptoRetention, + sshKey string, + infraID string, + pullSecret string, + kubeadminPasswordHash string, + proxy, + statusProxy *seedreconfig.Proxy, + installConfig string, +) *seedreconfig.SeedReconfiguration { return &seedreconfig.SeedReconfiguration{ APIVersion: seedreconfig.SeedReconfigurationVersion, BaseDomain: clusterInfo.BaseDomain, @@ -192,6 +201,9 @@ func SeedReconfigurationFromClusterInfo(clusterInfo *utils.ClusterInfo, SSHKey: sshKey, PullSecret: pullSecret, KubeadminPasswordHash: kubeadminPasswordHash, + Proxy: proxy, + StatusProxy: statusProxy, + InstallConfig: installConfig, } } @@ -208,17 +220,17 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust return fmt.Errorf("failed to get kubeconfig retention from crypto dir: %w", err) } - sshKey, err := r.fetchSSHPublicKey() + sshKey, err := r.getSSHPublicKey() if err != nil { return err } - infraID, err := r.fetchInfraID(ctx) + infraID, err := r.getInfraID(ctx) if err != nil { return err } - pullSecret, err := r.fetchPullSecret(ctx) + pullSecret, err := r.getPullSecret(ctx) if err != nil { return err } @@ -228,11 +240,24 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust return err } + proxy, statusProxy, err := r.GetProxy(ctx) + if err != nil { + return err + } + + installConfig, err := r.GetInstallConfig(ctx) + if err != nil { + return err + } + seedReconfiguration := SeedReconfigurationFromClusterInfo(clusterInfo, seedReconfigurationKubeconfigRetention, sshKey, infraID, pullSecret, kubeadminPasswordHash, + proxy, + statusProxy, + installConfig, ) filePath := filepath.Join(clusterConfigPath, common.SeedReconfigurationFileName) diff --git a/internal/clusterconfig/clusterconfig_test.go b/internal/clusterconfig/clusterconfig_test.go index 207737e44..b770922ca 100644 --- a/internal/clusterconfig/clusterconfig_test.go +++ b/internal/clusterconfig/clusterconfig_test.go @@ -213,6 +213,7 @@ func TestClusterConfig(t *testing.T) { }, Spec: ocpV1.ImageDigestMirrorSetSpec{ImageDigestMirrors: []ocpV1.ImageDigestMirrors{{Source: "data"}}}, } + defaultProxy := &ocpV1.Proxy{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -220,7 +221,19 @@ func TestClusterConfig(t *testing.T) { Spec: ocpV1.ProxySpec{ HTTPProxy: "some-http-proxy", }, + Status: ocpV1.ProxyStatus{ + HTTPProxy: "some-http-proxy-status", + }, + } + + installConfig := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.InstallConfigCM, + Namespace: common.InstallConfigCMNamespace, + }, + Data: map[string]string{"install-config": clusterCmData}, } + testcases := []struct { testCaseName string pullSecret client.Object @@ -251,14 +264,6 @@ func TestClusterConfig(t *testing.T) { } manifestsDir := filepath.Join(clusterConfigPath, manifestDir) - // validate proxy - proxy := &ocpV1.Proxy{} - if err := utils.ReadYamlOrJSONFile(filepath.Join(manifestsDir, proxyFileName), proxy); err != nil { - t.Errorf("unexpected error: %v", err) - } - assert.Equal(t, proxyName, proxy.Name) - assert.Equal(t, "some-http-proxy", proxy.Spec.HTTPProxy) - // validate pull idms idms := &ocpV1.ImageDigestMirrorSetList{} if err := utils.ReadYamlOrJSONFile(filepath.Join(manifestsDir, idmsFileName), idms); err != nil { @@ -282,6 +287,8 @@ func TestClusterConfig(t *testing.T) { assert.Equal(t, "redhat.com", seedReconfig.BaseDomain) assert.Equal(t, "192.168.121.10", seedReconfig.NodeIP) assert.Equal(t, "mirror.redhat.com:5005", seedReconfig.ReleaseRegistry) + assert.Equal(t, "some-http-proxy", seedReconfig.Proxy.HTTPProxy) + assert.Equal(t, "some-http-proxy-status", seedReconfig.StatusProxy.HTTPProxy) }, }, { @@ -351,7 +358,7 @@ func TestClusterConfig(t *testing.T) { if err != nil { t.Errorf("unexpected error: %v", err) } - assert.Equal(t, 1, len(dir)) + assert.Equal(t, 0, len(dir)) }, }, { @@ -435,36 +442,13 @@ func TestClusterConfig(t *testing.T) { }, } - for _, tc := range testcases { - tmpDir := t.TempDir() - t.Run(tc.testCaseName, func(t *testing.T) { - installConfig := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: common.InstallConfigCM, - Namespace: common.InstallConfigCMNamespace, - }, - Data: map[string]string{"install-config": clusterCmData}, - } - objs := []client.Object{tc.pullSecret, tc.clusterVersion, installConfig, tc.node, - tc.idms, tc.proxy, tc.caBundleCM, csvDeployment, infrastructure} - - if !tc.deleteKubeadmin { - objs = append(objs, kubeadminSecret) - } - - for _, kcro := range kubeconfigRetentionObjects { - objs = append(objs, kcro) - } - - if tc.icsps != nil { - for _, icsp := range tc.icsps { - objs = append(objs, icsp) - } - } + for _, testCase := range testcases { + clusterConfigDir := t.TempDir() + t.Run(testCase.testCaseName, func(t *testing.T) { + hostPath = clusterConfigDir - hostPath = tmpDir - if tc.caBundleCM != nil { - dir := filepath.Join(tmpDir, filepath.Dir(common.CABundleFilePath)) + if testCase.caBundleCM != nil { + dir := filepath.Join(clusterConfigDir, filepath.Dir(common.CABundleFilePath)) if err := os.MkdirAll(dir, 0o700); err != nil { t.Errorf("unexpected error: %v", err) } @@ -476,44 +460,62 @@ func TestClusterConfig(t *testing.T) { _ = f.Close() } - dir := filepath.Join(tmpDir, filepath.Dir(sshKeyFile)) - if err := os.MkdirAll(dir, 0o700); err != nil { + sshKeyDir := filepath.Join(clusterConfigDir, filepath.Dir(sshKeyFile)) + if err := os.MkdirAll(sshKeyDir, 0o700); err != nil { t.Errorf("unexpected error: %v", err) } - if err := os.WriteFile(filepath.Join(tmpDir, sshKeyFile), []byte("ssh-key"), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(clusterConfigDir, sshKeyFile), []byte("ssh-key"), 0o600); err != nil { t.Errorf("unexpected error: %v", err) } - fakeClient, err := getFakeClientFromObjects(objs...) - if err != nil { - t.Errorf("error in creating fake client") + k8sResources := []client.Object{ + testCase.pullSecret, testCase.clusterVersion, installConfig, testCase.node, + testCase.idms, testCase.proxy, testCase.caBundleCM, csvDeployment, infrastructure, } - ucc := UpgradeClusterConfigGather{ - Client: fakeClient, - Log: logr.Discard(), - Scheme: fakeClient.Scheme(), + if !testCase.deleteKubeadmin { + k8sResources = append(k8sResources, kubeadminSecret) + } + + for _, kcro := range kubeconfigRetentionObjects { + k8sResources = append(k8sResources, kcro) + } + + if testCase.icsps != nil { + for _, icsp := range testCase.icsps { + k8sResources = append(k8sResources, icsp) + } + } + + fakeK8sClient, err := getFakeClientFromObjects(k8sResources...) + if err != nil { + t.Errorf("error in creating fake client") } - if err := os.MkdirAll(filepath.Join(tmpDir, common.OptOpenshift), 0o700); err != nil { + if err := os.MkdirAll(filepath.Join(clusterConfigDir, common.OptOpenshift), 0o700); err != nil { t.Errorf("failed to create opt dir, error: %v", err) } - if err := os.MkdirAll(filepath.Join(tmpDir, common.SeedDataDir), 0o700); err != nil { + if err := os.MkdirAll(filepath.Join(clusterConfigDir, common.SeedDataDir), 0o700); err != nil { t.Errorf("failed to create %s dir, error: %v", common.SeedDataDir, err) } - err = utils.MarshalToFile(seedManifestData, filepath.Join(tmpDir, common.SeedDataDir, common.SeedClusterInfoFileName)) + err = utils.MarshalToFile(seedManifestData, filepath.Join(clusterConfigDir, common.SeedDataDir, common.SeedClusterInfoFileName)) if err != nil { t.Errorf("failed to create seed manifest, error: %v", err) } - err = ucc.FetchClusterConfig(context.TODO(), tmpDir) - if !tc.expectedErr && err != nil { + ucc := UpgradeClusterConfigGather{ + Client: fakeK8sClient, + Log: logr.Discard(), + Scheme: fakeK8sClient.Scheme(), + } + err = ucc.FetchClusterConfig(context.TODO(), clusterConfigDir) + if !testCase.expectedErr && err != nil { t.Errorf("unexpected error: %v", err) } - if tc.expectedErr && err == nil { + if testCase.expectedErr && err == nil { t.Errorf("expected error but it didn't happened") } - tc.validateFunc(t, tmpDir, err, ucc) + testCase.validateFunc(t, clusterConfigDir, err, ucc) }) } } diff --git a/internal/common/consts.go b/internal/common/consts.go index d9b22d37a..69d80a677 100644 --- a/internal/common/consts.go +++ b/internal/common/consts.go @@ -75,7 +75,10 @@ const ( InstallConfigCM = "cluster-config-v1" // InstallConfigCMNamespace cm namespace InstallConfigCMNamespace = "kube-system" - OpenshiftInfraCRName = "cluster" + // InstallConfigCMNamespace data key + InstallConfigCMInstallConfigDataKey = "install-config" + OpenshiftInfraCRName = "cluster" + OpenshiftProxyCRName = "cluster" // Env var to configure auto rollback for post-reboot config failure IBUPostRebootConfigAutoRollbackOnFailureEnv = "LCA_IBU_AUTO_ROLLBACK_ON_CONFIG_FAILURE" @@ -84,6 +87,8 @@ const ( SeedFormatVersion = 3 SeedFormatOCILabel = "com.openshift.lifecycle-agent.seed_format_version" + SeedClusterInfoOCILabel = "com.openshift.lifecycle-agent.seed_cluster_info" + PullSecretName = "pull-secret" PullSecretEmptyData = "{\"auths\":{\"registry.connect.redhat.com\":{\"username\":\"empty\",\"password\":\"empty\",\"auth\":\"ZW1wdHk6ZW1wdHk=\",\"email\":\"\"}}}" //nolint:gosec OpenshiftConfigNamespace = "openshift-config" diff --git a/internal/recert/recert.go b/internal/recert/recert.go index de081bfd2..b56bdb12d 100644 --- a/internal/recert/recert.go +++ b/internal/recert/recert.go @@ -29,6 +29,8 @@ type RecertConfig struct { ClusterRename string `json:"cluster_rename,omitempty"` Hostname string `json:"hostname,omitempty"` IP string `json:"ip,omitempty"` + Proxy string `json:"proxy,omitempty"` + InstallConfig string `json:"install_config,omitempty"` // We intentionally don't omitEmpty this field because an empty string here // means "delete the kubeadmin password secret" while a complete omission // of the field means "don't touch the secret". We never want the latter, @@ -44,6 +46,17 @@ type RecertConfig struct { PullSecret string `json:"pull_secret,omitempty"` } +func FormatRecertProxyFromSeedReconfigProxy(proxy, statusProxy *seedreconfig.Proxy) string { + if proxy == nil || statusProxy == nil { + // Both must be set, anything else is invalid + return "" + } + return fmt.Sprintf("%s|%s|%s|%s|%s|%s", + proxy.HTTPProxy, proxy.HTTPSProxy, proxy.NoProxy, + statusProxy.HTTPProxy, statusProxy.HTTPSProxy, statusProxy.NoProxy, + ) +} + // CreateRecertConfigFile function to create recert config file // those params will be provided to an installation script after reboot // that will run recert command with them @@ -63,6 +76,10 @@ func CreateRecertConfigFile(seedReconfig *seedreconfig.SeedReconfiguration, seed config.IP = seedReconfig.NodeIP } + config.Proxy = FormatRecertProxyFromSeedReconfigProxy(seedReconfig.Proxy, seedReconfig.StatusProxy) + + config.InstallConfig = seedReconfig.InstallConfig + config.SummaryFile = SummaryFile seedFullDomain := fmt.Sprintf("%s.%s", seedClusterInfo.ClusterName, seedClusterInfo.BaseDomain) clusterFullDomain := fmt.Sprintf("%s.%s", seedReconfig.ClusterName, seedReconfig.BaseDomain) @@ -163,7 +180,10 @@ func createBasicEmptyRecertConfig() RecertConfig { DryRun: false, EtcdEndpoint: common.EtcdDefaultEndpoint, StaticDirs: staticDirs, - StaticFiles: []string{"/host-etc/mcs-machine-config-content.json"}, + StaticFiles: []string{ + "/host-etc/mcs-machine-config-content.json", + "/host-etc/mco/proxy.env", + }, } } diff --git a/lca-cli/postpivot/postpivot.go b/lca-cli/postpivot/postpivot.go index 7e66823bf..77c877ae2 100644 --- a/lca-cli/postpivot/postpivot.go +++ b/lca-cli/postpivot/postpivot.go @@ -94,13 +94,13 @@ func (p *PostPivot) PostPivotConfiguration(ctx context.Context) error { p.log.Info("Reading seed reconfiguration info") seedReconfiguration, err := utils.ReadSeedReconfigurationFromFile( - path.Join(p.workingDir, common.ClusterConfigDir, common.SeedClusterInfoFileName)) + path.Join(p.workingDir, common.ClusterConfigDir, common.SeedReconfigurationFileName)) if err != nil { return fmt.Errorf("failed to get cluster info from %s, err: %w", "", err) } if err := utils.RunOnce("setSSHKey", p.workingDir, p.log, p.setSSHKey, - seedReconfiguration, sshKeyEarlyAccessFile); err != nil { + seedReconfiguration.SSHKey, sshKeyEarlyAccessFile); err != nil { return fmt.Errorf("failed to run once setSSHKey for post pivot: %w", err) } @@ -573,14 +573,14 @@ func (p *PostPivot) setNodeIPIfNotProvided(ctx context.Context, // setSSHKey sets ssh public key provided by user in 2 operations: // 1. as file in order to give early access to the node // 2. creates 2 machine configs in manifests dir that will be applied when cluster is up -func (p *PostPivot) setSSHKey(seedReconfiguration *clusterconfig_api.SeedReconfiguration, sshKeyFile string) error { - if seedReconfiguration.SSHKey == "" { +func (p *PostPivot) setSSHKey(sshKey, sshKeyFile string) error { + if sshKey == "" { p.log.Infof("No ssh public key was provided, skipping") return nil } p.log.Infof("Creating file %s with ssh keys for early connection", sshKeyFile) - if err := os.WriteFile(sshKeyFile, []byte(seedReconfiguration.SSHKey), 0o600); err != nil { + if err := os.WriteFile(sshKeyFile, []byte(sshKey), 0o600); err != nil { return fmt.Errorf("failed to write ssh key to file, err %w", err) } @@ -589,7 +589,7 @@ func (p *PostPivot) setSSHKey(seedReconfiguration *clusterconfig_api.SeedReconfi return fmt.Errorf("failed to set %s user ownership on %s, err :%w", userCore, sshKeyFile, err) } - return p.createSSHKeyMachineConfigs(seedReconfiguration.SSHKey) + return p.createSSHKeyMachineConfigs(sshKey) } func (p *PostPivot) createSSHKeyMachineConfigs(sshKey string) error { diff --git a/lca-cli/seedclusterinfo/seedclusterinfo.go b/lca-cli/seedclusterinfo/seedclusterinfo.go index a19a7e529..097fda0f9 100644 --- a/lca-cli/seedclusterinfo/seedclusterinfo.go +++ b/lca-cli/seedclusterinfo/seedclusterinfo.go @@ -60,9 +60,17 @@ type SeedClusterInfo struct { // certificates, so it has already proven to run successfully on the seed // data). RecertImagePullSpec string `json:"recert_image_pull_spec,omitempty"` + + // Whether the seed has a proxy configured or not. Seed clusters without a + // proxy cannot be used to upgrade or install clusters with a proxy. So + // whether or not the seed has a proxy configured is important for LCA to + // know so it can optionally deny the upgrade or installation of clusters + // with a proxy from seeds that don't have one. Similarly, installing a + // cluster without a proxy from a seed with a proxy is also not supported. + HasProxy bool `json:"has_proxy"` } -func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, seedImagePullSpec string) *SeedClusterInfo { +func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, seedImagePullSpec string, hasProxy bool) *SeedClusterInfo { return &SeedClusterInfo{ SeedClusterOCPVersion: clusterInfo.OCPVersion, BaseDomain: clusterInfo.BaseDomain, @@ -72,6 +80,7 @@ func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, seedImagePullSpec string SNOHostname: clusterInfo.Hostname, MirrorRegistryConfigured: clusterInfo.MirrorRegistryConfigured, RecertImagePullSpec: seedImagePullSpec, + HasProxy: hasProxy, } } diff --git a/lca-cli/seedcreator/seedcreator.go b/lca-cli/seedcreator/seedcreator.go index 22717977a..724717216 100644 --- a/lca-cli/seedcreator/seedcreator.go +++ b/lca-cli/seedcreator/seedcreator.go @@ -147,7 +147,13 @@ func (s *SeedCreator) CreateSeedImage() error { return fmt.Errorf("failed to run once backup_mco_config: %w", err) } - if err := s.createAndPushSeedImage(); err != nil { + clusterInfoJSONSBytes, err := os.ReadFile(path.Join(s.backupDir, common.SeedClusterInfoFileName)) + if err != nil { + return fmt.Errorf("failed to read cluster info file: %w", err) + } + + clusterInfoJSON := string(clusterInfoJSONSBytes) + if err := s.createAndPushSeedImage(clusterInfoJSON); err != nil { return fmt.Errorf("failed to create and push seed image: %w", err) } @@ -184,7 +190,12 @@ func (s *SeedCreator) gatherClusterInfo(ctx context.Context) error { return fmt.Errorf("failed to get cluster info: %w", err) } - seedClusterInfo := seedclusterinfo.NewFromClusterInfo(clusterInfo, s.recertContainerImage) + hasProxy, err := utils.HasProxy(ctx, s.client) + if err != nil { + return fmt.Errorf("failed to get proxy information: %w", err) + } + + seedClusterInfo := seedclusterinfo.NewFromClusterInfo(clusterInfo, s.recertContainerImage, hasProxy) if err := os.MkdirAll(common.SeedDataDir, os.ModePerm); err != nil { return fmt.Errorf("error creating SeedDataDir %s: %w", common.SeedDataDir, err) @@ -393,7 +404,7 @@ func (s *SeedCreator) backupMCOConfig() error { } // Building and pushing OCI image -func (s *SeedCreator) createAndPushSeedImage() error { +func (s *SeedCreator) createAndPushSeedImage(clusterInfo string) error { s.log.Info("Build and push OCI image to ", s.containerRegistry) s.log.Debug(s.ostreeClient.RpmOstreeVersion()) // If verbose, also dump out current rpm-ostree version available @@ -426,6 +437,7 @@ func (s *SeedCreator) createAndPushSeedImage() error { "--file", tmpfile.Name(), "--tag", s.containerRegistry, "--label", fmt.Sprintf("%s=%d", common.SeedFormatOCILabel, common.SeedFormatVersion), + "--label", fmt.Sprintf("%s=%s", common.SeedClusterInfoOCILabel, clusterInfo), s.backupDir, } _, err = s.ops.RunInHostNamespace( diff --git a/utils/client_helper.go b/utils/client_helper.go index 65236f143..02314b7f5 100644 --- a/utils/client_helper.go +++ b/utils/client_helper.go @@ -243,6 +243,19 @@ func GetMirrorRegistrySourceRegistries(ctx context.Context, client runtimeclient return sourceRegistries, nil } +func HasProxy(ctx context.Context, client runtimeclient.Client) (bool, error) { + proxy := &ocp_config_v1.Proxy{} + if err := client.Get(ctx, types.NamespacedName{Name: common.OpenshiftProxyCRName}, proxy); err != nil { + return false, fmt.Errorf("failed to get proxy CR: %w", err) + } + + if proxy.Spec.HTTPProxy == "" && proxy.Spec.HTTPSProxy == "" && proxy.Spec.NoProxy == "" { + return false, nil + } + + return true, nil +} + func ShouldOverrideSeedRegistry(ctx context.Context, client runtimeclient.Client, mirrorRegistryConfigured bool, releaseRegistry string) (bool, error) { mirroredRegistries, err := GetMirrorRegistrySourceRegistries(ctx, client) if err != nil {