Skip to content

Commit

Permalink
Remove use of lister from apply roles (#1028)
Browse files Browse the repository at this point in the history
* Removed need to have ports in components. In progress

* Corrected radix env sync for radix application sync

* Fixed reported issues

* Fixed reported issues
  • Loading branch information
satr authored Jan 3, 2024
1 parent 4385d9c commit 325a0cb
Show file tree
Hide file tree
Showing 10 changed files with 130 additions and 129 deletions.
4 changes: 2 additions & 2 deletions charts/radix-operator/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: v2
name: radix-operator
version: 1.27.6
appVersion: 1.47.6
version: 1.27.8
appVersion: 1.47.8
kubeVersion: ">=1.24.0"
description: Radix Operator
keywords:
Expand Down
105 changes: 1 addition & 104 deletions pkg/apis/applicationconfig/applicationconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package applicationconfig

import (
"context"
"encoding/json"
stderrors "errors"
"fmt"
"reflect"
"strings"
Expand All @@ -15,11 +13,9 @@ import (
"github.com/equinor/radix-operator/pkg/apis/utils"
"github.com/equinor/radix-operator/pkg/apis/utils/branch"
radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned"
radixTypes "github.com/equinor/radix-operator/pkg/client/clientset/versioned/typed/radix/v1"
log "github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/client-go/kubernetes"
)

Expand Down Expand Up @@ -126,7 +122,7 @@ func (app *ApplicationConfig) ApplyConfigToApplicationNamespace() error {
// It compares the actual state with the desired, and attempts to
// converge the two
func (app *ApplicationConfig) OnSync() error {
if err := app.createEnvironments(); err != nil {
if err := app.syncEnvironments(); err != nil {
log.Errorf("Failed to create namespaces for app environments %s. %v", app.config.Name, err)
return err
}
Expand All @@ -144,102 +140,3 @@ func (app *ApplicationConfig) OnSync() error {
}
return app.syncSubPipelineServiceAccounts()
}

func (app *ApplicationConfig) createEnvironments() error {
var errs []error
for _, env := range app.config.Spec.Environments {
err := app.applyEnvironment(utils.NewEnvironmentBuilder().
WithAppName(app.config.Name).
WithAppLabel().
WithEnvironmentName(env.Name).
WithRegistrationOwner(app.registration).
WithEgressConfig(env.Egress).
// Orphaned flag will be set by the environment handler but until
// reconciliation we must ensure it is false
// Update: It seems Update method does not update status object when using real k8s client, but the fake client does.
// Only an explicit call to UpdateStatus can update status object, and this is only done by the RadixEnvironment controller.
WithOrphaned(false).
BuildRE())
if err != nil {
errs = append(errs, err)
}
}
return stderrors.Join(errs...)
}

// applyEnvironment creates an environment or applies changes if it exists
func (app *ApplicationConfig) applyEnvironment(newRe *radixv1.RadixEnvironment) error {
logger := log.WithFields(log.Fields{"environment": newRe.ObjectMeta.Name})
logger.Debugf("Apply environment %s", newRe.Name)

repository := app.radixclient.RadixV1().RadixEnvironments()

// Get environment from cache, instead than for cluster
oldRe, err := app.kubeutil.GetEnvironment(newRe.Name)
if err != nil && errors.IsNotFound(err) {
// Environment does not exist yet

newRe, err = repository.Create(context.TODO(), newRe, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create RadixEnvironment object: %v", err)
}
logger.Debugf("Created RadixEnvironment: %s", newRe.Name)

} else if err != nil {
return fmt.Errorf("failed to get RadixEnvironment object: %v", err)

} else {
// Environment already exists

logger.Debugf("RadixEnvironment object %s already exists, updating the object now", oldRe.Name)
err = patchDifference(repository, oldRe, newRe, logger)
if err != nil {
return err
}
}
return nil
}

// patchDifference creates a mergepatch, comparing old and new RadixEnvironments and issues the patch to radix
func patchDifference(repository radixTypes.RadixEnvironmentInterface, oldRe *radixv1.RadixEnvironment, newRe *radixv1.RadixEnvironment, logger *log.Entry) error {
radixEnvironment := oldRe.DeepCopy()
radixEnvironment.ObjectMeta.Labels = newRe.ObjectMeta.Labels
radixEnvironment.ObjectMeta.OwnerReferences = newRe.ObjectMeta.OwnerReferences
radixEnvironment.ObjectMeta.Annotations = newRe.ObjectMeta.Annotations
radixEnvironment.ObjectMeta.Finalizers = newRe.ObjectMeta.Finalizers
radixEnvironment.ObjectMeta.DeletionTimestamp = newRe.ObjectMeta.DeletionTimestamp
radixEnvironment.Spec = newRe.Spec
radixEnvironment.Status = newRe.Status

oldReJSON, err := json.Marshal(oldRe)
if err != nil {
return fmt.Errorf("failed to marshal old RadixEnvironment object: %v", err)
}

radixEnvironmentJSON, err := json.Marshal(radixEnvironment)
if err != nil {
return fmt.Errorf("failed to marshal new RadixEnvironment object: %v", err)
}

patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldReJSON, radixEnvironmentJSON, radixv1.RadixEnvironment{})
if err != nil {
return fmt.Errorf("failed to create patch document for RadixEnvironment object: %v", err)
}

if !isEmptyPatch(patchBytes) {
// Will perform update as patching does not seem to work for this custom resource
patchedEnvironment, err := repository.Update(context.TODO(), radixEnvironment, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("failed to patch RadixEnvironment object: %v", err)
}
logger.Debugf("Patched RadixEnvironment: %s", patchedEnvironment.Name)
} else {
logger.Debugf("No need to patch RadixEnvironment: %s ", newRe.Name)
}

return nil
}

func isEmptyPatch(patchBytes []byte) bool {
return string(patchBytes) == "{}"
}
7 changes: 5 additions & 2 deletions pkg/apis/applicationconfig/applicationconfig_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
kubefake "k8s.io/client-go/kubernetes/fake"
secretproviderfake "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/fake"
Expand Down Expand Up @@ -87,7 +88,8 @@ func Test_Reconciles_Radix_Environments(t *testing.T) {
context.TODO(),
&radixv1.RadixEnvironment{
ObjectMeta: metav1.ObjectMeta{
Name: "any-app-qa",
Name: "any-app-qa",
Labels: labels.Set{kube.RadixAppLabel: "any-app"},
},
},
metav1.CreateOptions{})
Expand All @@ -97,7 +99,8 @@ func Test_Reconciles_Radix_Environments(t *testing.T) {
context.TODO(),
&radixv1.RadixEnvironment{
ObjectMeta: metav1.ObjectMeta{
Name: "any-app-prod",
Name: "any-app-prod",
Labels: labels.Set{kube.RadixAppLabel: "any-app"},
},
},
metav1.CreateOptions{})
Expand Down
84 changes: 84 additions & 0 deletions pkg/apis/applicationconfig/environment.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package applicationconfig

import (
"context"
stderrors "errors"
"fmt"

radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
"github.com/equinor/radix-operator/pkg/apis/utils"
log "github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
)

func (app *ApplicationConfig) syncEnvironments() error {
var errs []error
for _, env := range app.config.Spec.Environments {
if err := app.syncEnvironment(app.buildRadixEnvironment(env)); err != nil {
errs = append(errs, err)
}
}
return stderrors.Join(errs...)
}

func (app *ApplicationConfig) buildRadixEnvironment(env radixv1.Environment) *radixv1.RadixEnvironment {
return utils.NewEnvironmentBuilder().
WithAppName(app.config.Name).
WithAppLabel().
WithEnvironmentName(env.Name).
WithRegistrationOwner(app.registration).
WithEgressConfig(env.Egress).
BuildRE()
}

// syncEnvironment creates an environment or applies changes if it exists
func (app *ApplicationConfig) syncEnvironment(radixEnvironment *radixv1.RadixEnvironment) error {
logger := log.WithFields(log.Fields{"environment": radixEnvironment.GetName()})
logger.Debugf("Apply RadixEnvironment")
if _, err := app.getRadixEnvironment(radixEnvironment.GetName()); err != nil {
if errors.IsNotFound(err) {
return app.createRadixEnvironment(radixEnvironment, logger)
}
return fmt.Errorf("failed to get RadixEnvironment: %v", err)
}
return app.updateRadixEnvironment(radixEnvironment, logger)
}

func (app *ApplicationConfig) createRadixEnvironment(radixEnvironment *radixv1.RadixEnvironment, logger *log.Entry) error {
created, err := app.radixclient.RadixV1().RadixEnvironments().Create(context.Background(), radixEnvironment, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create RadixEnvironment: %v", err)
}
logger.Debugf("Created RadixEnvironment (revision %s)", created.GetResourceVersion())
return nil
}

// updateRadixEnvironment updates a RadixEnvironment
func (app *ApplicationConfig) updateRadixEnvironment(radixEnvironment *radixv1.RadixEnvironment, logger *log.Entry) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
existingRE, err := app.getRadixEnvironment(radixEnvironment.GetName())
if err != nil {
if errors.IsNotFound(err) {
return nil
}
return err
}
logger.Debugf("re-taken RadixEnvironment (revision %s)", existingRE.GetResourceVersion())

newRE := existingRE.DeepCopy()
newRE.Spec = radixEnvironment.Spec
// Will perform update as patching does not seem to work for this custom resource
updated, err := app.kubeutil.UpdateRadixEnvironment(newRE)
if err != nil {
return err
}
logger.Debugf("Updated RadixEnvironment (revision %s)", updated.GetResourceVersion())
return nil
})
}

func (app *ApplicationConfig) getRadixEnvironment(name string) (*radixv1.RadixEnvironment, error) {
return app.kubeutil.RadixClient().RadixV1().RadixEnvironments().Get(context.Background(), name, metav1.GetOptions{})
}
17 changes: 15 additions & 2 deletions pkg/apis/environment/environment.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package environment

import (
"context"
"fmt"

"github.com/equinor/radix-common/utils/slice"
Expand All @@ -14,6 +15,7 @@ import (
radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned"
"github.com/sirupsen/logrus"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -103,11 +105,22 @@ func (env *Environment) handleDeletedRadixEnvironment(re *v1.RadixEnvironment) e
if err := env.handleDeletedRadixEnvironmentDependencies(re); err != nil {
return err
}
updatingRE := re.DeepCopy()
updatingRE, err := env.radixclient.RadixV1().RadixEnvironments().Get(context.Background(), re.GetName(), metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return nil
}
return err
}
updatingRE.ObjectMeta.Finalizers = append(re.ObjectMeta.Finalizers[:finalizerIndex], re.ObjectMeta.Finalizers[finalizerIndex+1:]...)
logrus.Debugf("removed finalizer %s from the Radix environment %s in the application %s. Left finalizers: %d",
kube.RadixEnvironmentFinalizer, updatingRE.Name, updatingRE.Spec.AppName, len(updatingRE.ObjectMeta.Finalizers))
return env.kubeutil.UpdateRadixEnvironment(updatingRE)
updated, err := env.kubeutil.UpdateRadixEnvironment(updatingRE)
if err != nil {
return err
}
logrus.Debugf("updated RadixEnvironment %s revision %s", re.Name, updated.GetResourceVersion())
return nil
}

func (env *Environment) handleDeletedRadixEnvironmentDependencies(re *v1.RadixEnvironment) error {
Expand Down
16 changes: 10 additions & 6 deletions pkg/apis/environment/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,21 @@ func (env *Environment) syncStatus(re *radixv1.RadixEnvironment, time metav1.Tim
func (env *Environment) updateRadixEnvironmentStatus(re *radixv1.RadixEnvironment, changeStatusFunc func(currStatus *radixv1.RadixEnvironmentStatus)) error {
radixEnvironmentInterface := env.radixclient.RadixV1().RadixEnvironments()
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
currentEnv, err := radixEnvironmentInterface.Get(context.Background(), re.GetName(), metav1.GetOptions{})
name := re.GetName()
currentEnv, err := radixEnvironmentInterface.Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return err
}
changeStatusFunc(&currentEnv.Status)
_, err = radixEnvironmentInterface.UpdateStatus(context.Background(), currentEnv, metav1.UpdateOptions{})
if err == nil && env.config.GetName() == re.GetName() {
currentEnv, err = radixEnvironmentInterface.Get(context.Background(), re.GetName(), metav1.GetOptions{})
if err == nil {
env.config = currentEnv
updated, err := radixEnvironmentInterface.UpdateStatus(context.Background(), currentEnv, metav1.UpdateOptions{})
if err == nil && env.config.GetName() == name {
currentEnv, err = radixEnvironmentInterface.Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
return err
}
env.config = currentEnv
env.logger.Debugf("updated status of RadixEnvironment (revision %s)", updated.GetResourceVersion())
return nil
}
return err
})
Expand Down
14 changes: 7 additions & 7 deletions pkg/apis/kube/radix_environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@ func (kubeutil *Kube) ListEnvironments() ([]*radixv1.RadixEnvironment, error) {
return environments, nil
}

// UpdateRadixEnvironment Updates changes of Radix environment if any
func (kubeutil *Kube) UpdateRadixEnvironment(radixEnvironment *radixv1.RadixEnvironment) error {
log.Debugf("Update Radix environment %s in the application %s", radixEnvironment.Name, radixEnvironment.Spec.AppName)
_, err := kubeutil.RadixClient().RadixV1().RadixEnvironments().Update(context.TODO(), radixEnvironment, metav1.UpdateOptions{})
// UpdateRadixEnvironment Updates changes of RadixEnvironment if any
func (kubeutil *Kube) UpdateRadixEnvironment(radixEnvironment *radixv1.RadixEnvironment) (*radixv1.RadixEnvironment, error) {
log.Debugf("Update RadixEnvironment %s in the application %s", radixEnvironment.Name, radixEnvironment.Spec.AppName)
updated, err := kubeutil.RadixClient().RadixV1().RadixEnvironments().Update(context.TODO(), radixEnvironment, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("failed to patch Radix environment object: %v", err)
return nil, fmt.Errorf("failed to update RadixEnvironment object: %v", err)
}
log.Debugf("Updated Radix environment: %s in the application %s", radixEnvironment.Name, radixEnvironment.Spec.AppName)
return err
log.Debugf("Updated RadixEnvironment: %s in the application %s", radixEnvironment.Name, radixEnvironment.Spec.AppName)
return updated, nil
}
2 changes: 1 addition & 1 deletion pkg/apis/kube/rolebinding.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ func (kubeutil *Kube) ApplyRoleBinding(namespace string, role *rbacv1.RoleBindin
func (kubeutil *Kube) ApplyClusterRoleBinding(clusterrolebinding *rbacv1.ClusterRoleBinding) error {
logger = logger.WithFields(log.Fields{"clusterRoleBinding": clusterrolebinding.ObjectMeta.Name})
logger.Debugf("Apply clusterrolebinding %s", clusterrolebinding.Name)
oldClusterRoleBinding, err := kubeutil.GetClusterRoleBinding(clusterrolebinding.Name)
oldClusterRoleBinding, err := kubeutil.kubeClient.RbacV1().ClusterRoleBindings().Get(context.TODO(), clusterrolebinding.Name, metav1.GetOptions{})
if err != nil && errors.IsNotFound(err) {
createdClusterRoleBinding, err := kubeutil.kubeClient.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterrolebinding, metav1.CreateOptions{})
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/kube/roles.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (kubeutil *Kube) ApplyRole(namespace string, role *rbacv1.Role) error {
// ApplyClusterRole Creates or updates cluster-role
func (kubeutil *Kube) ApplyClusterRole(clusterrole *rbacv1.ClusterRole) error {
logger.Debugf("Apply clusterrole %s", clusterrole.Name)
oldClusterRole, err := kubeutil.GetClusterRole(clusterrole.GetName())
oldClusterRole, err := kubeutil.kubeClient.RbacV1().ClusterRoles().Get(context.TODO(), clusterrole.GetName(), metav1.GetOptions{})
if err != nil && errors.IsNotFound(err) {
createdClusterRole, err := kubeutil.kubeClient.RbacV1().ClusterRoles().Create(context.TODO(), clusterrole, metav1.CreateOptions{})
if err != nil {
Expand Down
8 changes: 4 additions & 4 deletions radix-operator/environment/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,11 @@ func NewController(client kubernetes.Interface,
oldRR := old.(*v1.RadixEnvironment)

if deepEqual(oldRR, newRR) {
logger.Debugf("Environment object is equal to old for %s. Do nothing", newRR.GetName())
logger.Debugf("RadixEnvironment %s (revision %s) is equal to old (revision %s). Do nothing", newRR.GetName(), newRR.GetResourceVersion(), oldRR.GetResourceVersion())
metrics.CustomResourceUpdatedButSkipped(crType)
return
}

logger.Debugf("update RadixEnvironment %s (from revision %s to %s)", oldRR.GetName(), oldRR.GetResourceVersion(), newRR.GetResourceVersion())
if _, err := controller.Enqueue(cur); err != nil {
utilruntime.HandleError(err)
}
Expand All @@ -89,12 +89,12 @@ func NewController(client kubernetes.Interface,
DeleteFunc: func(obj interface{}) {
radixEnvironment, converted := obj.(*v1.RadixEnvironment)
if !converted {
logger.Errorf("RadixEnvironment object cast failed during deleted event received.")
logger.Errorf("RadixEnvironment object cast failed during deleted event received")
return
}
key, err := cache.MetaNamespaceKeyFunc(radixEnvironment)
if err == nil {
logger.Debugf("Environment object deleted event received for %s. Do nothing", key)
logger.Debugf("RadixEnvironment object deleted event received for %s (revision %s). Do nothing", key, radixEnvironment.GetResourceVersion())
}
metrics.CustomResourceDeleted(crType)
},
Expand Down

0 comments on commit 325a0cb

Please sign in to comment.