ROX-26039: Create tenant argoCd app

Makefile

@@ -920,7 +920,7 @@ deploy/dev-fast/fleet-manager: image/build
 
 deploy/dev-fast/fleetshard-sync: image/build
 	kubectl -n $(NAMESPACE) set image deploy/fleetshard-sync fleetshard-sync=$(SHORT_IMAGE_REF)
-	kubectl -n $(NAMESPACE) delete pod -l application=fleetshard-sync
+	kubectl -n $(NAMESPACE) delete pod -l app=fleetshard-sync
 
 deploy/probe: IMAGE_REGISTRY?="$(external_image_registry)"
 deploy/probe: IMAGE_REPOSITORY?="$(probe_image_repository)"

dev/config/gitops-config.yaml

@@ -21,6 +21,8 @@ tenantResources:
       rhacs.redhat.com/org-name: "{{ .OrganizationName }}"
     secureTenantNetwork: false
     centralRdsCidrBlock: "10.1.0.0/16"
+    argoCd:
+      enabled: true
     verticalPodAutoscalers:
       central:
         enabled: true

dev/env/manifests/fleetshard-operator/28-deployment.yaml

@@ -32,8 +32,6 @@ spec:
                 fieldPath: metadata.namespace
           - name: ADDON_NAME
             value: acs-fleetshard-dev
-          - name: ARGOCD_NAMESPACE
-            value: "${ARGOCD_NAMESPACE}"
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

dev/env/manifests/fleetshard-operator/51-fleetshard-cr.yaml

@@ -15,6 +15,7 @@ spec:
     environment: "dev"
     fleetManagerEndpoint: "http://fleet-manager:8000"
     secureTenantNetwork: $SECURE_TENANT_NETWORK
+    argoCdNamespace: $ARGOCD_NAMESPACE
     managedDB:
       enabled: $MANAGED_DB_ENABLED
       subnetGroup: "$MANAGED_DB_SUBNET_GROUP"

dev/env/scripts/bootstrap.sh

@@ -58,12 +58,16 @@ if [[ "$INSTALL_ARGOCD" == "true" ]]; then
     kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
 elif [[ "$INSTALL_OPENSHIFT_GITOPS" == "true" ]]; then
     log "Installing Openshift GitOps"
+    kubectl create ns openshift-gitops || true
     apply "${MANIFESTS_DIR}/openshift-gitops"
 else
     log "One of ArgoCD or OpenShift GitOps must be installed"
     exit 1
 fi
 
+log "Installing acscs-manifests repository secret"
+"${GITROOT}/scripts/import_argocd_repo_secret.sh"
+
 # skip manifests if openshift cluster using is_openshift_cluster
 if ! is_openshift_cluster "$CLUSTER_TYPE"; then
     apply "${MANIFESTS_DIR}/monitoring"

fleetshard/config/config.go

@@ -35,13 +35,17 @@ type Config struct {
 	// If it is empty, nothing is injected (for example, it is not required when running on OpenShift).
 	// It is required when central images need to fetched from a private Quay registry.
 	// It needs to given as Docker Config JSON object.
-	TenantImagePullSecret string `env:"TENANT_IMAGE_PULL_SECRET"`
-	ManagedDB             ManagedDB
-	Telemetry             Telemetry
-	AuditLogging          AuditLogging
-	SecretEncryption      SecretEncryption
-	RouteParameters       RouteConfig
-	FleetshardAddonName   string `env:"FLEETSHARD_ADDON_NAME" envDefault:"acs-fleetshard"`
+	TenantImagePullSecret               string `env:"TENANT_IMAGE_PULL_SECRET"`
+	DefaultTenantArgoCdAppSourceRepoURL string `env:"DEFAULT_TENANT_ARGOCD_APP_SOURCE_REPO_URL" envDefault:"https://github.com/stackrox/acscs-manifests.git"`
+	DefaultTenantArgoCdAppSourceRef     string `env:"DEFAULT_TENANT_ARGOCD_APP_SOURCE_REF" envDefault:"HEAD"`
+	DefaultTenantArgoCdAppSourcePath    string `env:"DEFAULT_TENANT_ARGOCD_APP_SOURCE_PATH" envDefault:"tenant-resources"`
+	ArgoCdNamespace                     string `env:"ARGOCD_NAMESPACE" envDefault:"openshift-gitops"`
+	ManagedDB                           ManagedDB
+	Telemetry                           Telemetry
+	AuditLogging                        AuditLogging
+	SecretEncryption                    SecretEncryption
+	RouteParameters                     RouteConfig
+	FleetshardAddonName                 string `env:"FLEETSHARD_ADDON_NAME" envDefault:"acs-fleetshard"`
 
 	// The SecureTenantNetwork option controls whether the Tenant's K8s
 	// Namespace will be secured at the network level, e.g. by using

fleetshard/pkg/central/reconciler/reconciler.go

@@ -13,6 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	argocd "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
 	"github.com/golang/glog"
 	"github.com/hashicorp/go-multierror"
 	openshiftRouteV1 "github.com/openshift/api/route/v1"
@@ -40,6 +41,7 @@ import (
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/pointer"
@@ -115,16 +117,20 @@ type encryptedSecrets struct {
 
 // CentralReconcilerOptions are the static options for creating a reconciler.
 type CentralReconcilerOptions struct {
-	UseRoutes             bool
-	WantsAuthProvider     bool
-	ManagedDBEnabled      bool
-	Telemetry             config.Telemetry
-	ClusterName           string
-	Environment           string
-	AuditLogging          config.AuditLogging
-	TenantImagePullSecret string
-	RouteParameters       config.RouteConfig
-	SecureTenantNetwork   bool
+	UseRoutes                           bool
+	WantsAuthProvider                   bool
+	ManagedDBEnabled                    bool
+	Telemetry                           config.Telemetry
+	ClusterName                         string
+	Environment                         string
+	AuditLogging                        config.AuditLogging
+	TenantImagePullSecret               string
+	RouteParameters                     config.RouteConfig
+	SecureTenantNetwork                 bool
+	DefaultTenantArgoCdAppSourceRepoURL string
+	DefaultTenantArgoCdAppSourceRef     string
+	DefaultTenantArgoCdAppSourcePath    string
+	ArgoCdNamespace                     string
 }
 
 // CentralReconciler is a reconciler tied to a one Central instance. It installs, updates and deletes Central instances
@@ -163,6 +169,11 @@ type CentralReconciler struct {
 	areSecretsStoredFunc      areSecretsStoredFunc
 	needsReconcileFunc        needsReconcileFunc
 	restoreCentralSecretsFunc restoreCentralSecretsFunc
+
+	defaultTenantArgoCdAppSourceRepoURL string
+	defaultTenantArgoCdAppSourceRef     string
+	defaultTenantArgoCdAppSourcePath    string
+	argoCdNamespace                     string
 }
 
 // Reconcile takes a private.ManagedCentral and tries to install it into the cluster managed by the fleet-shard.
@@ -236,8 +247,27 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 		return nil, err
 	}
 
-	if err := r.ensureChartResourcesExist(ctx, remoteCentral); err != nil {
-		return nil, errors.Wrapf(err, "unable to install chart resource for central %s/%s", central.GetNamespace(), central.GetName())
+	if isArgoCdEnabledForTenant(remoteCentral) {
+		if err := r.ensureArgoCdApplicationExists(ctx, remoteCentral); err != nil {
+			return nil, errors.Wrapf(err, "unable to install ArgoCD application for central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name)
+		}
+	} else {
+
+		{
+			// This part handles the case where we enable, then disable ArgoCD for a tenant
+			// to make sure the ArgoCD application is cleaned up.
+			ok, err := r.ensureArgoCdApplicationDeleted(ctx, remoteCentral)
+			if err != nil {
+				return nil, errors.Wrapf(err, "unable to delete ArgoCD application for central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name)
+			}
+			if !ok {
+				return nil, errors.New("ArgoCD application not yet deleted")
+			}
+		}
+
+		if err := r.ensureChartResourcesExist(ctx, remoteCentral); err != nil {
+			return nil, errors.Wrapf(err, "unable to install chart resource for central %s/%s", central.GetNamespace(), central.GetName())
+		}
 	}
 
 	if err = r.reconcileCentralDBConfig(ctx, &remoteCentral, central); err != nil {
@@ -308,6 +338,30 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 	return status, nil
 }
 
+func isArgoCdEnabledForTenant(remoteCentral private.ManagedCentral) bool {
+	tenantResourceValues := remoteCentral.Spec.TenantResourcesValues
+	if tenantResourceValues == nil {
+		return false
+	}
+	argoCdIntf, ok := tenantResourceValues["argoCd"]
+	if !ok {
+		return false
+	}
+	argoCd, ok := argoCdIntf.(map[string]interface{})
+	if !ok {
+		return false
+	}
+	enabled, ok := argoCd["enabled"]
+	if !ok {
+		return false
+	}
+	enabledBool, ok := enabled.(bool)
+	if !ok {
+		return false
+	}
+	return enabledBool
+}
+
 func (r *CentralReconciler) getInstanceConfig(remoteCentral *private.ManagedCentral) (*v1alpha1.Central, error) {
 	var central = new(v1alpha1.Central)
 	if err := yaml2.Unmarshal([]byte(remoteCentral.Spec.CentralCRYAML), central); err != nil {
@@ -1106,6 +1160,12 @@ func (r *CentralReconciler) ensureCentralDeleted(ctx context.Context, remoteCent
 		globalDeleted = globalDeleted && secretDeleted
 	}
 
+	argoCdAppDeleted, err := r.ensureArgoCdApplicationDeleted(ctx, *remoteCentral)
+	if err != nil {
+		return false, err
+	}
+	globalDeleted = globalDeleted && argoCdAppDeleted
+
 	chartResourcesDeleted, err := r.ensureChartResourcesDeleted(ctx, remoteCentral)
 	if err != nil {
 		return false, err
@@ -1201,7 +1261,7 @@ func (r *CentralReconciler) createImagePullSecret(ctx context.Context, namespace
 }
 
 func (r *CentralReconciler) reconcileNamespace(ctx context.Context, c private.ManagedCentral) error {
-	desiredNamespace := getDesiredNamespace(c)
+	desiredNamespace := r.getDesiredNamespace(c)
 
 	existingNamespace, err := r.getNamespace(desiredNamespace.Name)
 	if err != nil {
@@ -1238,12 +1298,12 @@ func (r *CentralReconciler) reconcileNamespace(ctx context.Context, c private.Ma
 	return nil
 }
 
-func getDesiredNamespace(c private.ManagedCentral) *corev1.Namespace {
+func (r *CentralReconciler) getDesiredNamespace(c private.ManagedCentral) *corev1.Namespace {
 	return &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        c.Metadata.Namespace,
 			Annotations: getNamespaceAnnotations(c),
-			Labels:      getNamespaceLabels(c),
+			Labels:      r.getNamespaceLabels(c),
 		},
 	}
 }
@@ -1767,6 +1827,120 @@ func (r *CentralReconciler) isTenantResourcesChartObject(existingObject *unstruc
 		existingObject.GetNamespace() == remoteCentral.Metadata.Namespace
 }
 
+func (r *CentralReconciler) ensureArgoCdApplicationExists(ctx context.Context, remoteCentral private.ManagedCentral) error {
+	const lastAppliedHashLabel = "last-applied-hash"
+
+	want, err := r.makeDesiredArgoCDApplication(remoteCentral)
+	if err != nil {
+		return fmt.Errorf("getting ArgoCD application: %w", err)
+	}
+
+	hash, err := util.MD5SumFromJSONStruct(want)
+	if err != nil {
+		return fmt.Errorf("calculating MD5 from JSON: %w", err)
+	}
+	if want.Labels == nil {
+		want.Labels = map[string]string{}
+	}
+	want.Labels[lastAppliedHashLabel] = fmt.Sprintf("%x", hash)
+
+	var existing argocd.Application
+	err = r.client.Get(ctx, ctrlClient.ObjectKey{Namespace: want.Namespace, Name: want.Namespace}, &existing)
+	if err != nil {
+		if !apiErrors.IsNotFound(err) {
+			return fmt.Errorf("getting ArgoCD application: %w", err)
+		}
+		if err := r.client.Create(ctx, want); err != nil {
+			return fmt.Errorf("creating ArgoCD application: %w", err)
+		}
+		return nil
+	}
+
+	if existing.Labels == nil || existing.Labels[lastAppliedHashLabel] != want.Labels[lastAppliedHashLabel] {
+		want.ResourceVersion = existing.ResourceVersion
+		if err := r.client.Update(ctx, want); err != nil {
+			return fmt.Errorf("updating ArgoCD application: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (r *CentralReconciler) makeDesiredArgoCDApplication(remoteCentral private.ManagedCentral) (*argocd.Application, error) {
+
+	values := map[string]interface{}{
+		"tenant": map[string]interface{}{
+			"organizationId":   remoteCentral.Spec.Auth.OwnerOrgId,
+			"organizationName": remoteCentral.Spec.Auth.OwnerOrgName,
+			"id":               remoteCentral.Id,
+			"instanceType":     remoteCentral.Spec.InstanceType,
+		},
+		"centralRdsCidrBlock": "10.1.0.0/16",
+		"vpa": map[string]interface{}{
+			"central": map[string]interface{}{
+				"enabled": true,
+			},
+		},
+	}
+
+	valuesBytes, err := json.Marshal(values)
+	if err != nil {
+		return nil, fmt.Errorf("marshalling values: %w", err)
+	}
+
+	return &argocd.Application{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      remoteCentral.Metadata.Namespace,
+			Namespace: r.argoCdNamespace,
+		},
+		Spec: argocd.ApplicationSpec{
+			Project: "default",
+			Source: &argocd.ApplicationSource{
+				RepoURL: r.defaultTenantArgoCdAppSourceRepoURL,
+				Ref:     r.defaultTenantArgoCdAppSourceRef,
+				Path:    r.defaultTenantArgoCdAppSourcePath,
+				Helm: &argocd.ApplicationSourceHelm{
+					ValuesObject: &runtime.RawExtension{
+						Raw: valuesBytes,
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+func (r *CentralReconciler) ensureArgoCdApplicationDeleted(ctx context.Context, remoteCentral private.ManagedCentral) (bool, error) {
+	app := &argocd.Application{}
+	objectKey := ctrlClient.ObjectKey{Namespace: openshiftGitopsNamespace, Name: remoteCentral.Metadata.Namespace}
+
+	for i := 0; i < 5; i++ {
+
+		if i > 0 {
+			time.Sleep(time.Duration(i) * time.Millisecond * 200)
+		}
+
+		err := r.client.Get(ctx, objectKey, app)
+		if err != nil {
+			if apiErrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, fmt.Errorf("getting ArgoCD application: %w", err)
+		}
+
+		if app.DeletionTimestamp != nil {
+			continue
+		}
+
+		if err := r.client.Delete(ctx, app); err != nil {
+			return false, fmt.Errorf("deleting ArgoCD application: %w", err)
+		}
+
+	}
+
+	return false, nil
+
+}
+
 func (r *CentralReconciler) ensureRoutesExist(ctx context.Context, remoteCentral private.ManagedCentral) error {
 	err := r.ensureReencryptRouteExists(ctx, remoteCentral)
 	if err != nil {
@@ -1872,12 +2046,12 @@ func getTenantAnnotations(c private.ManagedCentral) map[string]string {
 	}
 }
 
-func getNamespaceLabels(c private.ManagedCentral) map[string]string {
+func (r *CentralReconciler) getNamespaceLabels(c private.ManagedCentral) map[string]string {
 	ret := map[string]string{}
 	for k, v := range getTenantLabels(c) {
 		ret[k] = v
 	}
-	ret[argoCdManagedBy] = openshiftGitopsNamespace
+	ret[argoCdManagedBy] = r.argoCdNamespace
 	return ret
 }
 
@@ -2106,6 +2280,14 @@ func NewCentralReconciler(k8sClient ctrlClient.Client, fleetmanagerClient *fleet
 
 		resourcesChart: resourcesChart,
 		clock:          realClock{},
+
+		// Todo: Allow overriding the tenant source path, repo URL, and ref
+		// on a per-tenant basis.
+
+		defaultTenantArgoCdAppSourcePath:    opts.DefaultTenantArgoCdAppSourcePath,
+		defaultTenantArgoCdAppSourceRepoURL: opts.DefaultTenantArgoCdAppSourceRepoURL,
+		defaultTenantArgoCdAppSourceRef:     opts.DefaultTenantArgoCdAppSourceRef,
+		argoCdNamespace:                     opts.ArgoCdNamespace,
 	}
 	r.needsReconcileFunc = r.needsReconcile