Skip to content

Commit

Permalink
ROX-18430: restore secrets from fleet-manager in fleetshard-sync (#1197)
Browse files Browse the repository at this point in the history 
* only report secrets if not stored yet

* remove resource version and owner reference on stored secrets

* add restore logic for secret data stored in fleet-manager

* added more unit test to restoreCentralSecrets

* add fleetmanager client and lazy loading of secrets to reconciler
  • Loading branch information
@johannes94
johannes94 authored Sep 5, 2023
1 parent 647e85a commit 2553649
Show file tree
Hide file tree
Showing 12 changed files with 521 additions and 29 deletions.
133 changes: 105 additions & 28 deletions fleetshard/pkg/central/reconciler/reconciler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
centralConstants "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/private"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/converters"
"github.com/stackrox/acs-fleet-manager/pkg/client/fleetmanager"
"github.com/stackrox/acs-fleet-manager/pkg/features"
"github.com/stackrox/rox/operator/apis/platform/v1alpha1"
"github.com/stackrox/rox/pkg/declarativeconfig"
Expand Down Expand Up @@ -102,20 +103,21 @@ type CentralReconcilerOptions struct {
// CentralReconciler is a reconciler tied to a one Central instance. It installs, updates and deletes Central instances
// in its Reconcile function.
type CentralReconciler struct {
client ctrlClient.Client
central private.ManagedCentral
status *int32
lastCentralHash [16]byte
useRoutes bool
Resources bool
routeService *k8s.RouteService
secretBackup *k8s.SecretBackup
secretCipher cipher.Cipher
egressProxyImage string
telemetry config.Telemetry
clusterName string
environment string
auditLogging config.AuditLogging
client ctrlClient.Client
fleetmanagerClient *fleetmanager.Client
central private.ManagedCentral
status *int32
lastCentralHash [16]byte
useRoutes bool
Resources bool
routeService *k8s.RouteService
secretBackup *k8s.SecretBackup
secretCipher cipher.Cipher
egressProxyImage string
telemetry config.Telemetry
clusterName string
environment string
auditLogging config.AuditLogging

managedDBEnabled bool
managedDBProvisioningClient cloudprovider.DBClient
Expand Down Expand Up @@ -173,6 +175,11 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
return nil, errors.Wrapf(err, "unable to ensure that namespace %s exists", remoteCentralNamespace)
}

err = r.restoreCentralSecrets(ctx, remoteCentral)
if err != nil {
return nil, err
}

if err := r.ensureChartResourcesExist(ctx, remoteCentral); err != nil {
return nil, errors.Wrapf(err, "unable to install chart resource for central %s/%s", central.GetNamespace(), central.GetName())
}
Expand Down Expand Up @@ -381,6 +388,50 @@ func (r *CentralReconciler) getInstanceConfig(remoteCentral *private.ManagedCent
return central, nil
}

func (r *CentralReconciler) restoreCentralSecrets(ctx context.Context, remoteCentral private.ManagedCentral) error {
restoreSecrets := []string{}
for _, secretName := range remoteCentral.Metadata.SecretsStored { // pragma: allowlist secret
exists, err := r.checkSecretExists(ctx, remoteCentral.Metadata.Namespace, secretName)
if err != nil {
return err
}

if !exists {
restoreSecrets = append(restoreSecrets, secretName)
}
}

if len(restoreSecrets) == 0 {
// nothing to restore
return nil
}

glog.Info(fmt.Sprintf("Restore secret for tenant: %s/%s", remoteCentral.Id, r.central.Metadata.Namespace), restoreSecrets)
central, _, err := r.fleetmanagerClient.PrivateAPI().GetCentral(ctx, remoteCentral.Id)
if err != nil {
return fmt.Errorf("loading secrets for central %s: %w", remoteCentral.Id, err)
}

decryptedSecrets, err := r.decryptSecrets(central.Metadata.Secrets)
if err != nil {
return fmt.Errorf("decrypting secrets for central %s: %w", central.Id, err)
}

for _, secretName := range restoreSecrets { // pragma: allowlist secret
secretToRestore, secretFound := decryptedSecrets[secretName]
if !secretFound {
return fmt.Errorf("finding secret %s in decrypted secret map", secretName)
}

if err := r.client.Create(ctx, secretToRestore); err != nil {
return fmt.Errorf("recreating secret %s for central %s: %w", secretName, central.Id, err)
}

}

return nil
}

func (r *CentralReconciler) reconcileAdminPasswordGeneration(central *v1alpha1.Central) error {
if !r.wantsAuthProvider {
central.Spec.Central.AdminPasswordGenerationDisabled = pointer.Bool(false)
Expand Down Expand Up @@ -814,6 +865,31 @@ func (r *CentralReconciler) collectSecretsEncrypted(ctx context.Context, remoteC
return encryptedSecrets, nil
}

func (r *CentralReconciler) decryptSecrets(secrets map[string]string) (map[string]*corev1.Secret, error) {
decryptedSecrets := map[string]*corev1.Secret{}

for secretName, ciphertext := range secrets {
decodedCipher, err := base64.StdEncoding.DecodeString(ciphertext)
if err != nil {
return nil, fmt.Errorf("decoding secret %s: %w", secretName, err)
}

plaintextSecret, err := r.secretCipher.Decrypt(decodedCipher)
if err != nil {
return nil, fmt.Errorf("decrypting secret %s: %w", secretName, err)
}

var secret corev1.Secret
if err := json.Unmarshal(plaintextSecret, &secret); err != nil {
return nil, fmt.Errorf("unmarshaling secret %s: %w", secretName, err)
}

decryptedSecrets[secretName] = &secret // pragma: allowlist secret
}

return decryptedSecrets, nil
}

func (r *CentralReconciler) encryptSecrets(secrets map[string]*corev1.Secret) (map[string]string, error) {
encryptedSecrets := map[string]string{}

Expand Down Expand Up @@ -1479,25 +1555,26 @@ func (r *CentralReconciler) ensureSecretExists(
}

// NewCentralReconciler ...
func NewCentralReconciler(k8sClient ctrlClient.Client, central private.ManagedCentral,
func NewCentralReconciler(k8sClient ctrlClient.Client, fleetmanagerClient *fleetmanager.Client, central private.ManagedCentral,
managedDBProvisioningClient cloudprovider.DBClient, managedDBInitFunc postgres.CentralDBInitFunc,
secretCipher cipher.Cipher,
opts CentralReconcilerOptions,
) *CentralReconciler {
return &CentralReconciler{
client: k8sClient,
central: central,
status: pointer.Int32(FreeStatus),
useRoutes: opts.UseRoutes,
wantsAuthProvider: opts.WantsAuthProvider,
routeService: k8s.NewRouteService(k8sClient),
secretBackup: k8s.NewSecretBackup(k8sClient),
secretCipher: secretCipher, // pragma: allowlist secret
egressProxyImage: opts.EgressProxyImage,
telemetry: opts.Telemetry,
clusterName: opts.ClusterName,
environment: opts.Environment,
auditLogging: opts.AuditLogging,
client: k8sClient,
fleetmanagerClient: fleetmanagerClient,
central: central,
status: pointer.Int32(FreeStatus),
useRoutes: opts.UseRoutes,
wantsAuthProvider: opts.WantsAuthProvider,
routeService: k8s.NewRouteService(k8sClient),
secretBackup: k8s.NewSecretBackup(k8sClient),
secretCipher: secretCipher, // pragma: allowlist secret
egressProxyImage: opts.EgressProxyImage,
telemetry: opts.Telemetry,
clusterName: opts.ClusterName,
environment: opts.Environment,
auditLogging: opts.AuditLogging,

managedDBEnabled: opts.ManagedDBEnabled,
managedDBProvisioningClient: managedDBProvisioningClient,
Expand Down
131 changes: 131 additions & 0 deletions fleetshard/pkg/central/reconciler/reconciler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"bytes"
"context"
"embed"
"encoding/base64"
"fmt"
"net/http"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -33,6 +35,7 @@ import (
"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/util"
centralConstants "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/private"
"github.com/stackrox/acs-fleet-manager/pkg/client/fleetmanager"
"github.com/stackrox/acs-fleet-manager/pkg/features"
"github.com/stackrox/rox/operator/apis/platform/v1alpha1"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -138,6 +141,7 @@ func getClientTrackerAndReconciler(
fakeClient, tracker := testutils.NewFakeClientWithTracker(t, k8sObjects...)
reconciler := NewCentralReconciler(
fakeClient,
fleetmanager.NewClientMock().Client(),
centralConfig,
managedDBClient,
centralDBInitFunc,
Expand Down Expand Up @@ -1698,3 +1702,130 @@ func TestReconcileDeclarativeConfigurationData(t *testing.T) {
})
}
}

func TestRestoreCentralSecrets(t *testing.T) {
testCases := []struct {
name string
buildCentral func() private.ManagedCentral
mockObjects []client.Object
buildFMClient func() *fleetmanager.Client
expectedErrorMsgContains string
expectedObjects []client.Object
}{
{
name: "no error for SecretsStored not set",
buildCentral: func() private.ManagedCentral {
return simpleManagedCentral
},
},
{
name: "no error for existing secrets in SecretsStored",
buildCentral: func() private.ManagedCentral {
newCentral := simpleManagedCentral
newCentral.Metadata.SecretsStored = []string{"central-tls", "central-db-password"}
return newCentral
},
mockObjects: []client.Object{
centralTLSSecretObject(),
centralDBPasswordSecretObject(),
},
},
{
name: "return errors from fleetmanager",
buildCentral: func() private.ManagedCentral {
newCentral := simpleManagedCentral
newCentral.Metadata.SecretsStored = []string{"central-tls", "central-db-password"}
return newCentral
},
mockObjects: []client.Object{
centralTLSSecretObject(),
},
buildFMClient: func() *fleetmanager.Client {
mockClient := fleetmanager.NewClientMock()
mockClient.PrivateAPIMock.GetCentralFunc = func(ctx context.Context, centralID string) (private.ManagedCentral, *http.Response, error) {
return private.ManagedCentral{}, nil, errors.New("test error")
}
return mockClient.Client()
},
expectedErrorMsgContains: "loading secrets for central cb45idheg5ip6dq1jo4g: test error",
},
{
// force encrypt error by using non base64 value for central-db-password
name: "return errors from decryptSecrets",
buildCentral: func() private.ManagedCentral {
newCentral := simpleManagedCentral
newCentral.Metadata.SecretsStored = []string{"central-tls", "central-db-password"}
return newCentral
},
mockObjects: []client.Object{
centralTLSSecretObject(),
},
buildFMClient: func() *fleetmanager.Client {
mockClient := fleetmanager.NewClientMock()
mockClient.PrivateAPIMock.GetCentralFunc = func(ctx context.Context, centralID string) (private.ManagedCentral, *http.Response, error) {
returnCentral := simpleManagedCentral
returnCentral.Metadata.Secrets = map[string]string{"central-db-password": "testpw"}
return returnCentral, nil, nil
}
return mockClient.Client()
},
expectedErrorMsgContains: "decrypting secrets for central",
},
{
name: "expect secrets to exist after secret restore",
buildCentral: func() private.ManagedCentral {
newCentral := simpleManagedCentral
newCentral.Metadata.SecretsStored = []string{"central-tls", "central-db-password"}
return newCentral
},
buildFMClient: func() *fleetmanager.Client {
mockClient := fleetmanager.NewClientMock()
mockClient.PrivateAPIMock.GetCentralFunc = func(ctx context.Context, centralID string) (private.ManagedCentral, *http.Response, error) {
returnCentral := simpleManagedCentral
centralTLS := `{"metadata":{"name":"central-tls","namespace":"rhacs-cb45idheg5ip6dq1jo4g","creationTimestamp":null}}`
centralDBPW := `{"metadata":{"name":"central-db-password","namespace":"rhacs-cb45idheg5ip6dq1jo4g","creationTimestamp":null}}`

encode := base64.StdEncoding.EncodeToString
// we need to encode twice, once for b64 test cipher used
// once for the b64 encoding done to transfer secret data via API
returnCentral.Metadata.Secrets = map[string]string{
"central-tls": encode([]byte(encode([]byte(centralTLS)))),
"central-db-password": encode([]byte(encode([]byte(centralDBPW)))),
}
return returnCentral, nil, nil
}
return mockClient.Client()
},
expectedObjects: []client.Object{
centralTLSSecretObject(),
centralDBPasswordSecretObject(),
},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fakeClient, _, r := getClientTrackerAndReconciler(t, simpleManagedCentral, nil, defaultReconcilerOptions, tc.mockObjects...)
managedCentral := tc.buildCentral()

if tc.buildFMClient != nil {
r.fleetmanagerClient = tc.buildFMClient()
}

err := r.restoreCentralSecrets(context.Background(), managedCentral)

if err != nil && tc.expectedErrorMsgContains != "" {
require.Contains(t, err.Error(), tc.expectedErrorMsgContains)
} else {
require.NoError(t, err)
}

for _, obj := range tc.expectedObjects {
s := v1.Secret{}
err := fakeClient.Get(context.Background(), client.ObjectKey{Namespace: obj.GetNamespace(), Name: obj.GetName()}, &s)
require.NoErrorf(t, err, "finding expected object %s/%s", obj.GetNamespace(), obj.GetName())
}

})
}
}
2 changes: 1 addition & 1 deletion fleetshard/pkg/runtime/runtime.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func (r *Runtime) Start() error {
logger.InfoChangedInt32(&reconciledCentralCountCache, "Received central count changed: received %d centrals", reconciledCentralCountCache)
for _, central := range list.Items {
if _, ok := r.reconcilers[central.Id]; !ok {
r.reconcilers[central.Id] = centralReconciler.NewCentralReconciler(r.k8sClient, central,
r.reconcilers[central.Id] = centralReconciler.NewCentralReconciler(r.k8sClient, r.client, central,
r.dbProvisionClient, postgres.InitializeDatabase, r.secretCipher, reconcilerOpts)
}

Expand Down
Loading

0 comments on commit 2553649

Please sign in to comment.