diff --git a/cmd/argocd-application-controller/commands/argocd_application_controller.go b/cmd/argocd-application-controller/commands/argocd_application_controller.go index 5d7fd803e7aca..81b7f2481189a 100644 --- a/cmd/argocd-application-controller/commands/argocd_application_controller.go +++ b/cmd/argocd-application-controller/commands/argocd_application_controller.go @@ -31,6 +31,7 @@ import ( "github.com/argoproj/argo-cd/v2/util/settings" "github.com/argoproj/argo-cd/v2/util/tls" "github.com/argoproj/argo-cd/v2/util/trace" + argoio "github.com/argoproj/gitops-engine/pkg/utils/io" ) const ( @@ -74,6 +75,7 @@ func NewCommand() *cobra.Command { enableDynamicClusterDistribution bool serverSideDiff bool ignoreNormalizerOpts normalizers.IgnoreNormalizerOpts + manifestGenerationTmpPath string ) command := cobra.Command{ Use: cliName, @@ -172,6 +174,7 @@ func NewCommand() *cobra.Command { serverSideDiff, enableDynamicClusterDistribution, ignoreNormalizerOpts, + manifestGenerationTmpPath, ) errors.CheckError(err) cacheutil.CollectMetrics(redisClient, appController.GetMetricsServer()) @@ -233,6 +236,9 @@ func NewCommand() *cobra.Command { command.Flags().BoolVar(&enableDynamicClusterDistribution, "dynamic-cluster-distribution-enabled", env.ParseBoolFromEnv(common.EnvEnableDynamicClusterDistribution, false), "Enables dynamic cluster distribution.") command.Flags().BoolVar(&serverSideDiff, "server-side-diff-enabled", env.ParseBoolFromEnv(common.EnvServerSideDiff, false), "Feature flag to enable ServerSide diff. Default (\"false\")") command.Flags().DurationVar(&ignoreNormalizerOpts.JQExecutionTimeout, "ignore-normalizer-jq-execution-timeout-seconds", env.ParseDurationFromEnv("ARGOCD_IGNORE_NORMALIZER_JQ_TIMEOUT", 0*time.Second, 0, math.MaxInt64), "Set ignore normalizer JQ execution timeout") + + command.Flags().StringVar(&manifestGenerationTmpPath, "manifestGenerationTmpPath", env.StringFromEnv("ARGOCD_APPLICATION_CONTROLLER_MANIFEST_GENERATION_PATH", argoio.TempPathUseDevShmIfAvailable()), "Set the path within application-controller container where Kubernetes manifests will be generated before being passed to kubectl code, default value is \"/dev/shm\"") + cacheSource = appstatecache.AddCacheFlagsToCmd(&command, cacheutil.Options{ OnClientCreated: func(client *redis.Client) { redisClient = client diff --git a/cmd/argocd/commands/admin/app.go b/cmd/argocd/commands/admin/app.go index 5c3bd761fe00c..62740e46d64ce 100644 --- a/cmd/argocd/commands/admin/app.go +++ b/cmd/argocd/commands/admin/app.go @@ -41,6 +41,7 @@ import ( "github.com/argoproj/argo-cd/v2/util/io" kubeutil "github.com/argoproj/argo-cd/v2/util/kube" "github.com/argoproj/argo-cd/v2/util/settings" + argoio "github.com/argoproj/gitops-engine/pkg/utils/io" ) func NewAppCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command { @@ -452,5 +453,5 @@ func reconcileApplications( } func newLiveStateCache(argoDB db.ArgoDB, appInformer kubecache.SharedIndexInformer, settingsMgr *settings.SettingsManager, server *metrics.MetricsServer) cache.LiveStateCache { - return cache.NewLiveStateCache(argoDB, appInformer, settingsMgr, kubeutil.NewKubectl(), server, func(managedByApp map[string]bool, ref apiv1.ObjectReference) {}, &sharding.ClusterSharding{}, argo.NewResourceTracking()) + return cache.NewLiveStateCache(argoDB, appInformer, settingsMgr, kubeutil.NewKubectl(), argoio.TempPathUseDevShmIfAvailable(), server, func(managedByApp map[string]bool, ref apiv1.ObjectReference) {}, &sharding.ClusterSharding{}, argo.NewResourceTracking()) } diff --git a/controller/appcontroller.go b/controller/appcontroller.go index f608e53d5aac6..5b740ed88f137 100644 --- a/controller/appcontroller.go +++ b/controller/appcontroller.go @@ -172,6 +172,7 @@ func NewApplicationController( serverSideDiff bool, dynamicClusterDistributionEnabled bool, ignoreNormalizerOpts normalizers.IgnoreNormalizerOpts, + manifestGenerationTmpPath string, ) (*ApplicationController, error) { log.Infof("appResyncPeriod=%v, appHardResyncPeriod=%v, appResyncJitter=%v", appResyncPeriod, appHardResyncPeriod, appResyncJitter) db := db.NewDB(namespace, settingsMgr, kubeClientset) @@ -289,7 +290,8 @@ func NewApplicationController( return nil, err } } - stateCache := statecache.NewLiveStateCache(db, appInformer, ctrl.settingsMgr, kubectl, ctrl.metricsServer, ctrl.handleObjectUpdated, clusterSharding, argo.NewResourceTracking()) + + stateCache := statecache.NewLiveStateCache(db, appInformer, ctrl.settingsMgr, kubectl, manifestGenerationTmpPath, ctrl.metricsServer, ctrl.handleObjectUpdated, clusterSharding, argo.NewResourceTracking()) appStateManager := NewAppStateManager(db, applicationClientset, repoClientset, namespace, kubectl, ctrl.settingsMgr, stateCache, projInformer, ctrl.metricsServer, argoCache, ctrl.statusRefreshTimeout, argo.NewResourceTracking(), persistResourceHealth, repoErrorGracePeriod, serverSideDiff, ignoreNormalizerOpts) ctrl.appInformer = appInformer ctrl.appLister = appLister diff --git a/controller/appcontroller_test.go b/controller/appcontroller_test.go index e9fa91f61f301..32cb4ad9236c2 100644 --- a/controller/appcontroller_test.go +++ b/controller/appcontroller_test.go @@ -47,6 +47,7 @@ import ( cacheutil "github.com/argoproj/argo-cd/v2/util/cache" appstatecache "github.com/argoproj/argo-cd/v2/util/cache/appstate" "github.com/argoproj/argo-cd/v2/util/settings" + argoio "github.com/argoproj/gitops-engine/pkg/utils/io" ) type namespacedResource struct { @@ -163,6 +164,7 @@ func newFakeController(data *fakeData, repoErr error) *ApplicationController { false, false, normalizers.IgnoreNormalizerOpts{}, + argoio.TempPathUseDevShmIfAvailable(), ) db := &dbmocks.ArgoDB{} db.On("GetApplicationControllerReplicas").Return(1) diff --git a/controller/cache/cache.go b/controller/cache/cache.go index 5b163ad54d111..f4dca548aae65 100644 --- a/controller/cache/cache.go +++ b/controller/cache/cache.go @@ -170,21 +170,23 @@ func NewLiveStateCache( appInformer cache.SharedIndexInformer, settingsMgr *settings.SettingsManager, kubectl kube.Kubectl, + manifestGenTmpPath string, metricsServer *metrics.MetricsServer, onObjectUpdated ObjectUpdatedHandler, clusterSharding sharding.ClusterShardingCache, resourceTracking argo.ResourceTracking, ) LiveStateCache { return &liveStateCache{ - appInformer: appInformer, - db: db, - clusters: make(map[string]clustercache.ClusterCache), - onObjectUpdated: onObjectUpdated, - kubectl: kubectl, - settingsMgr: settingsMgr, - metricsServer: metricsServer, - clusterSharding: clusterSharding, - resourceTracking: resourceTracking, + appInformer: appInformer, + db: db, + clusters: make(map[string]clustercache.ClusterCache), + onObjectUpdated: onObjectUpdated, + kubectl: kubectl, + settingsMgr: settingsMgr, + metricsServer: metricsServer, + clusterSharding: clusterSharding, + resourceTracking: resourceTracking, + manifestGenTmpPath: manifestGenTmpPath, } } @@ -213,6 +215,9 @@ type liveStateCache struct { clusters map[string]clustercache.ClusterCache cacheSettings cacheSettings lock sync.RWMutex + + // manifestGenTmpPath is the temporary path value passed to gitops-engine, which gitops-engine uses when passing kubernetes manifests/cluster credentials to kubectl code. See 'github.com/argoproj/gitops-engine/pkg/utils/io' for details. + manifestGenTmpPath string } func (c *liveStateCache) loadCacheSettings() (*cacheSettings, error) { @@ -524,6 +529,7 @@ func (c *liveStateCache) getCluster(server string) (clustercache.ClusterCache, e clustercache.SetLogr(logutils.NewLogrusLogger(log.WithField("server", cluster.Server))), clustercache.SetRetryOptions(clusterCacheAttemptLimit, clusterCacheRetryUseBackoff, isRetryableError), clustercache.SetRespectRBAC(respectRBAC), + clustercache.SetTmpPath(c.manifestGenTmpPath), } clusterCache = clustercache.NewClusterCache(clusterCacheConfig, clusterCacheOpts...) diff --git a/docs/operator-manual/argocd-cmd-params-cm.yaml b/docs/operator-manual/argocd-cmd-params-cm.yaml index f5d13b5522f65..9cbbf5b4ae136 100644 --- a/docs/operator-manual/argocd-cmd-params-cm.yaml +++ b/docs/operator-manual/argocd-cmd-params-cm.yaml @@ -73,6 +73,10 @@ data: # diff cache is unavailable). controller.diff.server.side: "false" + # controller.manifest.generation.path can be used to overwrite the default path (within the application-controller container) used to store the temporary files which are passed to kubectl code via gitops-engine. These temporary files are security sensitive resources, such as generated kubernetes manifests, or cluster credentials. + # Due to security sensitivity, by default '/dev/shm' is used. However, 'controller.manifest.generation.path' allows this value to be customized, for example, using '/tmp' to overcome the limited disk space of '/dev/shm'. + controller.manifest.generation.path: "/dev/shm" + ## Server properties # Listen on given address for incoming connections (default "0.0.0.0") server.listen.address: "0.0.0.0" diff --git a/go.mod b/go.mod index b39aff4f2539c..9a980bf06152a 100644 --- a/go.mod +++ b/go.mod @@ -291,6 +291,7 @@ require ( ) replace ( + github.com/argoproj/gitops-engine => github.com/jgwest/gitops-engine v0.1.3-0.20240726204147-73c025b8bd5a // https://github.com/golang/go/issues/33546#issuecomment-519656923 github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 diff --git a/go.sum b/go.sum index 72c2b02ff50d6..ae21f5aa02eb6 100644 --- a/go.sum +++ b/go.sum @@ -692,8 +692,6 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/appscode/go v0.0.0-20191119085241-0887d8ec2ecc/go.mod h1:OawnOmAL4ZX3YaPdN+8HTNwBveT1jMsqP74moa9XUbE= -github.com/argoproj/gitops-engine v0.7.1-0.20240718175351-6b2984ebc470 h1:RUo6je4n+FgNEkGsONhwxUtT67YqyEtrvMNd+t8pKSo= -github.com/argoproj/gitops-engine v0.7.1-0.20240718175351-6b2984ebc470/go.mod h1:xMIbuLg9Qj2e0egTy+8NcukbhRaVmWwK9vm3aAQZoi4= github.com/argoproj/notifications-engine v0.4.1-0.20240606074338-0802cd427621 h1:Yg1nt+D2uDK1SL2jSlfukA4yc7db184TTN7iWy3voRE= github.com/argoproj/notifications-engine v0.4.1-0.20240606074338-0802cd427621/go.mod h1:N0A4sEws2soZjEpY4hgZpQS8mRIEw6otzwfkgc3g9uQ= github.com/argoproj/pkg v0.13.7-0.20230626144333-d56162821bd1 h1:qsHwwOJ21K2Ao0xPju1sNuqphyMnMYkyB3ZLoLtxWpo= @@ -1294,6 +1292,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jeremywohl/flatten v1.0.1 h1:LrsxmB3hfwJuE+ptGOijix1PIfOoKLJ3Uee/mzbgtrs= github.com/jeremywohl/flatten v1.0.1/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= +github.com/jgwest/gitops-engine v0.1.3-0.20240726204147-73c025b8bd5a h1:By0LRk/OotWPmZ6CtXgo6tAXudWQlJeVmakT3vQTMGI= +github.com/jgwest/gitops-engine v0.1.3-0.20240726204147-73c025b8bd5a/go.mod h1:xMIbuLg9Qj2e0egTy+8NcukbhRaVmWwK9vm3aAQZoi4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= diff --git a/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml b/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml index 815e4123d05e3..995980402d6c4 100644 --- a/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml +++ b/manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml @@ -199,6 +199,12 @@ spec: name: argocd-cmd-params-cm key: controller.diff.server.side optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_MANIFEST_GENERATION_PATH + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: controller.manifest.generation.path + optional: true image: quay.io/argoproj/argocd:latest imagePullPolicy: Always name: argocd-application-controller diff --git a/manifests/base/application-controller/argocd-application-controller-statefulset.yaml b/manifests/base/application-controller/argocd-application-controller-statefulset.yaml index 2219f5f9b4731..64c07c221bf93 100644 --- a/manifests/base/application-controller/argocd-application-controller-statefulset.yaml +++ b/manifests/base/application-controller/argocd-application-controller-statefulset.yaml @@ -208,6 +208,12 @@ spec: name: argocd-cmd-params-cm key: controller.ignore.normalizer.jq.timeout optional: true + - name: ARGOCD_APPLICATION_CONTROLLER_MANIFEST_GENERATION_PATH + valueFrom: + configMapKeyRef: + name: argocd-cmd-params-cm + key: controller.manifest.generation.path + optional: true image: quay.io/argoproj/argocd:latest imagePullPolicy: Always name: argocd-application-controller diff --git a/util/git/creds.go b/util/git/creds.go index 4341f40ce83d3..0e2b11a75f39b 100644 --- a/util/git/creds.go +++ b/util/git/creds.go @@ -172,10 +172,10 @@ func (c HTTPSCreds) Environ() (io.Closer, []string, error) { // We need to actually create two temp files, one for storing cert data and // another for storing the key. If we fail to create second fail, the first // must be removed. - certFile, err := os.CreateTemp(argoio.TempDir, "") + certFile, err := os.CreateTemp(argoio.TempPathUseDevShmIfAvailable(), "") if err == nil { defer certFile.Close() - keyFile, err = os.CreateTemp(argoio.TempDir, "") + keyFile, err = os.CreateTemp(argoio.TempPathUseDevShmIfAvailable(), "") if err != nil { removeErr := os.Remove(certFile.Name()) if removeErr != nil { @@ -270,7 +270,7 @@ func (f authFilePaths) Close() error { func (c SSHCreds) Environ() (io.Closer, []string, error) { // use the SHM temp dir from util, more secure - file, err := os.CreateTemp(argoio.TempDir, "") + file, err := os.CreateTemp(argoio.TempPathUseDevShmIfAvailable(), "") if err != nil { return nil, nil, err } @@ -371,10 +371,10 @@ func (g GitHubAppCreds) Environ() (io.Closer, []string, error) { // We need to actually create two temp files, one for storing cert data and // another for storing the key. If we fail to create second fail, the first // must be removed. - certFile, err := os.CreateTemp(argoio.TempDir, "") + certFile, err := os.CreateTemp(argoio.TempPathUseDevShmIfAvailable(), "") if err == nil { defer certFile.Close() - keyFile, err = os.CreateTemp(argoio.TempDir, "") + keyFile, err = os.CreateTemp(argoio.TempPathUseDevShmIfAvailable(), "") if err != nil { removeErr := os.Remove(certFile.Name()) if removeErr != nil { diff --git a/util/git/creds_test.go b/util/git/creds_test.go index c4b0b29fd1e5a..a3bb2fe309aaa 100644 --- a/util/git/creds_test.go +++ b/util/git/creds_test.go @@ -307,9 +307,9 @@ func Test_SSHCreds_Environ_WithProxyUserNamePassword(t *testing.T) { func Test_SSHCreds_Environ_TempFileCleanupOnInvalidProxyURL(t *testing.T) { // Previously, if the proxy URL was invalid, a temporary file would be left in /dev/shm. This ensures the file is cleaned up in this case. - // countDev returns the number of files in /dev/shm (argoio.TempDir) + // countDev returns the number of files in /dev/shm (argoio.TempPathUseDevShmIfAvailable()) countFilesInDevShm := func() int { - entries, err := os.ReadDir(argoio.TempDir) + entries, err := os.ReadDir(argoio.TempPathUseDevShmIfAvailable()) require.NoError(t, err) return len(entries)