diff --git a/pkg/cache/cluster.go b/pkg/cache/cluster.go index 52be5b7da..22392d623 100644 --- a/pkg/cache/cluster.go +++ b/pkg/cache/cluster.go @@ -89,6 +89,8 @@ type ClusterInfo struct { SyncError error // APIResources holds list of API resources supported by the cluster APIResources []kube.APIResourceInfo + // ConnectionStatus indicates the status of the connection with the cluster. + ConnectionStatus ConnectionStatus } // OnEventHandler is a function that handles Kubernetes event @@ -170,6 +172,7 @@ func NewClusterCache(config *rest.Config, opts ...UpdateSettingsFunc) *clusterCa listRetryLimit: 1, listRetryUseBackoff: false, listRetryFunc: ListRetryFuncNever, + connectionStatus: ConnectionStatusUnknown, } for i := range opts { opts[i](cache) @@ -177,9 +180,21 @@ func NewClusterCache(config *rest.Config, opts ...UpdateSettingsFunc) *clusterCa return cache } +// ConnectionStatus indicates the status of the connection with the cluster. +type ConnectionStatus string + +const ( + ConnectionStatusSuccessful ConnectionStatus = "Successful" + ConnectionStatusFailed ConnectionStatus = "Failed" + ConnectionStatusUnknown ConnectionStatus = "Unknown" +) + type clusterCache struct { syncStatus clusterCacheSync + // connectionStatus indicates the status of the connection with the cluster. + connectionStatus ConnectionStatus + apisMeta map[schema.GroupKind]*apiMeta serverVersion string apiResources []kube.APIResourceInfo @@ -615,6 +630,25 @@ func (c *clusterCache) watchEvents(ctx context.Context, api kube.APIResourceInfo if errors.IsNotFound(err) { c.stopWatching(api.GroupKind, ns) } + var connectionUpdated bool + if err != nil { + if c.connectionStatus != ConnectionStatusFailed { + c.log.Info("unable to access cluster", "cluster", c.config.Host, "reason", err.Error()) + c.connectionStatus = ConnectionStatusFailed + connectionUpdated = true + } + } else if c.connectionStatus != ConnectionStatusSuccessful { + c.connectionStatus = ConnectionStatusSuccessful + connectionUpdated = true + } + + if connectionUpdated { + c.Invalidate() + if err := c.EnsureSynced(); err != nil { + return nil, err + } + } + return res, err }, }) @@ -808,8 +842,14 @@ func (c *clusterCache) sync() error { version, err := c.kubectl.GetServerVersion(config) if err != nil { + if c.connectionStatus != ConnectionStatusFailed { + c.log.Info("unable to access cluster", "cluster", c.config.Host, "reason", err.Error()) + c.connectionStatus = ConnectionStatusFailed + } return err } + + c.connectionStatus = ConnectionStatusSuccessful c.serverVersion = version apiResources, err := c.kubectl.GetAPIResources(config, false, NewNoopSettings()) if err != nil { @@ -1184,6 +1224,7 @@ func (c *clusterCache) GetClusterInfo() ClusterInfo { LastCacheSyncTime: c.syncStatus.syncTime, SyncError: c.syncStatus.syncError, APIResources: c.apiResources, + ConnectionStatus: c.connectionStatus, } } diff --git a/pkg/cache/cluster_test.go b/pkg/cache/cluster_test.go index 68221ab89..a0f464e7e 100644 --- a/pkg/cache/cluster_test.go +++ b/pkg/cache/cluster_test.go @@ -3,12 +3,13 @@ package cache import ( "context" "fmt" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "sort" "strings" "testing" "time" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" @@ -148,6 +149,7 @@ func TestEnsureSynced(t *testing.T) { } cluster := newCluster(t, obj1, obj2) + assert.Equal(t, cluster.connectionStatus, ConnectionStatusUnknown) err := cluster.EnsureSynced() require.NoError(t, err) @@ -160,6 +162,7 @@ func TestEnsureSynced(t *testing.T) { names = append(names, k.Name) } assert.ElementsMatch(t, []string{"helm-guestbook1", "helm-guestbook2"}, names) + assert.Equal(t, cluster.connectionStatus, ConnectionStatusSuccessful) } func TestStatefulSetOwnershipInferred(t *testing.T) { @@ -492,23 +495,23 @@ metadata: func TestGetManagedLiveObjsFailedConversion(t *testing.T) { cronTabGroup := "stable.example.com" - testCases := []struct{ - name string - localConvertFails bool + testCases := []struct { + name string + localConvertFails bool expectConvertToVersionCalled bool - expectGetResourceCalled bool + expectGetResourceCalled bool }{ { - name: "local convert fails, so GetResource is called", - localConvertFails: true, + name: "local convert fails, so GetResource is called", + localConvertFails: true, expectConvertToVersionCalled: true, - expectGetResourceCalled: true, + expectGetResourceCalled: true, }, { - name: "local convert succeeds, so GetResource is not called", - localConvertFails: false, + name: "local convert succeeds, so GetResource is not called", + localConvertFails: false, expectConvertToVersionCalled: true, - expectGetResourceCalled: false, + expectGetResourceCalled: false, }, } @@ -557,7 +560,6 @@ metadata: return testCronTab(), nil }) - managedObjs, err := cluster.GetManagedLiveObjs([]*unstructured.Unstructured{targetDeploy}, func(r *Resource) bool { return true }) @@ -716,9 +718,10 @@ func TestGetClusterInfo(t *testing.T) { cluster.serverVersion = "v1.16" info := cluster.GetClusterInfo() assert.Equal(t, ClusterInfo{ - Server: cluster.config.Host, - APIResources: cluster.apiResources, - K8SVersion: cluster.serverVersion, + Server: cluster.config.Host, + APIResources: cluster.apiResources, + K8SVersion: cluster.serverVersion, + ConnectionStatus: ConnectionStatusUnknown, }, info) } @@ -816,25 +819,25 @@ func testPod() *corev1.Pod { func testCRD() *apiextensions.CustomResourceDefinition { return &apiextensions.CustomResourceDefinition{ - TypeMeta: metav1.TypeMeta{ + TypeMeta: metav1.TypeMeta{ APIVersion: "apiextensions.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: "crontabs.stable.example.com", }, - Spec: apiextensions.CustomResourceDefinitionSpec{ + Spec: apiextensions.CustomResourceDefinitionSpec{ Group: "stable.example.com", Versions: []apiextensions.CustomResourceDefinitionVersion{ { - Name: "v1", - Served: true, + Name: "v1", + Served: true, Storage: true, Schema: &apiextensions.CustomResourceValidation{ OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ Type: "object", Properties: map[string]apiextensions.JSONSchemaProps{ "cronSpec": {Type: "string"}, - "image": {Type: "string"}, + "image": {Type: "string"}, "replicas": {Type: "integer"}, }, }, @@ -855,14 +858,14 @@ func testCRD() *apiextensions.CustomResourceDefinition { func testCronTab() *unstructured.Unstructured { return &unstructured.Unstructured{Object: map[string]interface{}{ "apiVersion": "stable.example.com/v1", - "kind": "CronTab", + "kind": "CronTab", "metadata": map[string]interface{}{ - "name": "test-crontab", + "name": "test-crontab", "namespace": "default", }, "spec": map[string]interface{}{ "cronSpec": "* * * * */5", - "image": "my-awesome-cron-image", + "image": "my-awesome-cron-image", }, }} }