Skip to content

Commit

Permalink
Log partial K8s metrics API errors
Browse files Browse the repository at this point in the history
Partial errors are now logged to help with debugging.
Partial errors cause full error if requireKubernetesMetrics is true.
Partial errors do not stop autoscaling if requireKubernetesMetrics is false.
Update to k8shorizmetrics/v3.
  • Loading branch information
jthomperoo committed Mar 22, 2024
1 parent a262d98 commit c4fc6bc
Show file tree
Hide file tree
Showing 14 changed files with 356 additions and 94 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
### Changed
- Updated to `k8shorizmetrics/v3`.
- Partial K8s metrics server errors are logged to help with debugging.

## [v2.10.0] - 2023-12-24
### Added
Expand Down
48 changes: 24 additions & 24 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,52 +6,52 @@ require (
github.com/go-chi/chi v4.1.2+incompatible
github.com/golang/glog v1.2.0
github.com/google/go-cmp v0.6.0
github.com/jthomperoo/k8shorizmetrics/v2 v2.0.2
k8s.io/api v0.29.0
k8s.io/apimachinery v0.29.0
k8s.io/client-go v0.29.0
github.com/jthomperoo/k8shorizmetrics/v3 v3.0.0
k8s.io/api v0.29.3
k8s.io/apimachinery v0.29.3
k8s.io/client-go v0.29.3
)

require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v5.7.0+incompatible // indirect
github.com/emicklei/go-restful/v3 v3.12.0 // indirect
github.com/evanphx/json-patch v5.9.0+incompatible // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-openapi/jsonpointer v0.20.2 // indirect
github.com/go-openapi/jsonreference v0.20.4 // indirect
github.com/go-openapi/swag v0.22.6 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.5.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/oauth2 v0.15.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20240318143956-a85f2c67cd81 // indirect
golang.org/x/mod v0.16.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.16.1 // indirect
golang.org/x/tools v0.19.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.32.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
honnef.co/go/tools v0.4.6 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/kube-openapi v0.0.0-20231214164306-ab13479f8bf8 // indirect
k8s.io/metrics v0.29.0 // indirect
k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
honnef.co/go/tools v0.4.7 // indirect
k8s.io/klog/v2 v2.120.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/metrics v0.29.3 // indirect
k8s.io/utils v0.0.0-20240310230437-4693a0247e57 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
Expand Down
105 changes: 52 additions & 53 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion internal/fake/gather.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ limitations under the License.
package fake

import (
"github.com/jthomperoo/k8shorizmetrics/v2/metrics"
"github.com/jthomperoo/k8shorizmetrics/v3/metrics"
autoscalingv2 "k8s.io/api/autoscaling/v2"
"k8s.io/apimachinery/pkg/labels"
)
Expand Down
25 changes: 22 additions & 3 deletions internal/metricget/metricget.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,16 @@ package metricget
import (
"context"
"encoding/json"
"errors"
"fmt"

"github.com/golang/glog"
"github.com/jthomperoo/custom-pod-autoscaler/v2/config"
"github.com/jthomperoo/custom-pod-autoscaler/v2/internal/execute"
"github.com/jthomperoo/custom-pod-autoscaler/v2/k8smetric"
"github.com/jthomperoo/custom-pod-autoscaler/v2/metric"
"github.com/jthomperoo/k8shorizmetrics/v2/metrics"
"github.com/jthomperoo/k8shorizmetrics/v3"
"github.com/jthomperoo/k8shorizmetrics/v3/metrics"
autoscalingv2 "k8s.io/api/autoscaling/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -60,9 +62,25 @@ func (m *Gatherer) GetMetrics(info metric.Info, podSelector labels.Selector, cur
gatheredMetrics, err := m.K8sMetricGatherer.Gather(
convertCPAMetricSpecsToK8sMetricSpecs(m.Config.KubernetesMetricSpecs), m.Config.Namespace, podSelector)
if err != nil {
if m.Config.RequireKubernetesMetrics {
return nil, fmt.Errorf("failed to get required Kubernetes metrics: %w", err)
gatherErr := &k8shorizmetrics.GathererMultiMetricError{}
if errors.As(err, &gatherErr) {
for i, subErr := range gatherErr.Errors {
glog.Errorf("Error when retrieving K8s metrics (%d/%d): %+v", i+1, len(gatherErr.Errors), subErr)
}

if m.Config.RequireKubernetesMetrics {
if gatherErr.Partial {
return nil, fmt.Errorf("failed to get some (%d/%d) required Kubernetes metrics: %w", len(gatherErr.Errors), len(m.Config.KubernetesMetricSpecs), err)
}

return nil, fmt.Errorf("failed to get all required Kubernetes metrics: %w", err)
}
} else {
if m.Config.RequireKubernetesMetrics {
return nil, fmt.Errorf("failed to get required Kubernetes metrics: %w", err)
}
}

glog.Errorf("Failed to retrieve K8s metrics, not required so continuing: %+v", err)
} else {
glog.V(3).Infof("Successfully retrieved K8s metrics: %+v", gatheredMetrics)
Expand Down Expand Up @@ -221,5 +239,6 @@ func convertCPAMetricSpecsToK8sMetricSpecs(specs []config.K8sMetricSpec) []autos
for _, spec := range specs {
k8sSpecs = append(k8sSpecs, (autoscalingv2.MetricSpec)(spec))
}
glog.V(1).Infof("specs: %+v", k8sSpecs)
return k8sSpecs
}
Loading

0 comments on commit c4fc6bc

Please sign in to comment.