From 22c3a7b06494d9d37088bea331d59ec309d515e3 Mon Sep 17 00:00:00 2001 From: ilyam8 Date: Wed, 3 Jan 2024 23:18:57 +0200 Subject: [PATCH] httpcheck: add header_match --- modules/httpcheck/charts.go | 1 + modules/httpcheck/collect.go | 28 +++++ modules/httpcheck/httpcheck.go | 30 +++-- modules/httpcheck/httpcheck_test.go | 175 ++++++++++++++++++++++++++++ modules/httpcheck/init.go | 43 +++++++ modules/httpcheck/metadata.yaml | 47 +++++++- modules/httpcheck/metrics.go | 1 + 7 files changed, 317 insertions(+), 8 deletions(-) diff --git a/modules/httpcheck/charts.go b/modules/httpcheck/charts.go index 81a081d778..c0ae78c22e 100644 --- a/modules/httpcheck/charts.go +++ b/modules/httpcheck/charts.go @@ -58,6 +58,7 @@ var responseStatusChart = module.Chart{ {ID: "redirect"}, {ID: "bad_content"}, {ID: "bad_status"}, + {ID: "bad_header"}, }, } diff --git a/modules/httpcheck/collect.go b/modules/httpcheck/collect.go index 1ac5fc02c9..7d1ba674de 100644 --- a/modules/httpcheck/collect.go +++ b/modules/httpcheck/collect.go @@ -102,9 +102,37 @@ func (hc *HTTPCheck) collectOKResponse(mx *metrics, resp *http.Response) { return } + if ok := hc.checkHeader(resp); !ok { + mx.Status.BadHeader = true + return + } + mx.Status.Success = true } +func (hc *HTTPCheck) checkHeader(resp *http.Response) bool { + for _, m := range hc.headerMatch { + value := resp.Header.Get(m.key) + + var ok bool + switch { + case value == "": + ok = m.exclude + case m.valMatcher == nil: + ok = !m.exclude + default: + ok = m.valMatcher.MatchString(value) + } + + if !ok { + hc.Debugf("headers match: bad header: exlude '%v' key '%s' value '%s'", m.exclude, m.key, value) + return false + } + } + + return true +} + func decodeReqError(err error) reqErrCode { if err == nil { panic("nil error") diff --git a/modules/httpcheck/httpcheck.go b/modules/httpcheck/httpcheck.go index 3cfaf80cf5..abcb5cf7bd 100644 --- a/modules/httpcheck/httpcheck.go +++ b/modules/httpcheck/httpcheck.go @@ -40,13 +40,21 @@ func New() *HTTPCheck { } } -type Config struct { - web.HTTP `yaml:",inline"` - UpdateEvery int `yaml:"update_every"` - AcceptedStatuses []int `yaml:"status_accepted"` - ResponseMatch string `yaml:"response_match"` - CookieFile string `yaml:"cookie_file"` -} +type ( + Config struct { + web.HTTP `yaml:",inline"` + UpdateEvery int `yaml:"update_every"` + AcceptedStatuses []int `yaml:"status_accepted"` + ResponseMatch string `yaml:"response_match"` + CookieFile string `yaml:"cookie_file"` + HeaderMatch []HeaderMatchConfig `yaml:"header_match"` + } + HeaderMatchConfig struct { + Exclude bool `yaml:"exclude"` + Key string `yaml:"key"` + Value string `yaml:"value"` + } +) type HTTPCheck struct { module.Base @@ -58,6 +66,7 @@ type HTTPCheck struct { acceptedStatuses map[int]bool reResponse *regexp.Regexp + headerMatch []headersMatch cookieFileModTime time.Time @@ -86,6 +95,13 @@ func (hc *HTTPCheck) Init() bool { } hc.reResponse = re + hm, err := hc.initHeadersMatch() + if err != nil { + hc.Errorf("init headers match: %v", err) + return false + } + hc.headerMatch = hm + for _, v := range hc.AcceptedStatuses { hc.acceptedStatuses[v] = true } diff --git a/modules/httpcheck/httpcheck_test.go b/modules/httpcheck/httpcheck_test.go index 45dc7d60b7..c6a8edcf54 100644 --- a/modules/httpcheck/httpcheck_test.go +++ b/modules/httpcheck/httpcheck_test.go @@ -144,12 +144,14 @@ func TestHTTPCheck_Check(t *testing.T) { func TestHTTPCheck_Collect(t *testing.T) { tests := map[string]struct { prepare func() (httpCheck *HTTPCheck, cleanup func()) + update func(check *HTTPCheck) wantMetrics map[string]int64 }{ "success case": { prepare: prepareSuccessCase, wantMetrics: map[string]int64{ "bad_content": 0, + "bad_headers": 0, "bad_status": 0, "in_state": 2, "length": 5, @@ -164,6 +166,7 @@ func TestHTTPCheck_Collect(t *testing.T) { prepare: prepareTimeoutCase, wantMetrics: map[string]int64{ "bad_content": 0, + "bad_headers": 0, "bad_status": 0, "in_state": 2, "length": 0, @@ -178,6 +181,7 @@ func TestHTTPCheck_Collect(t *testing.T) { prepare: prepareRedirectSuccessCase, wantMetrics: map[string]int64{ "bad_content": 0, + "bad_headers": 0, "bad_status": 0, "in_state": 2, "length": 0, @@ -192,6 +196,7 @@ func TestHTTPCheck_Collect(t *testing.T) { prepare: prepareRedirectFailCase, wantMetrics: map[string]int64{ "bad_content": 0, + "bad_headers": 0, "bad_status": 0, "in_state": 2, "length": 0, @@ -206,6 +211,7 @@ func TestHTTPCheck_Collect(t *testing.T) { prepare: prepareBadStatusCase, wantMetrics: map[string]int64{ "bad_content": 0, + "bad_headers": 0, "bad_status": 1, "in_state": 2, "length": 0, @@ -220,6 +226,7 @@ func TestHTTPCheck_Collect(t *testing.T) { prepare: prepareBadContentCase, wantMetrics: map[string]int64{ "bad_content": 1, + "bad_headers": 0, "bad_status": 0, "in_state": 2, "length": 17, @@ -234,6 +241,7 @@ func TestHTTPCheck_Collect(t *testing.T) { prepare: prepareNoConnectionCase, wantMetrics: map[string]int64{ "bad_content": 0, + "bad_headers": 0, "bad_status": 0, "in_state": 2, "length": 0, @@ -244,10 +252,171 @@ func TestHTTPCheck_Collect(t *testing.T) { "timeout": 0, }, }, + "headers match include no value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Key: "header-key2"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "headers match include with value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Key: "header-key2", Value: "= header-value"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "headers match include no value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Key: "header-key99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "headers match include with value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Key: "header-key2", Value: "= header-value99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "headers match exclude no value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Exclude: true, Key: "header-key99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "headers match exclude with value success case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Exclude: true, Key: "header-key2", Value: "= header-value99"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 0, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 1, + "time": 0, + "timeout": 0, + }, + }, + "headers match exclude no value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Exclude: true, Key: "header-key2"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, + "headers match exclude with value bad headers case": { + prepare: prepareSuccessCase, + update: func(httpCheck *HTTPCheck) { + httpCheck.HeaderMatch = []HeaderMatchConfig{ + {Exclude: true, Key: "header-key2", Value: "= header-value"}, + } + }, + wantMetrics: map[string]int64{ + "bad_content": 0, + "bad_headers": 1, + "bad_status": 0, + "in_state": 2, + "length": 5, + "no_connection": 0, + "redirect": 0, + "success": 0, + "time": 0, + "timeout": 0, + }, + }, "cookie auth case": { prepare: prepareCookieAuthCase, wantMetrics: map[string]int64{ "bad_content": 0, + "bad_headers": 0, "bad_status": 0, "in_state": 2, "length": 0, @@ -265,6 +434,10 @@ func TestHTTPCheck_Collect(t *testing.T) { httpCheck, cleanup := test.prepare() defer cleanup() + if test.update != nil { + test.update(httpCheck) + } + require.True(t, httpCheck.Init()) var mx map[string]int64 @@ -288,6 +461,8 @@ func prepareSuccessCase() (*HTTPCheck, func()) { srv := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("header-key1", "header-value") + w.Header().Set("header-key2", "header-value") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("match")) })) diff --git a/modules/httpcheck/init.go b/modules/httpcheck/init.go index 29f05f3390..41d2ca0962 100644 --- a/modules/httpcheck/init.go +++ b/modules/httpcheck/init.go @@ -4,13 +4,21 @@ package httpcheck import ( "errors" + "fmt" "net/http" "regexp" "github.com/netdata/go.d.plugin/agent/module" + "github.com/netdata/go.d.plugin/pkg/matcher" "github.com/netdata/go.d.plugin/pkg/web" ) +type headersMatch struct { + exclude bool + key string + valMatcher matcher.Matcher +} + func (hc *HTTPCheck) validateConfig() error { if hc.URL == "" { return errors.New("'url' not set") @@ -29,6 +37,41 @@ func (hc *HTTPCheck) initResponseMatchRegexp() (*regexp.Regexp, error) { return regexp.Compile(hc.ResponseMatch) } +func (hc *HTTPCheck) initHeadersMatch() ([]headersMatch, error) { + if len(hc.HeaderMatch) == 0 { + return nil, nil + } + + var hms []headersMatch + + for _, v := range hc.HeaderMatch { + if v.Key == "" { + continue + } + + hm := headersMatch{ + exclude: v.Exclude, + key: v.Key, + valMatcher: nil, + } + + if v.Value != "" { + m, err := matcher.Parse(v.Value) + if err != nil { + return nil, fmt.Errorf("parse key '%s value '%s': %v", v.Key, v.Value, err) + } + if v.Exclude { + m = matcher.Not(m) + } + hm.valMatcher = m + } + + hms = append(hms, hm) + } + + return hms, nil +} + func (hc *HTTPCheck) initCharts() *module.Charts { charts := httpCheckCharts.Copy() diff --git a/modules/httpcheck/metadata.yaml b/modules/httpcheck/metadata.yaml index ac0458f9d5..fc7b70b408 100644 --- a/modules/httpcheck/metadata.yaml +++ b/modules/httpcheck/metadata.yaml @@ -69,6 +69,22 @@ modules: description: If the status code is accepted, the content of the response will be matched against this regular expression. default_value: "" required: false + - name: headers_match + description: "This option defines a set of rules that check for specific key-value pairs in the HTTP headers of the response." + default_value: "[]" + required: false + - name: headers_match.exclude + description: " This option determines whether the rule should check for the presence of the specified key-value pair or the absence of it." + default_value: false + required: false + - name: headers_match.key + description: "The exact name of the HTTP header to check for." + default_value: "" + required: true + - name: headers_match.value + description: "The [pattern](https://github.com/netdata/go.d.plugin/tree/master/pkg/matcher#supported-format) to match against the value of the specified header." + default_value: "" + required: false - name: cookie_file description: Path to cookie file. See [cookie file format](https://everything.curl.dev/http/cookies/fileformat). default_value: "" @@ -140,7 +156,7 @@ modules: jobs: - name: local url: http://127.0.0.1:8080 - - name: With status_accepted + - name: With `status_accepted` description: A basic example configuration with non-default status_accepted. config: | jobs: @@ -149,6 +165,34 @@ modules: status_accepted: - 200 - 204 + - name: With `header_match` + description: Example configurations with `header_match`. + config: | + jobs: + # The x-robots-tag key must present in the HTTP response header + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: x-robots-tag + # The x-robots-tag key with a specific value must present in the HTTP response header + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: x-robots-tag + value: '= noindex,nofollow' + # The x-robots-tag key must not be present in the HTTP response header + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: x-robots-tag + exclude: yes + # The x-robots-tag key with a specific value must not be present in the HTTP response header + - name: local + url: http://127.0.0.1:8080 + header_match: + - key: x-robots-tag + exclude: yes + value: '= noindex,nofollow' - name: HTTP authentication description: Basic HTTP authentication. config: | @@ -216,6 +260,7 @@ modules: - name: redirect - name: no_connection - name: bad_content + - name: bad_header - name: bad_status - name: httpcheck.in_state description: HTTP Current State Duration diff --git a/modules/httpcheck/metrics.go b/modules/httpcheck/metrics.go index 8d807fe10b..676346fa0f 100644 --- a/modules/httpcheck/metrics.go +++ b/modules/httpcheck/metrics.go @@ -15,5 +15,6 @@ type status struct { Redirect bool `stm:"redirect"` BadContent bool `stm:"bad_content"` BadStatusCode bool `stm:"bad_status"` + BadHeader bool `stm:"bad_header"` NoConnection bool `stm:"no_connection"` // All other errors basically }