Merge pull request #108 from redpanda-data/http-no-redirect

feat: add follow_redirects field to http processor

CHANGELOG.md

@@ -3,6 +3,12 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## 4.40.0 - TBD
+
+### Added
+
+- Field `follow_redirects` added to the `http` processor. (@ooesili)
+
 ## 4.39.0 - 2024-10-14
 
 ### Added

internal/httpclient/client.go

@@ -29,12 +29,13 @@ type Client struct {
 	clientCancel func()
 
 	// Request execution and retry logic
-	rateLimit     string
-	numRetries    int
-	retryThrottle *throttle.Type
-	backoffOn     map[int]struct{}
-	dropOn        map[int]struct{}
-	successOn     map[int]struct{}
+	rateLimit       string
+	numRetries      int
+	followRedirects bool
+	retryThrottle   *throttle.Type
+	backoffOn       map[int]struct{}
+	dropOn          map[int]struct{}
+	successOn       map[int]struct{}
 
 	// Response extraction
 	metaExtractFilter *service.MetadataFilter
@@ -75,6 +76,13 @@ func NewClientFromOldConfig(conf OldConfig, mgr *service.Resources, opts ...Requ
 		h.client.Timeout = conf.Timeout
 	}
 
+	h.followRedirects = conf.FollowRedirects
+	if !h.followRedirects {
+		h.client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		}
+	}
+
 	if conf.TLSEnabled && conf.TLSConf != nil {
 		if c, ok := http.DefaultTransport.(*http.Transport); ok {
 			cloned := c.Clone()
@@ -293,6 +301,9 @@ func (h *Client) checkStatus(code int) (succeeded bool, retStrat retryStrategy)
 	if _, exists := h.successOn[code]; exists {
 		return true, noRetry
 	}
+	if !h.followRedirects && code >= 300 && code <= 399 {
+		return true, noRetry
+	}
 	if code < 200 || code > 299 {
 		return false, retryLinear
 	}

internal/httpclient/client_test.go

@@ -794,3 +794,35 @@ tls:
 	require.NoError(t, err)
 	assert.Equal(t, "HELLO WORLD", string(mBytes))
 }
+
+func TestHTTPClientNoFollowRedirects(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "https://example.com", http.StatusMovedPermanently)
+	}))
+	defer ts.Close()
+
+	conf := clientConfig(t, `
+url: %v
+follow_redirects: false
+extract_headers:
+  include_patterns:
+  - '^location$'
+`, ts.URL)
+
+	h, err := NewClientFromOldConfig(conf, service.MockResources())
+	require.NoError(t, err)
+
+	resBatch, err := h.Send(context.Background(), service.MessageBatch{
+		service.NewMessage([]byte("hello world")),
+	})
+	require.NoError(t, err)
+	require.Len(t, resBatch, 1)
+
+	status, ok := resBatch[0].MetaGet("http_status_code")
+	require.True(t, ok)
+	require.Equal(t, "301", status)
+
+	location, ok := resBatch[0].MetaGet("location")
+	require.True(t, ok)
+	require.Equal(t, "https://example.com", location)
+}

internal/httpclient/config.go

@@ -21,6 +21,7 @@ const (
 	hcFieldRetryPeriod         = "retry_period"
 	hcFieldMaxRetryBackoff     = "max_retry_backoff"
 	hcFieldRetries             = "retries"
+	hcFieldFollowRedirects     = "follow_redirects"
 	hcFieldBackoffOn           = "backoff_on"
 	hcFieldDropOn              = "drop_on"
 	hcFieldSuccessfulOn        = "successful_on"
@@ -85,6 +86,10 @@ func ConfigField(defaultVerb string, forOutput bool, extraChildren ...*service.C
 			Description("The maximum number of retry attempts to make.").
 			Advanced().
 			Default(3),
+		service.NewBoolField(hcFieldFollowRedirects).
+			Description("Whether or not to transparently follow redirects, i.e. responses with 300-399 status codes. If disabled, the response message will contain the body, status, and headers from the redirect response and the processor will not make a request to the URL set in the Location header of the response.").
+			Advanced().
+			Default(true),
 		service.NewIntListField(hcFieldBackoffOn).
 			Description("A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually.").
 			Advanced().
@@ -140,6 +145,9 @@ func ConfigFromParsed(pConf *service.ParsedConfig) (conf OldConfig, err error) {
 	if conf.NumRetries, err = pConf.FieldInt(hcFieldRetries); err != nil {
 		return
 	}
+	if conf.FollowRedirects, err = pConf.FieldBool(hcFieldFollowRedirects); err != nil {
+		return
+	}
 	if conf.BackoffOn, err = pConf.FieldIntList(hcFieldBackoffOn); err != nil {
 		return
 	}
@@ -175,6 +183,7 @@ type OldConfig struct {
 	Retry               time.Duration
 	MaxBackoff          time.Duration
 	NumRetries          int
+	FollowRedirects     bool
 	BackoffOn           []int
 	DropOn              []int
 	SuccessfulOn        []int