HTTP Analytics Module

adapters/appnexus/appnexus_test.go

@@ -50,3 +50,7 @@ type FakeRandomNumberGenerator struct {
 func (f FakeRandomNumberGenerator) GenerateInt63() int64 {
 	return f.Number
 }
+
+func (f FakeRandomNumberGenerator) GenerateFloat64() float64 {
+	return 0
+}

analytics/build/build.go

@@ -6,6 +6,7 @@ import (
 	"github.com/prebid/prebid-server/v2/analytics"
 	"github.com/prebid/prebid-server/v2/analytics/clients"
 	"github.com/prebid/prebid-server/v2/analytics/filesystem"
+	"github.com/prebid/prebid-server/v2/analytics/http"
 	"github.com/prebid/prebid-server/v2/analytics/pubstack"
 	"github.com/prebid/prebid-server/v2/config"
 	"github.com/prebid/prebid-server/v2/privacy"
@@ -38,6 +39,19 @@ func New(analytics *config.Analytics) analytics.Runner {
 			glog.Errorf("Could not initialize PubstackModule: %v", err)
 		}
 	}
+
+	if analytics.Http.Enabled {
+		httpModule, err := http.NewModule(
+			clients.GetDefaultHttpInstance(),
+			analytics.Http,
+			clock.New())
+		if err == nil {
+			modules["http"] = httpModule
+		} else {
+			glog.Errorf("Could not initialize Http Anayltics: %v", err)
+		}
+	}
+
 	return modules
 }
 

analytics/build/build_test.go

@@ -139,6 +139,43 @@ func TestNewPBSAnalytics_Pubstack(t *testing.T) {
 	assert.Equal(t, len(instanceWithError), 0)
 }
 
+func TestNewModuleHttp(t *testing.T) {
+	httpAnalyticsWithoutError := New(&config.Analytics{
+		Http: config.AnalyticsHttp{
+			Enabled: true,
+			Endpoint: config.AnalyticsHttpEndpoint{
+				Url:     "http://localhost:8080",
+				Timeout: "1s",
+			},
+			Buffers: config.AnalyticsBuffer{
+				BufferSize: "100KB",
+				EventCount: 50,
+				Timeout:    "30s",
+			},
+			Auction: config.AnalyticsFeature{
+				SampleRate: 1,
+			},
+		},
+	})
+	instanceWithoutError := httpAnalyticsWithoutError.(enabledAnalytics)
+
+	assert.Equal(t, len(instanceWithoutError), 1)
+
+	httpAnalyticsWithError := New(&config.Analytics{
+		Http: config.AnalyticsHttp{
+			Enabled: true,
+			Endpoint: config.AnalyticsHttpEndpoint{
+				Url: "http://localhost:8080",
+			},
+			Auction: config.AnalyticsFeature{
+				SampleRate: 1,
+				Filter:     "invalid == filter",
+			},
+		},
+	})
+	instanceWithError := httpAnalyticsWithError.(enabledAnalytics)
+	assert.Equal(t, len(instanceWithError), 0)
+}
 func TestSampleModuleActivitiesAllowed(t *testing.T) {
 	var count int
 	am := initAnalytics(&count)

analytics/http/Readme.md

@@ -0,0 +1,67 @@
+# Http Analytics
+
+This module sends selected analytics events to a http endpoint.
+
+Please make sure you take a look at the possible configuration and the filter options
+
+## Configuration
+
+```yaml
+analytics:
+    http:
+        # Required: enable the module
+        enabled: true
+        endpoint: 
+            # Required: url where the endpoint post data to
+            url: "https://my-rest-endpoint.com"
+            # Required: timeout for the request
+            timeout: "2s"
+            # Optional: enables gzip compression for the payload
+            gzip: false
+            # Optional: additional headers send in every request
+            additional_headers:
+                X-My-header: "some-thing"
+        buffer: # Flush events when (first condition reached)
+            # Size of the buffer in bytes
+            size: "2MB" # greater than 2MB
+            count : 100 # greater than 100 events
+            timeout: "15m" # greater than 15 minutes
+        auction: 
+            enabled: false # enable auction tracking
+            sample_rate: 1 # sample rate 0-1.0 to sample the event
+            filter: "RequestWrapper.BidRequest.App.ID == '123'" # Optional filter
+        video:
+            enabled: false # enable video tracking
+            sample_rate: 1 
+            filter: "" 
+        amp:
+            enabled: false # enable amp tracking
+            sample_rate: 1 
+            filter: "" 
+        setuid:
+            enabled: false # enable setuid tracking
+            sample_rate: 1 
+            filter: "" 
+        cookie_sync:
+            enabled: false # enable cookie sync tracking
+            sample_rate: 1 
+            filter: "" 
+        notification:
+            enabled: false # enable notification tracking
+            sample_rate: 1 
+            filter: "" 
+
+```
+
+### Sample Rate
+
+The sample rate has to be between `0.0` (never sample) and `1.0` (always sample). The sample rate is always evaluated and defaults to 0
+
+### Filter
+
+The module uses [github.com/antonmedv/expr](github.com/antonmedv/expr) for complexer filter options. The analytics object is always passed into the expression.
+
+#### Samples:
+
+- Auction: `RequestWrapper.BidRequest.App.ID == '123'`
+- Video: `VideoRequest.Video.MinDuration > 200`

analytics/http/filters.go

@@ -0,0 +1,213 @@
+package http
+
+import (
+	"github.com/antonmedv/expr"
+	"github.com/antonmedv/expr/vm"
+	"github.com/golang/glog"
+	"github.com/prebid/prebid-server/v2/analytics"
+	"github.com/prebid/prebid-server/v2/config"
+	"github.com/prebid/prebid-server/v2/util/randomutil"
+)
+
+type (
+	auctionFilter      func(event *analytics.AuctionObject) bool
+	ampFilter          func(event *analytics.AmpObject) bool
+	cookieSyncFilter   func(event *analytics.CookieSyncObject) bool
+	notificationFilter func(event *analytics.NotificationEvent) bool
+	setUIDFilter       func(event *analytics.SetUIDObject) bool
+	videoFilter        func(event *analytics.VideoObject) bool
+)
+
+func createAuctionFilter(
+	feature config.AnalyticsFeature,
+	randomGenerator randomutil.RandomGenerator,
+) (auctionFilter, error) {
+	var filterProgram *vm.Program
+	var err error
+	if feature.Filter != "" {
+		// precompile the filter expression for performance, make sure we return a boolean from the expression
+		filterProgram, err = expr.Compile(feature.Filter, expr.Env(analytics.AuctionObject{}), expr.AsBool())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return func(event *analytics.AuctionObject) bool {
+		// Disable tracking for nil events or events with a sample rate of 0
+		if event == nil || feature.SampleRate <= 0 || randomGenerator.GenerateFloat64() > feature.SampleRate {
+			return false
+		}
+
+		// Use a filter is one is defined
+		if filterProgram != nil {
+			output, err := expr.Run(filterProgram, event)
+			if err != nil {
+				glog.Errorf("[HttpAnalytics] Error filter auction: %v", err)
+				return false
+			}
+			return output.(bool)
+		}
+
+		return true
+	}, nil
+}
+
+func createAmpFilter(
+	feature config.AnalyticsFeature,
+	randomGenerator randomutil.RandomGenerator,
+) (ampFilter, error) {
+	var filterProgram *vm.Program
+	var err error
+
+	if feature.Filter != "" {
+		filterProgram, err = expr.Compile(feature.Filter, expr.Env(analytics.AmpObject{}), expr.AsBool())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return func(event *analytics.AmpObject) bool {
+		if event == nil || feature.SampleRate <= 0 || randomGenerator.GenerateFloat64() > feature.SampleRate {
+			return false
+		}
+
+		if filterProgram != nil {
+			output, err := expr.Run(filterProgram, event)
+			if err != nil {
+				glog.Errorf("[HttpAnalytics] Error filter amp: %v", err)
+				return false
+			}
+			return output.(bool)
+		}
+
+		return true
+	}, nil
+}
+
+func createCookieSyncFilter(
+	feature config.AnalyticsFeature,
+	randomGenerator randomutil.RandomGenerator,
+) (cookieSyncFilter, error) {
+	var filterProgram *vm.Program
+	var err error
+
+	if feature.Filter != "" {
+		filterProgram, err = expr.Compile(feature.Filter, expr.Env(analytics.CookieSyncObject{}), expr.AsBool())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return func(event *analytics.CookieSyncObject) bool {
+		if event == nil || feature.SampleRate <= 0 || randomGenerator.GenerateFloat64() > feature.SampleRate {
+			return false
+		}
+
+		if filterProgram != nil {
+			output, err := expr.Run(filterProgram, event)
+			if err != nil {
+				glog.Errorf("[HttpAnalytics] Error filter cookie sync: %v", err)
+				return false
+			}
+			return output.(bool)
+		}
+
+		return true
+	}, nil
+}
+
+func createNotificationFilter(
+	feature config.AnalyticsFeature,
+	randomGenerator randomutil.RandomGenerator,
+) (notificationFilter, error) {
+	var filterProgram *vm.Program
+	var err error
+
+	if feature.Filter != "" {
+		filterProgram, err = expr.Compile(feature.Filter, expr.Env(analytics.NotificationEvent{}), expr.AsBool())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return func(event *analytics.NotificationEvent) bool {
+		if event == nil || feature.SampleRate <= 0 || randomGenerator.GenerateFloat64() > feature.SampleRate {
+			return false
+		}
+
+		if filterProgram != nil {
+			output, err := expr.Run(filterProgram, event)
+			if err != nil {
+				glog.Errorf("[HttpAnalytics] Error filter notification: %v", err)
+				return false
+			}
+			return output.(bool)
+		}
+
+		return true
+	}, nil
+}
+
+func createSetUIDFilter(
+	feature config.AnalyticsFeature,
+	randomGenerator randomutil.RandomGenerator,
+) (setUIDFilter, error) {
+	var filterProgram *vm.Program
+	var err error
+
+	if feature.Filter != "" {
+		filterProgram, err = expr.Compile(feature.Filter, expr.Env(analytics.SetUIDObject{}), expr.AsBool())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return func(event *analytics.SetUIDObject) bool {
+		if event == nil || feature.SampleRate <= 0 || randomGenerator.GenerateFloat64() > feature.SampleRate {
+			return false
+		}
+
+		if filterProgram != nil {
+			output, err := expr.Run(filterProgram, event)
+			if err != nil {
+				glog.Errorf("[HttpAnalytics] Error filter setUID: %v", err)
+				return false
+			}
+			return output.(bool)
+		}
+
+		return true
+	}, nil
+}
+
+func createVideoFilter(
+	feature config.AnalyticsFeature,
+	randomGenerator randomutil.RandomGenerator,
+) (videoFilter, error) {
+	var filterProgram *vm.Program
+	var err error
+
+	if feature.Filter != "" {
+		filterProgram, err = expr.Compile(feature.Filter, expr.Env(analytics.VideoObject{}), expr.AsBool())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return func(event *analytics.VideoObject) bool {
+		if event == nil || feature.SampleRate <= 0 || randomGenerator.GenerateFloat64() > feature.SampleRate {
+			return false
+		}
+
+		if filterProgram != nil {
+			output, err := expr.Run(filterProgram, event)
+			if err != nil {
+				glog.Errorf("[HttpAnalytics] Error filter video: %v", err)
+				return false
+			}
+			return output.(bool)
+		}
+
+		return true
+	}, nil
+}