diff --git a/go.mod b/go.mod index e421fbd4..237bc87b 100644 --- a/go.mod +++ b/go.mod @@ -51,8 +51,8 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/dlclark/regexp2 v1.7.0 github.com/dop251/goja v0.0.0-20220722151623-4765a9872229 - github.com/goccy/go-json v0.9.10 github.com/hashicorp/hcl/v2 v2.13.0 + github.com/json-iterator/go v1.1.12 github.com/snowplow/snowplow-golang-tracker/v2 v2.4.1 github.com/yuin/gluamapper v0.0.0-20150323120927-d836955830e7 github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 @@ -91,7 +91,6 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 17e927ba..262d0250 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,6 @@ github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3a github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/goccy/go-json v0.9.10 h1:hCeNmprSNLB8B8vQKWl6DpuH0t60oEs+TAk9a7CScKc= -github.com/goccy/go-json v0.9.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= diff --git a/pkg/transform/engine/engine.go b/pkg/transform/engine/engine.go index 6b1cd84e..deb15e5f 100644 --- a/pkg/transform/engine/engine.go +++ b/pkg/transform/engine/engine.go @@ -7,9 +7,13 @@ package engine import ( + jsoniter "github.com/json-iterator/go" "github.com/snowplow-devops/stream-replicator/pkg/transform" ) +// make a jsoniter instance that won't escape html +var json = jsoniter.Config{}.Froze() + // functionMaker is the interface that wraps the MakeFunction method type functionMaker interface { // MakeFunction returns a TransformationFunction that runs diff --git a/pkg/transform/engine/engine_javascript.go b/pkg/transform/engine/engine_javascript.go index 0b32be03..60e81170 100644 --- a/pkg/transform/engine/engine_javascript.go +++ b/pkg/transform/engine/engine_javascript.go @@ -14,7 +14,6 @@ import ( goja "github.com/dop251/goja" gojaparser "github.com/dop251/goja/parser" - gojson "github.com/goccy/go-json" "github.com/mitchellh/mapstructure" "github.com/snowplow-devops/stream-replicator/pkg/models" @@ -153,7 +152,7 @@ func (e *JSEngine) MakeFunction(funcName string) transform.TransformationFunctio message.Data = []byte(protoData) case map[string]interface{}: // encode - encoded, err := gojson.MarshalWithOption(protoData, gojson.DisableHTMLEscape()) + encoded, err := json.Marshal(protoData) if err != nil { message.SetError(fmt.Errorf("error encoding message data")) return nil, nil, message, nil diff --git a/pkg/transform/engine/engine_javascript_test.go b/pkg/transform/engine/engine_javascript_test.go index 0736325f..03d6d324 100644 --- a/pkg/transform/engine/engine_javascript_test.go +++ b/pkg/transform/engine/engine_javascript_test.go @@ -3,7 +3,6 @@ // Unauthorized copying of this file via any medium is strictly prohibited. // // Copyright (c) 2020-2022 Snowplow Analytics Ltd. All rights reserved. -// package engine import ( @@ -20,6 +19,19 @@ import ( "github.com/snowplow-devops/stream-replicator/pkg/models" ) +type JSTestCase struct { + Scenario string + Src string + DisableSourceMaps bool + SpMode bool + Input *models.Message + InterState interface{} + Expected map[string]*models.Message + ExpInterState interface{} + IsJSON bool + Error error +} + func TestJSLayer(t *testing.T) { assert := assert.New(t) @@ -36,15 +48,7 @@ func TestJSLayer(t *testing.T) { func TestJSEngineMakeFunction_SpModeFalse_IntermediateNil(t *testing.T) { var testInterState interface{} = nil var testSpMode = false - testCases := []struct { - Src string - Scenario string - DisableSourceMaps bool - Input *models.Message - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []JSTestCase{ { Src: ` function main(x) { @@ -191,7 +195,8 @@ function main(x) { PartitionKey: "", Data: string(testJsJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Src: ` @@ -224,7 +229,8 @@ function main(x) { PartitionKey: "", Data: string(testJsJSONChanged1), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Src: ` @@ -260,7 +266,8 @@ function main(x) { PartitionKey: "", Data: string(testJsJSONChanged2), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Src: ` @@ -291,6 +298,7 @@ function main(x) { "failed": nil, }, ExpInterState: nil, + IsJSON: true, Error: nil, }, { @@ -478,9 +486,9 @@ function main(x) { } } - assertMessagesCompareJs(t, s, tt.Expected["success"]) - assertMessagesCompareJs(t, f, tt.Expected["filtered"]) - assertMessagesCompareJs(t, e, tt.Expected["failed"]) + assertMessagesCompareJs(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareJs(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareJs(t, e, tt.Expected["failed"], tt.IsJSON) }) } } @@ -488,15 +496,7 @@ function main(x) { func TestJSEngineMakeFunction_SpModeTrue_IntermediateNil(t *testing.T) { var testInterState interface{} = nil var testSpMode bool = true - testCases := []struct { - Scenario string - Src string - DisableSourceMaps bool - Input *models.Message - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []JSTestCase{ { Scenario: "identity", Src: ` @@ -522,7 +522,8 @@ function main(x) { PartitionKey: "", Data: testJSMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "filtering", @@ -674,25 +675,16 @@ function main(x) { } } - assertMessagesCompareJs(t, s, tt.Expected["success"]) - assertMessagesCompareJs(t, f, tt.Expected["filtered"]) - assertMessagesCompareJs(t, e, tt.Expected["failed"]) + assertMessagesCompareJs(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareJs(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareJs(t, e, tt.Expected["failed"], tt.IsJSON) }) } } func TestJSEngineMakeFunction_IntermediateState_SpModeFalse(t *testing.T) { testSpMode := false - testCases := []struct { - Scenario string - Src string - DisableSourceMaps bool - Input *models.Message - InterState interface{} - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []JSTestCase{ { Scenario: "intermediateState_EngineProtocol_Map", Src: ` @@ -723,7 +715,8 @@ function main(x) { PartitionKey: "", Data: testJSMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_EngineProtocol_String", @@ -755,7 +748,8 @@ function main(x) { PartitionKey: "", Data: string(testJsJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_not_EngineProtocol_spMode_true", @@ -783,7 +777,8 @@ function main(x) { PartitionKey: "", Data: string(testJsJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_not_EngineProtocol_spMode_false", @@ -811,7 +806,8 @@ function main(x) { PartitionKey: "", Data: string(testJsJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, } @@ -860,25 +856,16 @@ function main(x) { } } - assertMessagesCompareJs(t, s, tt.Expected["success"]) - assertMessagesCompareJs(t, f, tt.Expected["filtered"]) - assertMessagesCompareJs(t, e, tt.Expected["failed"]) + assertMessagesCompareJs(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareJs(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareJs(t, e, tt.Expected["failed"], tt.IsJSON) }) } } func TestJSEngineMakeFunction_IntermediateState_SpModeTrue(t *testing.T) { testSpMode := true - testCases := []struct { - Scenario string - Src string - DisableSourceMaps bool - Input *models.Message - InterState interface{} - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []JSTestCase{ { Scenario: "intermediateState_EngineProtocol_Map", Src: ` @@ -909,7 +896,8 @@ function main(x) { PartitionKey: "", Data: testJSMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_EngineProtocol_String", @@ -941,7 +929,8 @@ function main(x) { PartitionKey: "", Data: string(testJsJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_notEngineProtocol_notSpEnriched", @@ -965,6 +954,7 @@ function main(x) { }, }, ExpInterState: nil, + IsJSON: true, Error: fmt.Errorf("Cannot parse"), }, { @@ -993,7 +983,8 @@ function main(x) { PartitionKey: "", Data: testJSMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, } @@ -1042,25 +1033,16 @@ function main(x) { } } - assertMessagesCompareJs(t, s, tt.Expected["success"]) - assertMessagesCompareJs(t, f, tt.Expected["filtered"]) - assertMessagesCompareJs(t, e, tt.Expected["failed"]) + assertMessagesCompareJs(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareJs(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareJs(t, e, tt.Expected["failed"], tt.IsJSON) }) } } func TestJSEngineMakeFunction_SetPK(t *testing.T) { var testInterState interface{} = nil - testCases := []struct { - Scenario string - Src string - DisableSourceMaps bool - SpMode bool - Input *models.Message - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []JSTestCase{ { Scenario: "onlySetPk_spModeTrue", Src: ` @@ -1088,7 +1070,8 @@ function main(x) { PartitionKey: "newPk", Data: testJSMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "onlySetPk_spModeFalse", @@ -1194,9 +1177,9 @@ function main(x) { } } - assertMessagesCompareJs(t, s, tt.Expected["success"]) - assertMessagesCompareJs(t, f, tt.Expected["filtered"]) - assertMessagesCompareJs(t, e, tt.Expected["failed"]) + assertMessagesCompareJs(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareJs(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareJs(t, e, tt.Expected["failed"], tt.IsJSON) }) } } @@ -1433,7 +1416,7 @@ func testJSEngineFunc(c *JSEngineConfig) (*JSEngineConfig, error) { // Helper function to compare messages and avoid using reflect.DeepEqual // on errors. Compares all but the error field of messages. -func assertMessagesCompareJs(t *testing.T, act, exp *models.Message) { +func assertMessagesCompareJs(t *testing.T, act, exp *models.Message, isJSON bool) { t.Helper() ok := false @@ -1444,7 +1427,11 @@ func assertMessagesCompareJs(t *testing.T, act, exp *models.Message) { default: var dataOk bool pkOk := act.PartitionKey == exp.PartitionKey - dataOk = reflect.DeepEqual(act.Data, exp.Data) + if isJSON { + dataOk = assert.JSONEq(t, string(exp.Data), string(act.Data)) + } else { + dataOk = reflect.DeepEqual(act.Data, exp.Data) + } cTimeOk := reflect.DeepEqual(act.TimeCreated, exp.TimeCreated) pTimeOk := reflect.DeepEqual(act.TimePulled, exp.TimePulled) tTimeOk := reflect.DeepEqual(act.TimeTransformed, exp.TimeTransformed) @@ -1519,7 +1506,6 @@ var testJSMap = map[string]interface{}{ var testJsTsv = []byte(`test-data<> pc 2019-05-10 14:40:37.436 2019-05-10 14:40:35.972 2019-05-10 14:40:35.551 unstruct e9234345-f042-46ad-b1aa-424464066a33 py-0.8.2 ssc-0.15.0-googlepubsub beam-enrich-0.2.0-common-0.36.0 user 1.2.3.4 d26822f5-52cc-4292-8f77-14ef6b7a27e2 {"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/add_to_cart/jsonschema/1-0-0","data":{"sku":"item41","quantity":2,"unitPrice":32.4,"currency":"GBP"}}} python-requests/2.21.0 2019-05-10 14:40:35.000 {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-0","data":{"deviceBrand":"Unknown","deviceName":"Unknown","operatingSystemName":"Unknown","agentVersionMajor":"2","layoutEngineVersionMajor":"??","deviceClass":"Unknown","agentNameVersionMajor":"python-requests 2","operatingSystemClass":"Unknown","layoutEngineName":"Unknown","agentName":"python-requests","agentVersion":"2.21.0","layoutEngineClass":"Unknown","agentNameVersion":"python-requests 2.21.0","operatingSystemVersion":"??","agentClass":"Special","layoutEngineVersion":"??"}}]} 2019-05-10 14:40:35.972 com.snowplowanalytics.snowplow add_to_cart jsonschema 1-0-0 `) -// // corresponding JSON to previous TSV var testJsJSON = []byte(`{"app_id":"test-data<>","collector_tstamp":"2019-05-10T14:40:35.972Z","contexts_nl_basjes_yauaa_context_1":[{"agentClass":"Special","agentName":"python-requests","agentNameVersion":"python-requests 2.21.0","agentNameVersionMajor":"python-requests 2","agentVersion":"2.21.0","agentVersionMajor":"2","deviceBrand":"Unknown","deviceClass":"Unknown","deviceName":"Unknown","layoutEngineClass":"Unknown","layoutEngineName":"Unknown","layoutEngineVersion":"??","layoutEngineVersionMajor":"??","operatingSystemClass":"Unknown","operatingSystemName":"Unknown","operatingSystemVersion":"??"}],"derived_tstamp":"2019-05-10T14:40:35.972Z","dvce_created_tstamp":"2019-05-10T14:40:35.551Z","dvce_sent_tstamp":"2019-05-10T14:40:35Z","etl_tstamp":"2019-05-10T14:40:37.436Z","event":"unstruct","event_format":"jsonschema","event_id":"e9234345-f042-46ad-b1aa-424464066a33","event_name":"add_to_cart","event_vendor":"com.snowplowanalytics.snowplow","event_version":"1-0-0","network_userid":"d26822f5-52cc-4292-8f77-14ef6b7a27e2","platform":"pc","unstruct_event_com_snowplowanalytics_snowplow_add_to_cart_1":{"currency":"GBP","quantity":2,"sku":"item41","unitPrice":32.4},"user_id":"user","user_ipaddress":"1.2.3.4","useragent":"python-requests/2.21.0","v_collector":"ssc-0.15.0-googlepubsub","v_etl":"beam-enrich-0.2.0-common-0.36.0","v_tracker":"py-0.8.2"}`) diff --git a/pkg/transform/engine/engine_lua.go b/pkg/transform/engine/engine_lua.go index 2b6e6eca..dc2f307a 100644 --- a/pkg/transform/engine/engine_lua.go +++ b/pkg/transform/engine/engine_lua.go @@ -13,7 +13,6 @@ import ( "strings" "time" - gojson "github.com/goccy/go-json" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/yuin/gluamapper" @@ -189,7 +188,7 @@ func (e *LuaEngine) MakeFunction(funcName string) transform.TransformationFuncti // encode if encode { - encoded, err := gojson.MarshalWithOption(protocol.Data, gojson.DisableHTMLEscape()) + encoded, err := json.Marshal(protocol.Data) if err != nil { message.SetError(fmt.Errorf("error encoding message data")) return nil, nil, message, nil diff --git a/pkg/transform/engine/engine_lua_test.go b/pkg/transform/engine/engine_lua_test.go index e7dfdd09..287ef70a 100644 --- a/pkg/transform/engine/engine_lua_test.go +++ b/pkg/transform/engine/engine_lua_test.go @@ -20,6 +20,19 @@ import ( "github.com/snowplow-devops/stream-replicator/pkg/transform" ) +type LuaTestCase struct { + Scenario string + Src string + Sandbox bool + SpMode bool + Input *models.Message + InterState interface{} + Expected map[string]*models.Message + ExpInterState interface{} + IsJSON bool + Error error +} + func TestLuaLayer(t *testing.T) { assert := assert.New(t) layer, err := LuaEngineConfigFunction(&LuaEngineConfig{ @@ -35,15 +48,7 @@ func TestLuaLayer(t *testing.T) { func TestLuaEngineMakeFunction_SpModeFalse_IntermediateNil(t *testing.T) { var testInterState interface{} = nil var testSpMode bool = false - testCases := []struct { - Src string - Scenario string - Sandbox bool - Input *models.Message - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []LuaTestCase{ { Src: ` function main(x) @@ -188,7 +193,8 @@ end PartitionKey: "", Data: string(snowplowJSON1), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Src: ` @@ -227,7 +233,8 @@ end PartitionKey: "", Data: string(snowplowJSON1ChangedLua), }, - Error: nil, + Error: nil, + IsJSON: true, }, { Src: ` @@ -259,6 +266,7 @@ end "failed": nil, }, ExpInterState: nil, + IsJSON: true, Error: nil, }, { @@ -448,9 +456,9 @@ end } } - assertMessagesCompareLua(t, s, tt.Expected["success"]) - assertMessagesCompareLua(t, f, tt.Expected["filtered"]) - assertMessagesCompareLua(t, e, tt.Expected["failed"]) + assertMessagesCompareLua(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareLua(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareLua(t, e, tt.Expected["failed"], tt.IsJSON) }) } } @@ -458,15 +466,7 @@ end func TestLuaEngineMakeFunction_SpModeTrue_IntermediateNil(t *testing.T) { var testInterState interface{} = nil var testSpMode bool = true - testCases := []struct { - Scenario string - Src string - Sandbox bool - Input *models.Message - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []LuaTestCase{ { Scenario: "main", Src: ` @@ -492,7 +492,8 @@ end PartitionKey: "", Data: testLuaMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "filtering", @@ -643,25 +644,16 @@ end } } - assertMessagesCompareLua(t, s, tt.Expected["success"]) - assertMessagesCompareLua(t, f, tt.Expected["filtered"]) - assertMessagesCompareLua(t, e, tt.Expected["failed"]) + assertMessagesCompareLua(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareLua(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareLua(t, e, tt.Expected["failed"], tt.IsJSON) }) } } func TestLuaEngineMakeFunction_IntermediateState_SpModeFalse(t *testing.T) { testSpMode := false - testCases := []struct { - Scenario string - Src string - Sandbox bool - Input *models.Message - InterState interface{} - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []LuaTestCase{ { Scenario: "intermediateState_EngineProtocol_Map", Src: ` @@ -692,7 +684,8 @@ end PartitionKey: "", Data: testLuaMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_EngineProtocol_String", @@ -724,7 +717,8 @@ end PartitionKey: "", Data: string(testLuaJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_not_EngineProtocol_nonSpEnriched", @@ -752,7 +746,8 @@ end PartitionKey: "", Data: string(testLuaJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_not_EngineProtocol_SpEnriched", @@ -829,9 +824,9 @@ end } } - assertMessagesCompareLua(t, s, tt.Expected["success"]) - assertMessagesCompareLua(t, f, tt.Expected["filtered"]) - assertMessagesCompareLua(t, e, tt.Expected["failed"]) + assertMessagesCompareLua(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareLua(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareLua(t, e, tt.Expected["failed"], tt.IsJSON) }) } } @@ -839,16 +834,7 @@ end func TestLuaEngineMakeFunction_IntermediateState_SpModeTrue(t *testing.T) { testSpMode := true - testCases := []struct { - Scenario string - Src string - Sandbox bool - Input *models.Message - InterState interface{} - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []LuaTestCase{ { Scenario: "intermediateState_EngineProtocol_Map", Src: ` @@ -879,7 +865,8 @@ end PartitionKey: "", Data: testLuaMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_EngineProtocol_String", @@ -911,7 +898,8 @@ end PartitionKey: "", Data: string(testLuaJSON), }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "intermediateState_notEngineProtocol_notSpEnriched", @@ -935,6 +923,7 @@ end }, }, ExpInterState: nil, + IsJSON: true, Error: fmt.Errorf("Cannot parse"), }, { @@ -963,7 +952,8 @@ end PartitionKey: "", Data: testLuaMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, } @@ -1012,25 +1002,16 @@ end } } - assertMessagesCompareLua(t, s, tt.Expected["success"]) - assertMessagesCompareLua(t, f, tt.Expected["filtered"]) - assertMessagesCompareLua(t, e, tt.Expected["failed"]) + assertMessagesCompareLua(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareLua(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareLua(t, e, tt.Expected["failed"], tt.IsJSON) }) } } func TestLuaEngineMakeFunction_SetPK(t *testing.T) { var testInterState interface{} = nil - testCases := []struct { - Scenario string - Src string - Sandbox bool - SpMode bool - Input *models.Message - Expected map[string]*models.Message - ExpInterState interface{} - Error error - }{ + testCases := []LuaTestCase{ { Scenario: "onlySetPk_spModeTrue", Src: ` @@ -1058,7 +1039,8 @@ end PartitionKey: "newPk", Data: testLuaMap, }, - Error: nil, + IsJSON: true, + Error: nil, }, { Scenario: "onlySetPk_spModeFalse", @@ -1165,9 +1147,9 @@ end } } - assertMessagesCompareLua(t, s, tt.Expected["success"]) - assertMessagesCompareLua(t, f, tt.Expected["filtered"]) - assertMessagesCompareLua(t, e, tt.Expected["failed"]) + assertMessagesCompareLua(t, s, tt.Expected["success"], tt.IsJSON) + assertMessagesCompareLua(t, f, tt.Expected["filtered"], tt.IsJSON) + assertMessagesCompareLua(t, e, tt.Expected["failed"], tt.IsJSON) }) } } @@ -1695,7 +1677,7 @@ end // Helper function to compare messages and avoid using reflect.DeepEqual // on errors. Compares all but the error field of messages. -func assertMessagesCompareLua(t *testing.T, act, exp *models.Message) { +func assertMessagesCompareLua(t *testing.T, act, exp *models.Message, IsJSON bool) { t.Helper() ok := false @@ -1705,7 +1687,13 @@ func assertMessagesCompareLua(t *testing.T, act, exp *models.Message) { case exp == nil: default: pkOk := act.PartitionKey == exp.PartitionKey - dataOk := reflect.DeepEqual(act.Data, exp.Data) + var dataOk bool + if IsJSON { + dataOk = assert.JSONEq(t, string(exp.Data), string(act.Data)) + } else { + dataOk = reflect.DeepEqual(act.Data, exp.Data) + } + cTimeOk := reflect.DeepEqual(act.TimeCreated, exp.TimeCreated) pTimeOk := reflect.DeepEqual(act.TimePulled, exp.TimePulled) tTimeOk := reflect.DeepEqual(act.TimeTransformed, exp.TimeTransformed)