From 1de0f68de7fffb4ccba5151b6bfa0cd02644cc40 Mon Sep 17 00:00:00 2001 From: Nick Zelei <2420177+nickzelei@users.noreply.github.com> Date: Tue, 21 Jan 2025 13:25:23 -0800 Subject: [PATCH] NEOS-1712: Adds support for JS in Anon Api (#3163) --- .mockery.yml | 2 +- .../anonymization-service_integration_test.go | 60 ++- .../benthos-builder/builders/processors.go | 75 +--- .../builders/processors_test.go | 29 -- internal/benthos_slogger/logger.go | 93 +++++ .../javascript/functions/benthos/functions.go | 170 ++++++++ .../functions/benthos/functions_test.go | 12 + internal/javascript/functions/functions.go | 139 +++++++ .../javascript/functions/neosync/functions.go | 138 +++++++ .../functions/neosync}/functions_test.go | 8 +- internal/javascript/javascript.go | 52 +++ internal/javascript/userland/userland.go | 124 ++++++ internal/javascript/userland/userland_test.go | 369 +++++++++++++++++ internal/javascript/vm/console_logger.go | 34 ++ internal/javascript/vm/vm.go | 145 +++++++ internal/javascript/vm/vm_test.go | 91 +++++ internal/json-anonymizer/json-anonymizer.go | 40 +- internal/json-anonymizer/neosync-operator.go | 8 +- .../json-anonymizer/neosync-operator_test.go | 15 +- tools/go.mod | 2 - tools/go.sum | 72 ---- .../benthos/default_transform/processor.go | 10 +- .../default_transform/processor_test.go | 4 +- worker/pkg/benthos/environment/environment.go | 6 + .../benthos/javascript/benthos_value_api.go | 50 +++ worker/pkg/benthos/javascript/casts.go | 44 -- worker/pkg/benthos/javascript/functions.go | 377 ------------------ worker/pkg/benthos/javascript/ifs/http.go | 83 ---- worker/pkg/benthos/javascript/ifs/os.go | 95 ----- worker/pkg/benthos/javascript/ifs/os_test.go | 36 -- worker/pkg/benthos/javascript/logger.go | 23 -- worker/pkg/benthos/javascript/processor.go | 260 ++++-------- .../pkg/benthos/javascript/processor_test.go | 85 +--- worker/pkg/benthos/javascript/vm.go | 149 ------- .../transformer_executor/anon_value_api.go | 74 ++++ .../anon_value_api_test.go | 17 + .../executor.go} | 242 +++++++---- .../executor_test.go} | 37 +- .../mock_UserDefinedTransformerResolver.go | 96 +++++ 39 files changed, 1983 insertions(+), 1383 deletions(-) create mode 100644 internal/benthos_slogger/logger.go create mode 100644 internal/javascript/functions/benthos/functions.go create mode 100644 internal/javascript/functions/benthos/functions_test.go create mode 100644 internal/javascript/functions/functions.go create mode 100644 internal/javascript/functions/neosync/functions.go rename {worker/pkg/benthos/javascript => internal/javascript/functions/neosync}/functions_test.go (91%) create mode 100644 internal/javascript/javascript.go create mode 100644 internal/javascript/userland/userland.go create mode 100644 internal/javascript/userland/userland_test.go create mode 100644 internal/javascript/vm/console_logger.go create mode 100644 internal/javascript/vm/vm.go create mode 100644 internal/javascript/vm/vm_test.go create mode 100644 worker/pkg/benthos/javascript/benthos_value_api.go delete mode 100644 worker/pkg/benthos/javascript/casts.go delete mode 100644 worker/pkg/benthos/javascript/functions.go delete mode 100644 worker/pkg/benthos/javascript/ifs/http.go delete mode 100644 worker/pkg/benthos/javascript/ifs/os.go delete mode 100644 worker/pkg/benthos/javascript/ifs/os_test.go delete mode 100644 worker/pkg/benthos/javascript/logger.go delete mode 100644 worker/pkg/benthos/javascript/vm.go create mode 100644 worker/pkg/benthos/transformer_executor/anon_value_api.go create mode 100644 worker/pkg/benthos/transformer_executor/anon_value_api_test.go rename worker/pkg/benthos/{transformers/transformer_initializer.go => transformer_executor/executor.go} (67%) rename worker/pkg/benthos/{transformers/transformer_initializer_test.go => transformer_executor/executor_test.go} (98%) create mode 100644 worker/pkg/benthos/transformer_executor/mock_UserDefinedTransformerResolver.go diff --git a/.mockery.yml b/.mockery.yml index c2e310a769..2bdee9a0e6 100644 --- a/.mockery.yml +++ b/.mockery.yml @@ -123,6 +123,6 @@ packages: github.com/nucleuscloud/neosync/internal/ee/transformers/functions: interfaces: NeosyncOperatorApi: - github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers: + github.com/nucleuscloud/neosync/worker/pkg/benthos/transformer_executor: interfaces: UserDefinedTransformerResolver: diff --git a/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go b/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go index ffa5d297b1..f8ac451150 100644 --- a/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go +++ b/backend/services/mgmt/v1alpha1/integration_tests/anonymization-service_integration_test.go @@ -198,22 +198,74 @@ func (s *IntegrationTestSuite) Test_AnonymizeService_AnonymizeSingle() { }, }), ) - requireNoErrResp(s.T(), resp, err) - require.NotEmpty(s.T(), resp.Msg.OutputData) + requireNoErrResp(t, resp, err) + require.NotEmpty(t, resp.Msg.OutputData) var inputObject map[string]any err = json.Unmarshal([]byte(jsonStr), &inputObject) - require.NoError(s.T(), err) + require.NoError(t, err) output := resp.Msg.OutputData var result map[string]any err = json.Unmarshal([]byte(output), &result) - require.NoError(s.T(), err) + require.NoError(t, err) for _, sport := range result["sports"].([]any) { require.Equal(t, "A", sport) } }) + t.Run("javascript-transformers", func(t *testing.T) { + jsonStr := `{ + "sports": ["basketball", "golf", "swimming"], + "name": "bill" +}` + + accountId := s.createPersonalAccount(s.ctx, s.OSSUnauthenticatedLicensedClients.Users()) + + resp, err := s.OSSUnauthenticatedLicensedClients.Anonymize().AnonymizeSingle( + s.ctx, + connect.NewRequest(&mgmtv1alpha1.AnonymizeSingleRequest{ + AccountId: accountId, + InputData: jsonStr, + TransformerMappings: []*mgmtv1alpha1.TransformerMapping{ + { + Expression: ".sports", + Transformer: &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformJavascriptConfig{ + TransformJavascriptConfig: &mgmtv1alpha1.TransformJavascript{ + Code: "return value.map(v => v + ' updated');", + }, + }, + }, + }, + { + Expression: ".name", + Transformer: &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateJavascriptConfig{ + GenerateJavascriptConfig: &mgmtv1alpha1.GenerateJavascript{ + Code: "return 'jim';", + }, + }, + }, + }, + }, + }), + ) + requireNoErrResp(t, resp, err) + require.NotEmpty(t, resp.Msg.OutputData) + + var inputObject map[string]any + err = json.Unmarshal([]byte(jsonStr), &inputObject) + require.NoError(t, err) + + output := resp.Msg.OutputData + var result map[string]any + err = json.Unmarshal([]byte(output), &result) + require.NoError(t, err) + require.Equal(t, "jim", result["name"]) + require.Equal(t, []any{"basketball updated", "golf updated", "swimming updated"}, result["sports"]) + }) + t.Run("ok", func(t *testing.T) { jsonStr := `{ diff --git a/internal/benthos/benthos-builder/builders/processors.go b/internal/benthos/benthos-builder/builders/processors.go index deb77ef9d1..b2bd8c54d2 100644 --- a/internal/benthos/benthos-builder/builders/processors.go +++ b/internal/benthos/benthos-builder/builders/processors.go @@ -8,7 +8,6 @@ import ( "fmt" "slices" "strings" - "unicode" "connectrpc.com/connect" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" @@ -16,6 +15,7 @@ import ( sqlmanager_shared "github.com/nucleuscloud/neosync/backend/pkg/sqlmanager/shared" tabledependency "github.com/nucleuscloud/neosync/backend/pkg/table-dependency" bb_internal "github.com/nucleuscloud/neosync/internal/benthos/benthos-builder/internal" + javascript_userland "github.com/nucleuscloud/neosync/internal/javascript/userland" neosync_benthos "github.com/nucleuscloud/neosync/worker/pkg/benthos" "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" transformer_utils "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers/utils" @@ -242,7 +242,7 @@ func extractJsFunctionsAndOutputs(ctx context.Context, transformerclient mgmtv1a } if len(jsFunctions) > 0 { - return constructBenthosJsProcessor(jsFunctions, benthosOutputs), nil + return javascript_userland.GetFunction(jsFunctions, benthosOutputs), nil } else { return "", nil } @@ -385,83 +385,25 @@ func buildRedisGetBranchConfig( func constructJsFunction(jsCode, col string, source mgmtv1alpha1.TransformerSource) string { switch source { case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_JAVASCRIPT: - return fmt.Sprintf(` -function fn_%s(value, input){ - %s -}; -`, sanitizeJsFunctionName(col), jsCode) + return javascript_userland.GetTransformJavascriptFunction(jsCode, col, true) case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_JAVASCRIPT: - return fmt.Sprintf(` -function fn_%s(){ - %s -}; -`, sanitizeJsFunctionName(col), jsCode) + return javascript_userland.GetGenerateJavascriptFunction(jsCode, col) default: return "" } } -func sanitizeJsFunctionName(input string) string { - var result strings.Builder - - for i, r := range input { - if unicode.IsLetter(r) || r == '_' || r == '$' || (unicode.IsDigit(r) && i > 0) { - result.WriteRune(r) - } else if unicode.IsDigit(r) && i == 0 { - result.WriteRune('_') - result.WriteRune(r) - } else { - result.WriteRune('_') - } - } - - return result.String() -} - -func constructBenthosJsProcessor(jsFunctions, benthosOutputs []string) string { - jsFunctionStrings := strings.Join(jsFunctions, "\n") - - benthosOutputString := strings.Join(benthosOutputs, "\n") - - jsCode := fmt.Sprintf(` -(() => { -%s -const input = benthos.v0_msg_as_structured(); -const updatedValues = {} -%s -neosync.patchStructuredMessage(updatedValues) -})();`, jsFunctionStrings, benthosOutputString) - return jsCode -} - func constructBenthosJavascriptObject(col string, source mgmtv1alpha1.TransformerSource) string { switch source { case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_TRANSFORM_JAVASCRIPT: - return fmt.Sprintf( - `updatedValues[%q] = fn_%s(%s, input)`, - col, - sanitizeJsFunctionName(col), - convertJsObjPathToOptionalChain(fmt.Sprintf("input.%s", col)), - ) + return javascript_userland.BuildOutputSetter(col, true, true) case mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_JAVASCRIPT: - return fmt.Sprintf( - `updatedValues[%q] = fn_%s()`, - col, - sanitizeJsFunctionName(col), - ) + return javascript_userland.BuildOutputSetter(col, false, false) default: return "" } } -func convertJsObjPathToOptionalChain(inputPath string) string { - parts := strings.Split(inputPath, ".") - for i := 1; i < len(parts); i++ { - parts[i] = fmt.Sprintf("['%s']", parts[i]) - } - return strings.Join(parts, "?.") -} - // takes in an user defined config with just an id field and return the right transformer config for that user defined function id func convertUserDefinedFunctionConfig( ctx context.Context, @@ -479,11 +421,6 @@ func convertUserDefinedFunctionConfig( }, nil } -/* -function transformers -root.{destination_col} = transformerfunction(args) -*/ - func computeMutationFunction(col *mgmtv1alpha1.JobMapping, colInfo *sqlmanager_shared.DatabaseSchemaRow, splitColumnPath bool) (string, error) { var maxLen int64 = 10000 if colInfo != nil && colInfo.CharacterMaximumLength > 0 { diff --git a/internal/benthos/benthos-builder/builders/processors_test.go b/internal/benthos/benthos-builder/builders/processors_test.go index e01ac04f75..a113a7a019 100644 --- a/internal/benthos/benthos-builder/builders/processors_test.go +++ b/internal/benthos/benthos-builder/builders/processors_test.go @@ -14,35 +14,6 @@ import ( "github.com/stretchr/testify/require" ) -func Test_sanitizeFunctionName(t *testing.T) { - tests := []struct { - input string - expected string - }{ - {"123my Function!", "_123my_Function_"}, - {"validName", "validName"}, - {"name_with_underscores", "name_with_underscores"}, - {"$dollarSign", "$dollarSign"}, - {"invalid-char$", "invalid_char$"}, - {"spaces in name", "spaces_in_name"}, - {"!@#$%^&*()_+=", "___$_________"}, - {"_leadingUnderscore", "_leadingUnderscore"}, - {"$startingDollarSign", "$startingDollarSign"}, - {"endingWithNumber1", "endingWithNumber1"}, - {"functionName123", "functionName123"}, - {"中文字符", "中文字符"}, - } - - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - actual := sanitizeJsFunctionName(tt.input) - if actual != tt.expected { - t.Errorf("sanitizeJsFunctionName(%q) = %q; expected %q", tt.input, actual, tt.expected) - } - }) - } -} - func Test_buildProcessorConfigsJavascript(t *testing.T) { mockTransformerClient := mgmtv1alpha1connect.NewMockTransformersServiceClient(t) diff --git a/internal/benthos_slogger/logger.go b/internal/benthos_slogger/logger.go new file mode 100644 index 0000000000..73d0828c55 --- /dev/null +++ b/internal/benthos_slogger/logger.go @@ -0,0 +1,93 @@ +package benthos_slogger + +import ( + "context" + "log/slog" + + "github.com/warpstreamlabs/bento/public/service" +) + +var _ slog.Handler = (*benthosLogHandler)(nil) + +type benthosLogHandler struct { + logger *service.Logger + attrs []slog.Attr + groups []string +} + +func (h *benthosLogHandler) Enabled(ctx context.Context, level slog.Level) bool { + // We defer to the benthos logger and let it handle what leveling it wants to output + return true +} + +func (h *benthosLogHandler) Handle(ctx context.Context, r slog.Record) error { //nolint:gocritic // Needs to conform to the slog.Handler interface + // Combine pre-defined attrs with record attrs + allAttrs := make([]slog.Attr, 0, len(h.attrs)+r.NumAttrs()) + allAttrs = append(allAttrs, h.attrs...) + + r.Attrs(func(attr slog.Attr) bool { + if !attr.Equal(slog.Attr{}) { + // Handle groups + if len(h.groups) > 0 { + last := h.groups[len(h.groups)-1] + if last != "" { + attr.Key = last + "." + attr.Key + } + } + allAttrs = append(allAttrs, attr) + } + return true + }) + + // Convert to key-value pairs for temporal logger + keyvals := make([]any, 0, len(allAttrs)*2) + for _, attr := range allAttrs { + keyvals = append(keyvals, attr.Key, attr.Value.Any()) + } + + switch r.Level { + case slog.LevelDebug: + h.logger.With(keyvals...).Debug(r.Message) + case slog.LevelInfo: + h.logger.With(keyvals...).Info(r.Message) + case slog.LevelWarn: + h.logger.With(keyvals...).Warn(r.Message) + case slog.LevelError: + h.logger.With(keyvals...).Error(r.Message) + } + return nil +} + +func (h *benthosLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + newAttrs := []slog.Attr{} + newAttrs = append(newAttrs, h.attrs...) + newAttrs = append(newAttrs, attrs...) + return &benthosLogHandler{ + logger: h.logger, + attrs: newAttrs, + groups: h.groups, + } +} + +func (h *benthosLogHandler) WithGroup(name string) slog.Handler { + if name == "" { + return h + } + newGroups := []string{} + newGroups = append(newGroups, h.groups...) + newGroups = append(newGroups, name) + return &benthosLogHandler{ + logger: h.logger, + attrs: h.attrs, + groups: newGroups, + } +} + +func newBenthosLogHandler(logger *service.Logger) *benthosLogHandler { + return &benthosLogHandler{logger: logger} +} + +// Returns a benthos logger wrapped as a slog.Logger to ease plugging in to the rest of the system +func NewSlogger(logger *service.Logger) *slog.Logger { + return slog.New(newBenthosLogHandler(logger)) +} diff --git a/internal/javascript/functions/benthos/functions.go b/internal/javascript/functions/benthos/functions.go new file mode 100644 index 0000000000..064a676885 --- /dev/null +++ b/internal/javascript/functions/benthos/functions.go @@ -0,0 +1,170 @@ +package benthos_functions + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + + "github.com/dop251/goja" + javascript_functions "github.com/nucleuscloud/neosync/internal/javascript/functions" +) + +const ( + namespace = "benthos" +) + +func Get() []*javascript_functions.FunctionDefinition { + return []*javascript_functions.FunctionDefinition{ + getV0Fetch(namespace), + getV0MsgSetString(namespace), + getV0MsgAsString(namespace), + getV0MsgSetStructured(namespace), + getV0MsgAsStructured(namespace), + getV0MsgSetMeta(namespace), + getV0MsgGetMeta(namespace), + getV0MsgMetaExists(namespace), + } +} + +func getV0Fetch(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_fetch", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var ( + url string + httpHeaders map[string]any + method = "GET" + payload = "" + ) + if err := javascript_functions.ParseFunctionArguments(call, &url, &httpHeaders, &method, &payload); err != nil { + return nil, err + } + + var payloadReader io.Reader + if payload != "" { + payloadReader = strings.NewReader(payload) + } + + req, err := http.NewRequestWithContext(ctx, method, url, payloadReader) + if err != nil { + return nil, err + } + + // Parse HTTP headers + for k, v := range httpHeaders { + vStr, _ := v.(string) + req.Header.Add(k, vStr) + } + + // Do request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return map[string]any{ + "status": resp.StatusCode, + "body": string(respBody), + }, nil + } + }) +} + +func getV0MsgSetString(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_msg_set_string", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var value string + if err := javascript_functions.ParseFunctionArguments(call, &value); err != nil { + return nil, err + } + + r.ValueApi().SetBytes([]byte(value)) + return nil, nil + } + }) +} + +func getV0MsgAsString(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_msg_as_string", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + b, err := r.ValueApi().AsBytes() + if err != nil { + return nil, err + } + return string(b), nil + } + }) +} + +func getV0MsgSetStructured(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_msg_set_structured", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var value any + if err := javascript_functions.ParseFunctionArguments(call, &value); err != nil { + return nil, err + } + + r.ValueApi().SetStructured(value) + return nil, nil + } + }) +} + +func getV0MsgAsStructured(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_msg_as_structured", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + return r.ValueApi().AsStructured() + } + }) +} + +func getV0MsgSetMeta(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_msg_set_meta", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var key string + var value any + if err := javascript_functions.ParseFunctionArguments(call, &key, &value); err != nil { + return nil, err + } + r.ValueApi().MetaSetMut(key, value) + return nil, nil + } + }) +} + +func getV0MsgGetMeta(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_msg_get_meta", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var key string + if err := javascript_functions.ParseFunctionArguments(call, &key); err != nil { + return nil, err + } + result, ok := r.ValueApi().MetaGet(key) + if !ok { + return nil, fmt.Errorf("key %s not found", key) + } + return result, nil + } + }) +} + +func getV0MsgMetaExists(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "v0_msg_exists_meta", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var key string + if err := javascript_functions.ParseFunctionArguments(call, &key); err != nil { + return nil, err + } + _, ok := r.ValueApi().MetaGet(key) + return ok, nil + } + }) +} diff --git a/internal/javascript/functions/benthos/functions_test.go b/internal/javascript/functions/benthos/functions_test.go new file mode 100644 index 0000000000..2a1aaefee2 --- /dev/null +++ b/internal/javascript/functions/benthos/functions_test.go @@ -0,0 +1,12 @@ +package benthos_functions + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGet(t *testing.T) { + functions := Get() + require.NotEmpty(t, functions) +} diff --git a/internal/javascript/functions/functions.go b/internal/javascript/functions/functions.go new file mode 100644 index 0000000000..3b71779d74 --- /dev/null +++ b/internal/javascript/functions/functions.go @@ -0,0 +1,139 @@ +package javascript_functions + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/dop251/goja" +) + +type Runner interface { + ValueApi() ValueApi +} + +type ValueApi interface { + SetBytes(value []byte) + AsBytes() ([]byte, error) + SetStructured(value any) + AsStructured() (any, error) + + MetaGet(name string) (any, bool) + MetaSetMut(name string, value any) +} + +type Ctor func(r Runner) Function + +type FunctionDefinition struct { + namespace string + name string + // ctor means "constructor" + ctor Ctor +} + +func NewFunctionDefinition(namespace, name string, ctor Ctor) *FunctionDefinition { + return &FunctionDefinition{ + namespace: namespace, + name: name, + ctor: ctor, + } +} + +func (f *FunctionDefinition) Namespace() string { + return f.namespace +} + +func (f *FunctionDefinition) Name() string { + return f.name +} + +func (f *FunctionDefinition) Ctor() Ctor { + return f.ctor +} + +type Function func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) + +// Takes in a goja function call and returns the parsed arguments into the provided pointers. +// Returns an error if the arguments are not of the expected type. +func ParseFunctionArguments(call goja.FunctionCall, ptrs ...any) error { + if len(ptrs) < len(call.Arguments) { + return fmt.Errorf("have %d arguments, but only %d pointers to parse into", len(call.Arguments), len(ptrs)) + } + + for i := 0; i < len(call.Arguments); i++ { + arg, ptr := call.Argument(i), ptrs[i] + + if goja.IsUndefined(arg) { + return fmt.Errorf("argument at position %d is undefined", i) + } + + var err error + switch p := ptr.(type) { + case *string: + *p = arg.String() + case *int: + *p = int(arg.ToInteger()) + case *int64: + *p = arg.ToInteger() + case *float64: + *p = arg.ToFloat() + case *map[string]any: + *p, err = getMapFromValue(arg) + case *bool: + *p = arg.ToBoolean() + case *[]any: + *p, err = getSliceFromValue(arg) + case *[]map[string]any: + *p, err = getMapSliceFromValue(arg) + case *goja.Value: + *p = arg + case *any: + *p = arg.Export() + default: + return fmt.Errorf("encountered unhandled type %T while trying to parse %v into %v", arg.ExportType().String(), arg, p) + } + if err != nil { + return fmt.Errorf("could not parse %v (%s) into %v (%T): %v", arg, arg.ExportType().String(), ptr, ptr, err) + } + } + + return nil +} + +func getMapFromValue(val goja.Value) (map[string]any, error) { + outVal := val.Export() + v, ok := outVal.(map[string]any) + if !ok { + return nil, errors.New("value is not of type map") + } + return v, nil +} + +func getSliceFromValue(val goja.Value) ([]any, error) { + outVal := val.Export() + v, ok := outVal.([]any) + if !ok { + return nil, errors.New("value is not of type slice") + } + return v, nil +} + +func getMapSliceFromValue(val goja.Value) ([]map[string]any, error) { + outVal := val.Export() + if v, ok := outVal.([]map[string]any); ok { + return v, nil + } + vSlice, ok := outVal.([]any) + if !ok { + return nil, errors.New("value is not of type map slice") + } + v := make([]map[string]any, len(vSlice)) + for i, e := range vSlice { + v[i], ok = e.(map[string]any) + if !ok { + return nil, errors.New("value is not of type map slice") + } + } + return v, nil +} diff --git a/internal/javascript/functions/neosync/functions.go b/internal/javascript/functions/neosync/functions.go new file mode 100644 index 0000000000..6e41f218c6 --- /dev/null +++ b/internal/javascript/functions/neosync/functions.go @@ -0,0 +1,138 @@ +package neosync_functions + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/dop251/goja" + javascript_functions "github.com/nucleuscloud/neosync/internal/javascript/functions" + "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" +) + +const ( + namespace = "neosync" +) + +func Get() ([]*javascript_functions.FunctionDefinition, error) { + generatorFns, err := getNeosyncGenerators() + if err != nil { + return nil, err + } + transformerFns, err := getNeosyncTransformers() + if err != nil { + return nil, err + } + patchStructuredMessage := getPatchStructuredMessage(namespace) + + output := make([]*javascript_functions.FunctionDefinition, 0, len(generatorFns)+len(transformerFns)+1) + output = append(output, generatorFns...) + output = append(output, transformerFns...) + output = append(output, patchStructuredMessage) + return output, nil +} + +func getPatchStructuredMessage(namespace string) *javascript_functions.FunctionDefinition { + return javascript_functions.NewFunctionDefinition(namespace, "patchStructuredMessage", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var updates map[string]any + if err := javascript_functions.ParseFunctionArguments(call, &updates); err != nil { + return nil, err + } + + originalData, err := r.ValueApi().AsStructured() + if err != nil { + return nil, fmt.Errorf("failed to get structured data: %w", err) + } + + originalMap, ok := originalData.(map[string]any) + if !ok { + return nil, fmt.Errorf("structured data is not a map") + } + + for key, value := range updates { + setNestedProperty(originalMap, key, value) + } + + r.ValueApi().SetStructured(originalMap) + + return nil, nil + } + }) +} + +func setNestedProperty(obj map[string]any, path string, value any) { + parts := strings.Split(path, ".") + current := obj + + for i, part := range parts { + if i == len(parts)-1 { + current[part] = value + } else { + if _, ok := current[part]; !ok { + current[part] = make(map[string]any) + } + current = current[part].(map[string]any) + } + } +} + +func getNeosyncGenerators() ([]*javascript_functions.FunctionDefinition, error) { + generators := transformers.GetNeosyncGenerators() + fns := make([]*javascript_functions.FunctionDefinition, 0, len(generators)) + for _, f := range generators { + templateData, err := f.GetJsTemplateData() + if err != nil { + return nil, err + } + + fn := javascript_functions.NewFunctionDefinition(namespace, templateData.Name, func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var ( + opts map[string]any + ) + if err := javascript_functions.ParseFunctionArguments(call, &opts); err != nil { + return nil, err + } + goOpts, err := f.ParseOptions(opts) + if err != nil { + return nil, err + } + return f.Generate(goOpts) + } + }) + fns = append(fns, fn) + } + return fns, nil +} + +func getNeosyncTransformers() ([]*javascript_functions.FunctionDefinition, error) { + neosyncTransformers := transformers.GetNeosyncTransformers() + fns := make([]*javascript_functions.FunctionDefinition, 0, len(neosyncTransformers)) + for _, f := range neosyncTransformers { + templateData, err := f.GetJsTemplateData() + if err != nil { + return nil, err + } + + fn := javascript_functions.NewFunctionDefinition(namespace, templateData.Name, func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + var ( + value any + opts map[string]any + ) + if err := javascript_functions.ParseFunctionArguments(call, &value, &opts); err != nil { + return nil, err + } + goOpts, err := f.ParseOptions(opts) + if err != nil { + return nil, err + } + return f.Transform(value, goOpts) + } + }) + fns = append(fns, fn) + } + return fns, nil +} diff --git a/worker/pkg/benthos/javascript/functions_test.go b/internal/javascript/functions/neosync/functions_test.go similarity index 91% rename from worker/pkg/benthos/javascript/functions_test.go rename to internal/javascript/functions/neosync/functions_test.go index 2db19e6556..44390cd17f 100644 --- a/worker/pkg/benthos/javascript/functions_test.go +++ b/internal/javascript/functions/neosync/functions_test.go @@ -1,4 +1,4 @@ -package javascript +package neosync_functions import ( "testing" @@ -6,6 +6,12 @@ import ( "github.com/stretchr/testify/require" ) +func TestGet(t *testing.T) { + functions, err := Get() + require.NoError(t, err) + require.NotEmpty(t, functions) +} + func Test_setNestedProperty(t *testing.T) { t.Run("Set simple property", func(t *testing.T) { obj := make(map[string]any) diff --git a/internal/javascript/javascript.go b/internal/javascript/javascript.go new file mode 100644 index 0000000000..7caf6d93b8 --- /dev/null +++ b/internal/javascript/javascript.go @@ -0,0 +1,52 @@ +package javascript + +import ( + "log/slog" + + goja_require "github.com/dop251/goja_nodejs/require" + javascript_functions "github.com/nucleuscloud/neosync/internal/javascript/functions" + benthos_functions "github.com/nucleuscloud/neosync/internal/javascript/functions/benthos" + neosync_functions "github.com/nucleuscloud/neosync/internal/javascript/functions/neosync" + javascript_vm "github.com/nucleuscloud/neosync/internal/javascript/vm" +) + +// Comes full featured, but expects a value api that the benthos/neosync functions can manipulate +func NewDefaultValueRunner( + valueApi javascript_functions.ValueApi, + logger *slog.Logger, +) (*javascript_vm.Runner, error) { + functions, err := getDefaultFunctions() + if err != nil { + return nil, err + } + return javascript_vm.NewRunner( + javascript_vm.WithValueApi(valueApi), + javascript_vm.WithLogger(logger), + javascript_vm.WithConsole(), + javascript_vm.WithJsRegistry(goja_require.NewRegistry()), + javascript_vm.WithFunctions(functions...), + ) +} + +// Comes full featured but does not register any custom functions +func NewDefaultRunner( + logger *slog.Logger, +) (*javascript_vm.Runner, error) { + return javascript_vm.NewRunner( + javascript_vm.WithLogger(logger), + javascript_vm.WithConsole(), + javascript_vm.WithJsRegistry(goja_require.NewRegistry()), + ) +} + +func getDefaultFunctions() ([]*javascript_functions.FunctionDefinition, error) { + benthosFns := benthos_functions.Get() + neosyncFns, err := neosync_functions.Get() + if err != nil { + return nil, err + } + output := make([]*javascript_functions.FunctionDefinition, 0, len(benthosFns)+len(neosyncFns)) + output = append(output, benthosFns...) + output = append(output, neosyncFns...) + return output, nil +} diff --git a/internal/javascript/userland/userland.go b/internal/javascript/userland/userland.go new file mode 100644 index 0000000000..537fab446a --- /dev/null +++ b/internal/javascript/userland/userland.go @@ -0,0 +1,124 @@ +package javascript_userland + +import ( + "fmt" + "strings" + "unicode" + + "github.com/google/uuid" +) + +// GetGenerateJavascriptFunction returns a Javascript function that takes no inputs and generates a value +// fnNameSuffix is the suffix of the function name +func GetGenerateJavascriptFunction(jsCode, fnNameSuffix string) string { + return fmt.Sprintf(` +function fn_%s(){ + %s +}; +`, sanitizeFunctionName(fnNameSuffix), jsCode) +} + +// GetTransformJavascriptFunction returns a Javascript function that takes a value and input and returns a transformed value +// fnNameSuffix is the suffix of the function name +// includeRecord is true if the function should take in the input record +func GetTransformJavascriptFunction(jsCode, fnNameSuffix string, includeRecord bool) string { + if includeRecord { + return fmt.Sprintf(` +function fn_%s(value, input){ + %s +}; +`, sanitizeFunctionName(fnNameSuffix), jsCode) + } + + return fmt.Sprintf(` +function fn_%s(value){ + %s +}; +`, sanitizeFunctionName(fnNameSuffix), jsCode) +} + +func sanitizeFunctionName(input string) string { + var result strings.Builder + + for i, r := range input { + if unicode.IsLetter(r) || r == '_' || r == '$' || (unicode.IsDigit(r) && i > 0) { + result.WriteRune(r) + } else if unicode.IsDigit(r) && i == 0 { + result.WriteRune('_') + result.WriteRune(r) + } else { + result.WriteRune('_') + } + } + + return result.String() +} + +// Takes a userland function and returns a single function that can be invoked by the JS VM +// Returns the property key that the output will be set to +func GetSingleGenerateFunction(userCode string) (code, propertyPath string) { + propertyPath = uuid.NewString() + fn := GetGenerateJavascriptFunction(userCode, propertyPath) + outputSetter := BuildOutputSetter(propertyPath, false, false) + return GetFunction([]string{fn}, []string{outputSetter}), propertyPath +} + +// Takes a userland function and returns a single function that can be invoked by the JS VM +// Returns the property key that the output will be set to +func GetSingleTransformFunction(userCode string) (code, propertyPath string) { + propertyPath = uuid.NewString() + fn := GetTransformJavascriptFunction(userCode, propertyPath, false) + outputSetter := BuildOutputSetter(propertyPath, true, false) + return GetFunction([]string{fn}, []string{outputSetter}), propertyPath +} + +// Takes all of the built userland functions and output setters and stuffs them into a single function that can be invoked by the JS VM +// Calling the resulting program expects benthos.v0_msg_as_structured() and neosync.patchStructuredMessage() to be defined in the JS VM +func GetFunction(jsFuncs, outputSetters []string) string { + jsFunctionStrings := strings.Join(jsFuncs, "\n") + + benthosOutputString := strings.Join(outputSetters, "\n") + + jsCode := fmt.Sprintf(` +(() => { +%s +const input = benthos.v0_msg_as_structured(); +const updatedValues = {} +%s +neosync.patchStructuredMessage(updatedValues) +})();`, jsFunctionStrings, benthosOutputString) + return jsCode +} + +// BuildOutputSetter builds a string that sets the output of the function to the property path on the "updatedValues" object +// includeInput is true if the propertyPath's value should be passed to the function +// includeInputRecord is true if the entire "input" object should be passed to the function as the second argument +func BuildOutputSetter(propertyPath string, includeInput, includeInputRecord bool) string { + if includeInput { + var strTemplate string + if includeInputRecord { + strTemplate = `updatedValues[%q] = fn_%s(%s, input)` + } else { + strTemplate = `updatedValues[%q] = fn_%s(%s)` + } + return fmt.Sprintf( + strTemplate, + propertyPath, + sanitizeFunctionName(propertyPath), + convertJsObjPathToOptionalChain(fmt.Sprintf("input.%s", propertyPath)), + ) + } + return fmt.Sprintf( + `updatedValues[%q] = fn_%s()`, + propertyPath, + sanitizeFunctionName(propertyPath), + ) +} + +func convertJsObjPathToOptionalChain(inputPath string) string { + parts := strings.Split(inputPath, ".") + for i := 1; i < len(parts); i++ { + parts[i] = fmt.Sprintf("['%s']", parts[i]) + } + return strings.Join(parts, "?.") +} diff --git a/internal/javascript/userland/userland_test.go b/internal/javascript/userland/userland_test.go new file mode 100644 index 0000000000..2332fd72c5 --- /dev/null +++ b/internal/javascript/userland/userland_test.go @@ -0,0 +1,369 @@ +package javascript_userland + +import ( + "fmt" + "testing" + + "github.com/dop251/goja" + "github.com/stretchr/testify/require" +) + +func Test_sanitizeFunctionName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"123my Function!", "_123my_Function_"}, + {"validName", "validName"}, + {"name_with_underscores", "name_with_underscores"}, + {"$dollarSign", "$dollarSign"}, + {"invalid-char$", "invalid_char$"}, + {"spaces in name", "spaces_in_name"}, + {"!@#$%^&*()_+=", "___$_________"}, + {"_leadingUnderscore", "_leadingUnderscore"}, + {"$startingDollarSign", "$startingDollarSign"}, + {"endingWithNumber1", "endingWithNumber1"}, + {"functionName123", "functionName123"}, + {"中文字符", "中文字符"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + actual := sanitizeFunctionName(tt.input) + if actual != tt.expected { + t.Errorf("sanitizeJsFunctionName(%q) = %q; expected %q", tt.input, actual, tt.expected) + } + }) + } +} + +func Test_GetSingleGenerateFunction(t *testing.T) { + t.Parallel() + t.Run("string", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleGenerateFunction("return 'hello world';") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, code) + + runTestProgram(t, wrappedCode, propertyPath, "hello world") + }) + t.Run("number", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleGenerateFunction("return 123;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, code) + + runTestProgram(t, wrappedCode, propertyPath, int64(123)) + }) + + t.Run("boolean", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleGenerateFunction("return true;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, code) + + runTestProgram(t, wrappedCode, propertyPath, true) + }) + + t.Run("object", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleGenerateFunction("return {a: 1, b: 2};") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, code) + + runTestProgram(t, wrappedCode, propertyPath, map[string]any{"a": int64(1), "b": int64(2)}) + }) + + t.Run("array", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleGenerateFunction("return [1, 2, 3];") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, code) + + runTestProgram(t, wrappedCode, propertyPath, []any{int64(1), int64(2), int64(3)}) + }) + + t.Run("null", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleGenerateFunction("return null;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, code) + + runTestProgram(t, wrappedCode, propertyPath, nil) + }) + t.Run("undefined", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleGenerateFunction("return undefined;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, code) + + runTestProgram(t, wrappedCode, propertyPath, nil) + }) +} + +func Test_GetSingleTransformFunction(t *testing.T) { + t.Parallel() + t.Run("string", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleTransformFunction("return 'hello ' + value;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({%q: "world"}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, propertyPath, code) + + runTestProgram(t, wrappedCode, propertyPath, "hello world") + }) + + t.Run("number", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleTransformFunction("return value + 1;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({%q: 123}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, propertyPath, code) + + runTestProgram(t, wrappedCode, propertyPath, int64(124)) + }) + + t.Run("boolean", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleTransformFunction("return !value;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({%q: true}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, propertyPath, code) + + runTestProgram(t, wrappedCode, propertyPath, false) + }) + + t.Run("object", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleTransformFunction("return { ...value, c: 3 };") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({%q: {a: 1, b: 2}}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, propertyPath, code) + + runTestProgram(t, wrappedCode, propertyPath, map[string]any{"a": int64(1), "b": int64(2), "c": int64(3)}) + }) + + t.Run("array", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleTransformFunction("return [...value, 3];") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({%q: [1, 2]}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, propertyPath, code) + + runTestProgram(t, wrappedCode, propertyPath, []any{int64(1), int64(2), int64(3)}) + }) + + t.Run("null", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleTransformFunction("return value;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({%q: null}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, propertyPath, code) + + runTestProgram(t, wrappedCode, propertyPath, nil) + }) + t.Run("undefined", func(t *testing.T) { + t.Parallel() + code, propertyPath := GetSingleTransformFunction("return value;") + require.NotEmpty(t, code) + require.NotEmpty(t, propertyPath) + + wrappedCode := fmt.Sprintf(` + let programOutput = undefined; + const benthos = { + v0_msg_as_structured: () => ({%q: undefined}), + }; + const neosync = { + patchStructuredMessage: (val) => { + programOutput = val; + } + }; + %s + `, propertyPath, code) + + runTestProgram(t, wrappedCode, propertyPath, nil) + }) +} + +func Test_convertJsObjPathToOptionalChain(t *testing.T) { + require.Equal(t, "address", convertJsObjPathToOptionalChain("address")) + require.Equal(t, "address?.['city']", convertJsObjPathToOptionalChain("address.city")) + require.Equal(t, "address?.['city']?.['state']", convertJsObjPathToOptionalChain("address.city.state")) +} + +func runTestProgram(t testing.TB, code string, propertyPath string, expectedOutput any) { + t.Helper() + program, err := goja.Compile("test.js", code, true) + require.NoError(t, err) + rt := goja.New() + _, err = rt.RunProgram(program) + require.NoError(t, err) + programOutput := rt.Get("programOutput").Export() + require.NotNil(t, programOutput) + outputMap, ok := programOutput.(map[string]any) + require.True(t, ok) + require.Equal(t, expectedOutput, outputMap[propertyPath]) +} diff --git a/internal/javascript/vm/console_logger.go b/internal/javascript/vm/console_logger.go new file mode 100644 index 0000000000..f2c0cfe232 --- /dev/null +++ b/internal/javascript/vm/console_logger.go @@ -0,0 +1,34 @@ +package javascript_vm + +import ( + "fmt" + "log/slog" + + "github.com/dop251/goja_nodejs/console" +) + +var _ console.Printer = &consoleLogger{} + +// adds a standard prefix to the message to make it easier to identify logs that originate from the JS VM. +const stdPrefix = "[js]: " + +func newConsoleLogger(prefix string, logger *slog.Logger) *consoleLogger { + return &consoleLogger{prefix: prefix, logger: logger} +} + +type consoleLogger struct { + prefix string + logger *slog.Logger +} + +func (l *consoleLogger) Log(message string) { + l.logger.Info(fmt.Sprintf("%s%s", l.prefix, message)) +} + +func (l *consoleLogger) Warn(message string) { + l.logger.Warn(fmt.Sprintf("%s%s", l.prefix, message)) +} + +func (l *consoleLogger) Error(message string) { + l.logger.Error(fmt.Sprintf("%s%s", l.prefix, message)) +} diff --git a/internal/javascript/vm/vm.go b/internal/javascript/vm/vm.go new file mode 100644 index 0000000000..ebb1f9c6f3 --- /dev/null +++ b/internal/javascript/vm/vm.go @@ -0,0 +1,145 @@ +package javascript_vm + +import ( + "context" + "fmt" + "log/slog" + "sync" + + "github.com/dop251/goja" + "github.com/dop251/goja_nodejs/console" + "github.com/dop251/goja_nodejs/require" + javascript_functions "github.com/nucleuscloud/neosync/internal/javascript/functions" +) + +type Runner struct { + vm *goja.Runtime + options Options + mu sync.Mutex +} + +func (r *Runner) ValueApi() javascript_functions.ValueApi { + return r.options.valueApi +} + +type Options struct { + logger *slog.Logger + requireRegistry *require.Registry + functions []*javascript_functions.FunctionDefinition + consoleEnabled bool + valueApi javascript_functions.ValueApi +} + +type Option func(*Options) + +// Sets the value api for the runner +// This allows custom functions to access an underlying data structure that can be manipulated by the runner +func WithValueApi(valueApi javascript_functions.ValueApi) Option { + return func(opts *Options) { + opts.valueApi = valueApi + } +} + +// Sets the logger for the runner +func WithLogger(logger *slog.Logger) Option { + return func(opts *Options) { + opts.logger = logger + } +} + +// Sets the require registry for the runner +// This allows custom modules to be registered with the runner +// If the logger is provided, the console module will be registered with the logger +func WithJsRegistry(registry *require.Registry) Option { + return func(opts *Options) { + opts.requireRegistry = registry + } +} + +// Sets the functions for the runner +// These functions will be registered with the runner +// Functions may interact with the value api +func WithFunctions(functions ...*javascript_functions.FunctionDefinition) Option { + return func(opts *Options) { + opts.functions = functions + } +} + +func WithConsole() Option { + return func(opts *Options) { + opts.consoleEnabled = true + } +} + +// Creates a new JS Runner +func NewRunner(opts ...Option) (*Runner, error) { + options := Options{logger: slog.Default()} + for _, opt := range opts { + opt(&options) + } + + vm := goja.New() + + // if the stars align, we'll register the custom console module with the logger + // must come before requireRegistry.Enable() + if options.requireRegistry != nil && options.consoleEnabled && options.logger != nil { + options.requireRegistry.RegisterNativeModule(console.ModuleName, console.RequireWithPrinter(newConsoleLogger(stdPrefix, options.logger))) + } + + if options.requireRegistry != nil { + options.requireRegistry.Enable(vm) + } + + // must come after requireRegistry.Enable() + if options.consoleEnabled { + console.Enable(vm) + } + + runner := &Runner{ + vm: vm, + options: options, + } + + for _, function := range options.functions { + if err := registerFunction(runner, function); err != nil { + return nil, err + } + } + + return runner, nil +} + +func (r *Runner) Run(ctx context.Context, program *goja.Program) (goja.Value, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.vm.RunProgram(program) +} + +// Registers a custom function with the vm +func registerFunction(runner *Runner, function *javascript_functions.FunctionDefinition) error { + var targetObj *goja.Object + if targetObjValue := runner.vm.GlobalObject().Get(function.Namespace()); targetObjValue != nil { + targetObj = targetObjValue.ToObject(runner.vm) + } + if targetObj == nil { + if err := runner.vm.GlobalObject().Set(function.Namespace(), map[string]any{}); err != nil { + return fmt.Errorf("failed to set global %s object: %w", function.Namespace(), err) + } + targetObj = runner.vm.GlobalObject().Get(function.Namespace()).ToObject(runner.vm) + } + + if err := targetObj.Set(function.Name(), func(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + l := runner.options.logger.With("function", function.Name()) + fn := function.Ctor()(runner) + result, err := fn(context.Background(), call, rt, l) + if err != nil { + // This _has_ to be a panic so that the error is properly thrown in the JS runtime + // Otherwise things like try/catch will not work properly + panic(rt.ToValue(err.Error())) + } + return rt.ToValue(result) + }); err != nil { + return fmt.Errorf("failed to set global %s function %v: %w", function.Namespace(), function.Name(), err) + } + return nil +} diff --git a/internal/javascript/vm/vm_test.go b/internal/javascript/vm/vm_test.go new file mode 100644 index 0000000000..c8bd91db1f --- /dev/null +++ b/internal/javascript/vm/vm_test.go @@ -0,0 +1,91 @@ +package javascript_vm + +import ( + "context" + "log/slog" + "sync" + "testing" + + "github.com/dop251/goja" + goja_require "github.com/dop251/goja_nodejs/require" + javascript_functions "github.com/nucleuscloud/neosync/internal/javascript/functions" + "github.com/nucleuscloud/neosync/internal/testutil" + + "github.com/stretchr/testify/require" +) + +func TestRunner(t *testing.T) { + t.Run("basic", func(t *testing.T) { + runner, err := NewRunner() + require.NoError(t, err) + + program := goja.MustCompile("test.js", "1+1", true) + result, err := runner.Run(context.Background(), program) + require.NoError(t, err) + require.Equal(t, int64(2), result.ToInteger()) + }) + + t.Run("with_console", func(t *testing.T) { + runner, err := NewRunner(WithConsole(), WithJsRegistry(goja_require.NewRegistry())) + require.NoError(t, err) + + program := goja.MustCompile("test.js", "console.log('hello world')", true) + _, err = runner.Run(context.Background(), program) + require.NoError(t, err) + }) + + t.Run("with_console_and_logger", func(t *testing.T) { + runner, err := NewRunner(WithConsole(), WithJsRegistry(goja_require.NewRegistry()), WithLogger(testutil.GetTestLogger(t))) + require.NoError(t, err) + + program := goja.MustCompile("test.js", `console.log('hello world');`, true) + _, err = runner.Run(context.Background(), program) + require.NoError(t, err) + }) + + t.Run("parallel_runs", func(t *testing.T) { + runner, err := NewRunner(WithConsole(), WithJsRegistry(goja_require.NewRegistry()), WithLogger(testutil.GetTestLogger(t))) + require.NoError(t, err) + + program := goja.MustCompile("test.js", `console.log('hello world');`, true) + wg := sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err = runner.Run(context.Background(), program) + require.NoError(t, err) + }() + } + wg.Wait() + }) + + t.Run("with_functions", func(t *testing.T) { + customFn := javascript_functions.NewFunctionDefinition("test", "test", func(r javascript_functions.Runner) javascript_functions.Function { + return func(ctx context.Context, call goja.FunctionCall, rt *goja.Runtime, l *slog.Logger) (any, error) { + return "hello world", nil + } + }) + + runner, err := NewRunner(WithFunctions(customFn)) + require.NoError(t, err) + + program := goja.MustCompile("test.js", `test.test();`, true) + result, err := runner.Run(context.Background(), program) + require.NoError(t, err) + require.Equal(t, "hello world", result.String()) + }) +} + +func BenchmarkRunner_Single(b *testing.B) { + runner, err := NewRunner(WithConsole(), WithJsRegistry(goja_require.NewRegistry()), WithLogger(testutil.GetTestLogger(b))) + require.NoError(b, err) + + program := goja.MustCompile("test.js", `console.log('hello world');`, true) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err = runner.Run(context.Background(), program) + require.NoError(b, err) + } +} diff --git a/internal/json-anonymizer/json-anonymizer.go b/internal/json-anonymizer/json-anonymizer.go index 15f6bbb6c3..c808ace9e4 100644 --- a/internal/json-anonymizer/json-anonymizer.go +++ b/internal/json-anonymizer/json-anonymizer.go @@ -12,7 +12,7 @@ import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect" presidioapi "github.com/nucleuscloud/neosync/internal/ee/presidio" - transformer "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" + transformer_executor "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformer_executor" ) type AnonymizeJsonError struct { @@ -22,7 +22,7 @@ type AnonymizeJsonError struct { type JsonAnonymizer struct { transformerMappings []*mgmtv1alpha1.TransformerMapping - transformerExecutors []*transformer.TransformerExecutor + transformerExecutors []*transformer_executor.TransformerExecutor defaultTransformers *mgmtv1alpha1.DefaultTransformersConfig defaultTransformerExecutor *DefaultExecutors compiledQuery *gojq.Code @@ -359,21 +359,21 @@ func initTransformerExecutors( anonymizeConfig *anonymizeConfig, transformerClient mgmtv1alpha1connect.TransformersServiceClient, logger *slog.Logger, -) ([]*transformer.TransformerExecutor, error) { - executors := []*transformer.TransformerExecutor{} - execOpts := []transformer.TransformerExecutorOption{ - transformer.WithLogger(logger), - transformer.WithUserDefinedTransformerResolver(newUdtResolver(transformerClient)), +) ([]*transformer_executor.TransformerExecutor, error) { + executors := []*transformer_executor.TransformerExecutor{} + execOpts := []transformer_executor.TransformerExecutorOption{ + transformer_executor.WithLogger(logger), + transformer_executor.WithUserDefinedTransformerResolver(newUdtResolver(transformerClient)), } if anonymizeConfig != nil && anonymizeConfig.analyze != nil && anonymizeConfig.anonymize != nil { execOpts = append( execOpts, - transformer.WithTransformPiiTextConfig(anonymizeConfig.analyze, anonymizeConfig.anonymize, newNeosyncOperatorApi(execOpts), anonymizeConfig.defaultLanguage), + transformer_executor.WithTransformPiiTextConfig(anonymizeConfig.analyze, anonymizeConfig.anonymize, newNeosyncOperatorApi(execOpts), anonymizeConfig.defaultLanguage), ) } for _, mapping := range transformerMappings { - executor, err := transformer.InitializeTransformerByConfigType(mapping.GetTransformer(), execOpts...) + executor, err := transformer_executor.InitializeTransformerByConfigType(mapping.GetTransformer(), execOpts...) if err != nil { return nil, fmt.Errorf("failed to initialize transformer for expression '%s': %v", mapping.GetExpression(), err) } @@ -384,9 +384,9 @@ func initTransformerExecutors( } type DefaultExecutors struct { - S *transformer.TransformerExecutor - N *transformer.TransformerExecutor - Boolean *transformer.TransformerExecutor + S *transformer_executor.TransformerExecutor + N *transformer_executor.TransformerExecutor + Boolean *transformer_executor.TransformerExecutor } func initDefaultTransformerExecutors( @@ -395,30 +395,30 @@ func initDefaultTransformerExecutors( transformerClient mgmtv1alpha1connect.TransformersServiceClient, logger *slog.Logger, ) (*DefaultExecutors, error) { - execOpts := []transformer.TransformerExecutorOption{ - transformer.WithLogger(logger), - transformer.WithUserDefinedTransformerResolver(newUdtResolver(transformerClient)), + execOpts := []transformer_executor.TransformerExecutorOption{ + transformer_executor.WithLogger(logger), + transformer_executor.WithUserDefinedTransformerResolver(newUdtResolver(transformerClient)), } if anonymizeConfig != nil && anonymizeConfig.analyze != nil && anonymizeConfig.anonymize != nil { - execOpts = append(execOpts, transformer.WithTransformPiiTextConfig(anonymizeConfig.analyze, anonymizeConfig.anonymize, newNeosyncOperatorApi(execOpts), anonymizeConfig.defaultLanguage)) + execOpts = append(execOpts, transformer_executor.WithTransformPiiTextConfig(anonymizeConfig.analyze, anonymizeConfig.anonymize, newNeosyncOperatorApi(execOpts), anonymizeConfig.defaultLanguage)) } - var stringExecutor, numberExecutor, booleanExecutor *transformer.TransformerExecutor + var stringExecutor, numberExecutor, booleanExecutor *transformer_executor.TransformerExecutor var err error if defaultTransformer.S != nil { - stringExecutor, err = transformer.InitializeTransformerByConfigType(defaultTransformer.S, execOpts...) + stringExecutor, err = transformer_executor.InitializeTransformerByConfigType(defaultTransformer.S, execOpts...) if err != nil { return nil, err } } if defaultTransformer.N != nil { - numberExecutor, err = transformer.InitializeTransformerByConfigType(defaultTransformer.N, execOpts...) + numberExecutor, err = transformer_executor.InitializeTransformerByConfigType(defaultTransformer.N, execOpts...) if err != nil { return nil, err } } if defaultTransformer.Boolean != nil { - booleanExecutor, err = transformer.InitializeTransformerByConfigType(defaultTransformer.Boolean, execOpts...) + booleanExecutor, err = transformer_executor.InitializeTransformerByConfigType(defaultTransformer.Boolean, execOpts...) if err != nil { return nil, err } diff --git a/internal/json-anonymizer/neosync-operator.go b/internal/json-anonymizer/neosync-operator.go index 4dbc40bd45..7fe21d9249 100644 --- a/internal/json-anonymizer/neosync-operator.go +++ b/internal/json-anonymizer/neosync-operator.go @@ -7,19 +7,19 @@ import ( "connectrpc.com/connect" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect" - transformer "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" + "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformer_executor" ) type neosyncOperatorApi struct { - opts []transformer.TransformerExecutorOption + opts []transformer_executor.TransformerExecutorOption } -func newNeosyncOperatorApi(executorOpts []transformer.TransformerExecutorOption) *neosyncOperatorApi { +func newNeosyncOperatorApi(executorOpts []transformer_executor.TransformerExecutorOption) *neosyncOperatorApi { return &neosyncOperatorApi{opts: executorOpts} } func (n *neosyncOperatorApi) Transform(ctx context.Context, config *mgmtv1alpha1.TransformerConfig, value string) (string, error) { - executor, err := transformer.InitializeTransformerByConfigType(config, n.opts...) + executor, err := transformer_executor.InitializeTransformerByConfigType(config, n.opts...) if err != nil { return "", err } diff --git a/internal/json-anonymizer/neosync-operator_test.go b/internal/json-anonymizer/neosync-operator_test.go index b9808f7c17..5c9b20ceaa 100644 --- a/internal/json-anonymizer/neosync-operator_test.go +++ b/internal/json-anonymizer/neosync-operator_test.go @@ -6,15 +6,16 @@ import ( mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" "github.com/nucleuscloud/neosync/internal/testutil" - "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" + "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformer_executor" + "github.com/stretchr/testify/require" ) func Test_NeosyncOperator(t *testing.T) { t.Run("Transform", func(t *testing.T) { t.Run("string", func(t *testing.T) { - operator := newNeosyncOperatorApi([]transformers.TransformerExecutorOption{ - transformers.WithLogger(testutil.GetTestLogger(t)), + operator := newNeosyncOperatorApi([]transformer_executor.TransformerExecutorOption{ + transformer_executor.WithLogger(testutil.GetTestLogger(t)), }) actual, err := operator.Transform(context.Background(), &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateFirstNameConfig{ @@ -26,8 +27,8 @@ func Test_NeosyncOperator(t *testing.T) { require.IsType(t, "", actual) }) t.Run("default_empty_string", func(t *testing.T) { - operator := newNeosyncOperatorApi([]transformers.TransformerExecutorOption{ - transformers.WithLogger(testutil.GetTestLogger(t)), + operator := newNeosyncOperatorApi([]transformer_executor.TransformerExecutorOption{ + transformer_executor.WithLogger(testutil.GetTestLogger(t)), }) actual, err := operator.Transform(context.Background(), &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig{ @@ -39,8 +40,8 @@ func Test_NeosyncOperator(t *testing.T) { require.IsType(t, "", actual) }) t.Run("default_number", func(t *testing.T) { - operator := newNeosyncOperatorApi([]transformers.TransformerExecutorOption{ - transformers.WithLogger(testutil.GetTestLogger(t)), + operator := newNeosyncOperatorApi([]transformer_executor.TransformerExecutorOption{ + transformer_executor.WithLogger(testutil.GetTestLogger(t)), }) actual, err := operator.Transform(context.Background(), &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig{ diff --git a/tools/go.mod b/tools/go.mod index 498430a6ba..2e4bc4ce8f 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -15,7 +15,6 @@ require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.3-20241127180247-a33202765966.1 // indirect github.com/Jeffail/gabs/v2 v2.7.0 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect - github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -25,7 +24,6 @@ require ( github.com/gorilla/css v1.0.1 // indirect github.com/matoous/go-nanoid/v2 v2.0.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect - github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/segmentio/ksuid v1.0.4 // indirect github.com/stretchr/objx v0.5.2 // indirect diff --git a/tools/go.sum b/tools/go.sum index 9ba237473c..159a3ca98b 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -4,51 +4,27 @@ connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= cuelang.org/go v0.7.1 h1:wSuUSIKR9M1yrph57l8EJATWVRWHaq/Zd0dFUL10PC8= cuelang.org/go v0.7.1/go.mod h1:ix+3dM/bSpdG9xg6qpCgnJnpeLtciZu+O/rDbywoMII= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg= github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= github.com/Jeffail/grok v1.1.0 h1:kiHmZ+0J5w/XUihRgU3DY9WIxKrNQCDjnfAb6bMLFaE= github.com/Jeffail/grok v1.1.0/go.mod h1:dm0hLksrDwOMa6To7ORXCuLbuNtASIZTfYheavLpsuE= github.com/Jeffail/shutdown v1.0.0 h1:afYjnY4pksqP/012m3NGJVccDI+WATdSzIMVHZKU8/Y= github.com/Jeffail/shutdown v1.0.0/go.mod h1:5dT4Y1oe60SJELCkmAB1pr9uQyHBhh6cwDLQTfmuO5U= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= -github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= -github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= -github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4= -github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -60,12 +36,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= -github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= @@ -98,7 +70,6 @@ github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/my github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= @@ -107,8 +78,6 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a h1:3Bm7EwfUQUvhNeKIkUct/gl9eod1TcXuj8stxvi/GoI= -github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/matoous/go-nanoid v1.5.0/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U= @@ -122,34 +91,12 @@ github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwX github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= -github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= -github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= -github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto= github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY= -github.com/neilotoole/slogt v1.1.0 h1:c7qE92sq+V0yvCuaxph+RQ2jOKL61c4hqS1Bv9W7FZE= -github.com/neilotoole/slogt v1.1.0/go.mod h1:RCrGXkPc/hYybNulqQrMHRtvlQ7F6NktNVLuLwk6V+w= github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM= github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= github.com/nucleuscloud/go-antlrv4-parser v0.0.0-20240830015744-041b46c70aa5 h1:MRc2C7I88nGhKbk1VRT3nnr6XtpClkHT35lhKbg6kD4= github.com/nucleuscloud/go-antlrv4-parser v0.0.0-20240830015744-041b46c70aa5/go.mod h1:MtI3ufP40Bql/YkhZFvDcR9h479JKb+hEUElwweQMzo= -github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= -github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg= @@ -157,13 +104,9 @@ github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= -github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/quipo/dependencysolver v0.0.0-20170801134659-2b009cb4ddcc h1:hK577yxEJ2f5s8w2iy2KimZmgrdAUZUNftE1ESmg2/Q= github.com/quipo/dependencysolver v0.0.0-20170801134659-2b009cb4ddcc/go.mod h1:OQt6Zo5B3Zs+C49xul8kcHo+fZ1mCLPvd0LFxiZ2DHc= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= @@ -185,10 +128,6 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= -github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= -github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -201,7 +140,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -211,14 +149,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo= -github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4= github.com/tilinna/z85 v1.0.0 h1:uqFnJBlD01dosSeo5sK1G1YGbPuwqVHqR+12OJDRjUw= github.com/tilinna/z85 v1.0.0/go.mod h1:EfpFU/DUY4ddEy6CRvk2l+UQNEzHbh+bqBQS+04Nkxs= -github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= -github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= -github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= -github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/warpstreamlabs/bento v1.4.1 h1:DL+j+0iIqiIq4fPreSShPJWNPC7JbYj4x/RMr8rwPAg= @@ -234,12 +166,8 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGC github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= diff --git a/worker/pkg/benthos/default_transform/processor.go b/worker/pkg/benthos/default_transform/processor.go index b9ca6cfda9..ce0c48bd85 100644 --- a/worker/pkg/benthos/default_transform/processor.go +++ b/worker/pkg/benthos/default_transform/processor.go @@ -7,7 +7,7 @@ import ( "reflect" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" - transformer "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" + transformer_executor "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformer_executor" "google.golang.org/protobuf/encoding/protojson" @@ -45,7 +45,7 @@ func ReisterDefaultTransformerProcessor(env *service.Environment) error { type defaultTransformerProcessor struct { mappedKeys map[string]struct{} - defaultTransformersInitMap map[primitiveType]*transformer.TransformerExecutor + defaultTransformersInitMap map[primitiveType]*transformer_executor.TransformerExecutor logger *service.Logger } @@ -203,13 +203,13 @@ func (m *defaultTransformerProcessor) getValue(transformerKey primitiveType, val return value, nil } -func initDefaultTransformers(defaultTransformerMap map[primitiveType]*mgmtv1alpha1.JobMappingTransformer) (map[primitiveType]*transformer.TransformerExecutor, error) { - transformersInit := map[primitiveType]*transformer.TransformerExecutor{} +func initDefaultTransformers(defaultTransformerMap map[primitiveType]*mgmtv1alpha1.JobMappingTransformer) (map[primitiveType]*transformer_executor.TransformerExecutor, error) { + transformersInit := map[primitiveType]*transformer_executor.TransformerExecutor{} for k, t := range defaultTransformerMap { if !shouldProcess(t) { continue } - init, err := transformer.InitializeTransformer(t) + init, err := transformer_executor.InitializeTransformer(t) if err != nil { return nil, err } diff --git a/worker/pkg/benthos/default_transform/processor_test.go b/worker/pkg/benthos/default_transform/processor_test.go index 28f66b13e7..04d6175bdb 100644 --- a/worker/pkg/benthos/default_transform/processor_test.go +++ b/worker/pkg/benthos/default_transform/processor_test.go @@ -3,7 +3,7 @@ package neosync_benthos_defaulttransform import ( "testing" - transformer "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" + transformer_executor "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformer_executor" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -95,7 +95,7 @@ func Test_transformRoot(t *testing.T) { func createMockProcessor(mappedKeys map[string]struct{}) *defaultTransformerProcessor { return &defaultTransformerProcessor{ mappedKeys: mappedKeys, - defaultTransformersInitMap: map[primitiveType]*transformer.TransformerExecutor{ + defaultTransformersInitMap: map[primitiveType]*transformer_executor.TransformerExecutor{ String: { Mutate: func(value any, opts any) (any, error) { return "transformed_" + value.(string), nil diff --git a/worker/pkg/benthos/environment/environment.go b/worker/pkg/benthos/environment/environment.go index a451300262..c11d156d00 100644 --- a/worker/pkg/benthos/environment/environment.go +++ b/worker/pkg/benthos/environment/environment.go @@ -9,6 +9,7 @@ import ( neosync_benthos_defaulttransform "github.com/nucleuscloud/neosync/worker/pkg/benthos/default_transform" neosync_benthos_dynamodb "github.com/nucleuscloud/neosync/worker/pkg/benthos/dynamodb" neosync_benthos_error "github.com/nucleuscloud/neosync/worker/pkg/benthos/error" + javascript_processor "github.com/nucleuscloud/neosync/worker/pkg/benthos/javascript" neosync_benthos_json "github.com/nucleuscloud/neosync/worker/pkg/benthos/json" benthos_metrics "github.com/nucleuscloud/neosync/worker/pkg/benthos/metrics" neosync_benthos_mongodb "github.com/nucleuscloud/neosync/worker/pkg/benthos/mongodb" @@ -189,6 +190,11 @@ func NewWithEnvironment(env *service.Environment, logger *slog.Logger, opts ...O return nil, fmt.Errorf("unable to register Neosync to MSSQL processor to benthos instance: %w", err) } + err = javascript_processor.RegisterNeosyncJavascriptProcessor(env) + if err != nil { + return nil, fmt.Errorf("unable to register javascript processor to benthos instance: %w", err) + } + if config.blobEnv != nil { env.UseBloblangEnvironment(config.blobEnv) } diff --git a/worker/pkg/benthos/javascript/benthos_value_api.go b/worker/pkg/benthos/javascript/benthos_value_api.go new file mode 100644 index 0000000000..f06edabb56 --- /dev/null +++ b/worker/pkg/benthos/javascript/benthos_value_api.go @@ -0,0 +1,50 @@ +package javascript_processor + +import ( + javascript_functions "github.com/nucleuscloud/neosync/internal/javascript/functions" + "github.com/warpstreamlabs/bento/public/service" +) + +// this is not thread safe +type benthosValueApi struct { + message *service.Message +} + +func newBatchBenthosValueApi() *benthosValueApi { + return &benthosValueApi{} +} + +// used by batch processor to update the target message while being able to reuse the same VM +func (b *benthosValueApi) SetMessage(message *service.Message) { + b.message = message +} + +func (b *benthosValueApi) Message() *service.Message { + return b.message +} + +var _ javascript_functions.ValueApi = (*benthosValueApi)(nil) + +func (b *benthosValueApi) SetBytes(bytes []byte) { + b.message.SetBytes(bytes) +} + +func (b *benthosValueApi) AsBytes() ([]byte, error) { + return b.message.AsBytes() +} + +func (b *benthosValueApi) SetStructured(value any) { + b.message.SetStructured(value) +} + +func (b *benthosValueApi) AsStructured() (any, error) { + return b.message.AsStructured() +} + +func (b *benthosValueApi) MetaGet(key string) (any, bool) { + return b.message.MetaGet(key) +} + +func (b *benthosValueApi) MetaSetMut(key string, value any) { + b.message.MetaSetMut(key, value) +} diff --git a/worker/pkg/benthos/javascript/casts.go b/worker/pkg/benthos/javascript/casts.go deleted file mode 100644 index 667f53f993..0000000000 --- a/worker/pkg/benthos/javascript/casts.go +++ /dev/null @@ -1,44 +0,0 @@ -package javascript - -import ( - "errors" - - "github.com/dop251/goja" -) - -func getMapFromValue(val goja.Value) (map[string]any, error) { - outVal := val.Export() - v, ok := outVal.(map[string]any) - if !ok { - return nil, errors.New("value is not of type map") - } - return v, nil -} - -func getSliceFromValue(val goja.Value) ([]any, error) { - outVal := val.Export() - v, ok := outVal.([]any) - if !ok { - return nil, errors.New("value is not of type slice") - } - return v, nil -} - -func getMapSliceFromValue(val goja.Value) ([]map[string]any, error) { - outVal := val.Export() - if v, ok := outVal.([]map[string]any); ok { - return v, nil - } - vSlice, ok := outVal.([]any) - if !ok { - return nil, errors.New("value is not of type map slice") - } - v := make([]map[string]any, len(vSlice)) - for i, e := range vSlice { - v[i], ok = e.(map[string]any) - if !ok { - return nil, errors.New("value is not of type map slice") - } - } - return v, nil -} diff --git a/worker/pkg/benthos/javascript/functions.go b/worker/pkg/benthos/javascript/functions.go deleted file mode 100644 index bf0b147010..0000000000 --- a/worker/pkg/benthos/javascript/functions.go +++ /dev/null @@ -1,377 +0,0 @@ -package javascript - -import ( - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/dop251/goja" - - "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" - "github.com/warpstreamlabs/bento/public/service" -) - -type jsFunction func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) - -type jsFunctionParam struct { - name string - typeStr string - what string -} - -type jsFunctionDefinition struct { - namespace string - name string - description string - params []jsFunctionParam - examples []string - ctor func(r *vmRunner) jsFunction -} - -func (j *jsFunctionDefinition) Param(name, typeStr, what string) *jsFunctionDefinition { - j.params = append(j.params, jsFunctionParam{ - name: name, - typeStr: typeStr, - what: what, - }) - return j -} - -func (j *jsFunctionDefinition) Example(example string) *jsFunctionDefinition { - j.examples = append(j.examples, example) - return j -} - -func (j *jsFunctionDefinition) FnCtor(ctor func(r *vmRunner) jsFunction) *jsFunctionDefinition { - j.ctor = ctor - return j -} - -func (j *jsFunctionDefinition) Namespace(namespace string) *jsFunctionDefinition { - j.namespace = namespace - return j -} - -func (j *jsFunctionDefinition) String() string { - var description strings.Builder - - _, _ = fmt.Fprintf(&description, "### `benthos.%v`\n\n", j.name) - _, _ = description.WriteString(j.description + "\n\n") - if len(j.params) > 0 { - _, _ = description.WriteString("#### Parameters\n\n") - for _, p := range j.params { - _, _ = fmt.Fprintf(&description, "**`%v`** <%v> %v \n", p.name, p.typeStr, p.what) - } - _, _ = description.WriteString("\n") - } - - if len(j.examples) > 0 { - _, _ = description.WriteString("#### Examples\n\n") - for _, e := range j.examples { - _, _ = description.WriteString("```javascript\n") - _, _ = description.WriteString(strings.Trim(e, "\n")) - _, _ = description.WriteString("\n```\n") - } - } - - return description.String() -} - -var vmRunnerFunctionCtors = map[string]*jsFunctionDefinition{} - -func registerVMRunnerFunction(name, description string) *jsFunctionDefinition { - fn := &jsFunctionDefinition{ - name: name, - description: description, - } - vmRunnerFunctionCtors[name] = fn - return fn -} - -func init() { - // registers neosync transformers - neosyncTransformers := transformers.GetNeosyncTransformers() - for _, f := range neosyncTransformers { - templateData, err := f.GetJsTemplateData() - if err != nil { - panic(err) - } - - def := registerVMRunnerFunction(templateData.Name, templateData.Description) - def.Param("value", "any", "The value to be transformed.") - def.Param("opts", "object", "Transformer options config") - def.Example(templateData.Example) - def.Namespace(neosyncFnCtxName) - def.FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var ( - value any - opts map[string]any - ) - if err := parseArgs(call, &value, &opts); err != nil { - return nil, err - } - goOpts, err := f.ParseOptions(opts) - if err != nil { - return nil, err - } - return f.Transform(value, goOpts) - } - }) - } - - // registers neosync generators - neosyncGenerators := transformers.GetNeosyncGenerators() - for _, f := range neosyncGenerators { - templateData, err := f.GetJsTemplateData() - if err != nil { - panic(err) - } - - def := registerVMRunnerFunction(templateData.Name, templateData.Description) - def.Param("opts", "object", "Transformer options config") - def.Namespace(neosyncFnCtxName) - def.FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var ( - opts map[string]any - ) - if err := parseArgs(call, &opts); err != nil { - return nil, err - } - goOpts, err := f.ParseOptions(opts) - if err != nil { - return nil, err - } - return f.Generate(goOpts) - } - }) - } -} - -var _ = registerVMRunnerFunction( - "v0_fetch", - `Executes an HTTP request synchronously and returns the result as an object of the form `+"`"+`{"status":200,"body":"foo"}`+"`"+`.`, -). - Namespace(benthosFnCtxName). - Param("url", "string", "The URL to fetch"). - Param("headers", "object(string,string)", "An object of string/string key/value pairs to add the request as headers."). - Param("method", "string", "The method of the request."). - Param("body", "(optional) string", "A body to send."). - Example(` -let result = benthos.v0_fetch("http://example.com", {}, "GET", "") -benthos.v0_msg_set_structured(result); -`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var ( - url string - httpHeaders map[string]any - method = "GET" - payload = "" - ) - if err := parseArgs(call, &url, &httpHeaders, &method, &payload); err != nil { - return nil, err - } - - var payloadReader io.Reader - if payload != "" { - payloadReader = strings.NewReader(payload) - } - - req, err := http.NewRequest(method, url, payloadReader) - if err != nil { - return nil, err - } - - // Parse HTTP headers - for k, v := range httpHeaders { - vStr, _ := v.(string) - req.Header.Add(k, vStr) - } - - // Do request - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - return map[string]any{ - "status": resp.StatusCode, - "body": string(respBody), - }, nil - } - }) - -var _ = registerVMRunnerFunction("v0_msg_set_string", `Set the contents of the processed message to a given string.`). - Namespace(benthosFnCtxName). - Param("value", "string", "The value to set it to."). - Example(`benthos.v0_msg_set_string("hello world");`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var value string - if err := parseArgs(call, &value); err != nil { - return nil, err - } - - r.targetMessage.SetBytes([]byte(value)) - return nil, nil - } - }) - -var _ = registerVMRunnerFunction("v0_msg_as_string", `Obtain the raw contents of the processed message as a string.`). - Namespace(benthosFnCtxName). - Example(`let contents = benthos.v0_msg_as_string();`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - b, err := r.targetMessage.AsBytes() - if err != nil { - return nil, err - } - return string(b), nil - } - }) - -var _ = registerVMRunnerFunction("v0_msg_set_structured", `Set the root of the processed message to a given value of any type.`). - Namespace(benthosFnCtxName). - Param("value", "anything", "The value to set it to."). - Example(` -benthos.v0_msg_set_structured({ - "foo": "a thing", - "bar": "something else", - "baz": 1234 -}); -`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var value any - if err := parseArgs(call, &value); err != nil { - return nil, err - } - - r.targetMessage.SetStructured(value) - return nil, nil - } - }) - -var _ = registerVMRunnerFunction("v0_msg_as_structured", `Obtain the root of the processed message as a structured value. If the message is not valid JSON or has not already been expanded into a structured form this function will throw an error.`). - Namespace(benthosFnCtxName). - Example(`let foo = benthos.v0_msg_as_structured().foo;`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - return r.targetMessage.AsStructured() - } - }) - -var _ = registerVMRunnerFunction("v0_msg_exists_meta", `Check that a metadata key exists.`). - Namespace(benthosFnCtxName). - Param("name", "string", "The metadata key to search for."). - Example(`if (benthos.v0_msg_exists_meta("kafka_key")) {}`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var name string - if err := parseArgs(call, &name); err != nil { - return nil, err - } - - _, ok := r.targetMessage.MetaGet(name) - if !ok { - return false, nil - } - return true, nil - } - }) - -var _ = registerVMRunnerFunction("v0_msg_get_meta", `Get the value of a metadata key from the processed message.`). - Namespace(benthosFnCtxName). - Param("name", "string", "The metadata key to search for."). - Example(`let key = benthos.v0_msg_get_meta("kafka_key");`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var name string - if err := parseArgs(call, &name); err != nil { - return nil, err - } - - result, ok := r.targetMessage.MetaGet(name) - if !ok { - return nil, errors.New("key not found") - } - return result, nil - } - }) - -var _ = registerVMRunnerFunction("v0_msg_set_meta", `Set a metadata key on the processed message to a value.`). - Namespace(benthosFnCtxName). - Param("name", "string", "The metadata key to set."). - Param("value", "anything", "The value to set it to."). - Example(`benthos.v0_msg_set_meta("thing", "hello world");`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var ( - name string - value any - ) - if err := parseArgs(call, &name, &value); err != nil { - return "", err - } - r.targetMessage.MetaSetMut(name, value) - return nil, nil - } - }) - -var _ = registerVMRunnerFunction("patchStructuredMessage", `Update multiple fields in the structured data of the processed message.`). - Namespace(neosyncFnCtxName). - Param("updates", "object", "A map of field names to their new values."). - Example(`neosync.patchStructuredMessage({"user_id": 12345, "timestamp": "2024-09-23T12:34:56Z"});`). - FnCtor(func(r *vmRunner) jsFunction { - return func(call goja.FunctionCall, rt *goja.Runtime, l *service.Logger) (any, error) { - var updates map[string]any - if err := parseArgs(call, &updates); err != nil { - return nil, err - } - - // original structured data - originalData, err := r.targetMessage.AsStructuredMut() - if err != nil { - return nil, fmt.Errorf("failed to get structured data: %w", err) - } - - originalMap, ok := originalData.(map[string]any) - if !ok { - return nil, fmt.Errorf("structured data is not a map") - } - - for key, value := range updates { - setNestedProperty(originalMap, key, value) - } - - r.targetMessage.SetStructured(originalMap) - - return nil, nil - } - }) - -func setNestedProperty(obj map[string]any, path string, value any) { - parts := strings.Split(path, ".") - current := obj - - for i, part := range parts { - if i == len(parts)-1 { - current[part] = value - } else { - if _, ok := current[part]; !ok { - current[part] = make(map[string]any) - } - current = current[part].(map[string]any) - } - } -} diff --git a/worker/pkg/benthos/javascript/ifs/http.go b/worker/pkg/benthos/javascript/ifs/http.go deleted file mode 100644 index eded35ae91..0000000000 --- a/worker/pkg/benthos/javascript/ifs/http.go +++ /dev/null @@ -1,83 +0,0 @@ -package ifs - -import ( - "errors" - "io" - "io/fs" - "net/http" -) - -var _ http.FileSystem = ToHTTP(OS()) - -type asHTTP struct { - f fs.FS -} - -type asHTTPFile struct { - file fs.File -} - -// ToHTTP converts an fs.FS into an http.FileSystem in a way that doesn't -// modify the root path. -func ToHTTP(f fs.FS) *asHTTP { - return &asHTTP{f: f} -} - -func (h *asHTTP) Open(name string) (http.File, error) { - f, err := h.f.Open(name) - if err != nil { - return nil, err - } - return asHTTPFile{file: f}, nil -} - -func (f asHTTPFile) ReadDir(count int) ([]fs.DirEntry, error) { - d, ok := f.file.(fs.ReadDirFile) - if !ok { - return nil, errMissingReadDir - } - return d.ReadDir(count) -} - -func (f asHTTPFile) Close() error { return f.file.Close() } -func (f asHTTPFile) Read(b []byte) (int, error) { return f.file.Read(b) } -func (f asHTTPFile) Stat() (fs.FileInfo, error) { return f.file.Stat() } - -var ( - errMissingSeek = errors.New("io.File missing Seek method") - errMissingReadDir = errors.New("io.File directory missing ReadDir method") -) - -func (f asHTTPFile) Seek(offset int64, whence int) (int64, error) { - s, ok := f.file.(io.Seeker) - if !ok { - return 0, errMissingSeek - } - return s.Seek(offset, whence) -} - -func (f asHTTPFile) Readdir(count int) ([]fs.FileInfo, error) { - d, ok := f.file.(fs.ReadDirFile) - if !ok { - return nil, errMissingReadDir - } - var list []fs.FileInfo - for { - dirs, err := d.ReadDir(count - len(list)) - for _, dir := range dirs { - info, err := dir.Info() - if err != nil { - // Pretend it doesn't exist, like (*os.File).Readdir does. - continue - } - list = append(list, info) - } - if err != nil { - return list, err - } - if count < 0 || len(list) >= count { - break - } - } - return list, nil -} diff --git a/worker/pkg/benthos/javascript/ifs/os.go b/worker/pkg/benthos/javascript/ifs/os.go deleted file mode 100644 index 6339a10a6f..0000000000 --- a/worker/pkg/benthos/javascript/ifs/os.go +++ /dev/null @@ -1,95 +0,0 @@ -package ifs - -import ( - "errors" - "io" - "io/fs" - "os" -) - -var _ fs.FS = OS() - -// FS is a superset of fs.FS that includes goodies that benthos components -// specifically need. -type FS interface { - Open(name string) (fs.File, error) - OpenFile(name string, flag int, perm fs.FileMode) (fs.File, error) - Stat(name string) (fs.FileInfo, error) - Remove(name string) error - MkdirAll(path string, perm fs.FileMode) error -} - -// ReadFile opens a file with the RDONLY flag and returns all bytes from it. -func ReadFile(f fs.FS, name string) ([]byte, error) { - var i fs.File - var err error - if ef, ok := f.(FS); ok { - i, err = ef.OpenFile(name, os.O_RDONLY, 0) - } else { - i, err = f.Open(name) - } - if err != nil { - return nil, err - } - return io.ReadAll(i) -} - -// WriteFile opens a file with O_WRONLY|O_CREATE|O_TRUNC flags and writes the -// data to it. -func WriteFile(f fs.FS, name string, data []byte, perm fs.FileMode) error { - h, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - if err != nil { - return err - } - _, err = h.Write(data) - if err1 := h.Close(); err1 != nil && err == nil { - err = err1 - } - return err -} - -// FileWrite attempts to write to an fs.File provided it supports io.Writer. -func FileWrite(file fs.File, data []byte) (int, error) { - writer, isw := file.(io.Writer) - if !isw { - return 0, errors.New("failed to open a writable file") - } - return writer.Write(data) -} - -// OS implements fs.FS as if calls were being made directly via the os package, -// with which relative paths are resolved from the directory the process is -// executed from. -func OS() FS { - return osPTI -} - -// IsOS returns true if the provided FS implementation is a wrapper around OS -// access obtained via OS(). -func IsOS(f FS) bool { - return f == osPTI -} - -var osPTI = &osPT{} - -type osPT struct{} - -func (o *osPT) Open(name string) (fs.File, error) { - return os.Open(name) -} - -func (o *osPT) OpenFile(name string, flag int, perm fs.FileMode) (fs.File, error) { - return os.OpenFile(name, flag, perm) -} - -func (o *osPT) Stat(name string) (fs.FileInfo, error) { - return os.Stat(name) -} - -func (o *osPT) Remove(name string) error { - return os.Remove(name) -} - -func (o *osPT) MkdirAll(path string, perm fs.FileMode) error { - return os.MkdirAll(path, perm) -} diff --git a/worker/pkg/benthos/javascript/ifs/os_test.go b/worker/pkg/benthos/javascript/ifs/os_test.go deleted file mode 100644 index cdad9a4d80..0000000000 --- a/worker/pkg/benthos/javascript/ifs/os_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package ifs - -import ( - "errors" - "io/fs" - "testing" - "testing/fstest" - - "github.com/stretchr/testify/require" -) - -type testFS struct { - fstest.MapFS -} - -func (t testFS) MkdirAll(path string, perm fs.FileMode) error { - return errors.New("not implemented") -} - -func (t testFS) OpenFile(name string, flag int, perm fs.FileMode) (fs.File, error) { - return nil, errors.New("not implemented") -} - -func (t testFS) Remove(name string) error { - return errors.New("not implemented") -} - -func TestOSAccess(t *testing.T) { - var fss FS = testFS{} - - require.False(t, IsOS(fss)) - - fss = OS() - - require.True(t, IsOS(fss)) -} diff --git a/worker/pkg/benthos/javascript/logger.go b/worker/pkg/benthos/javascript/logger.go deleted file mode 100644 index ca138ff825..0000000000 --- a/worker/pkg/benthos/javascript/logger.go +++ /dev/null @@ -1,23 +0,0 @@ -package javascript - -import "github.com/warpstreamlabs/bento/public/service" - -// Logger wraps the service.Logger so that we can define the below methods. -type Logger struct { - l *service.Logger -} - -// Log will be used for "console.log()" in JS -func (l *Logger) Log(message string) { - l.l.Info(message) -} - -// Warn will be used for "console.warn()" in JS -func (l *Logger) Warn(message string) { - l.l.Warn(message) -} - -// Error will be used for "console.error()" in JS -func (l *Logger) Error(message string) { - l.l.Error(message) -} diff --git a/worker/pkg/benthos/javascript/processor.go b/worker/pkg/benthos/javascript/processor.go index 64083ec632..18498d844b 100644 --- a/worker/pkg/benthos/javascript/processor.go +++ b/worker/pkg/benthos/javascript/processor.go @@ -1,243 +1,121 @@ -package javascript +package javascript_processor import ( "context" - "errors" "fmt" - "io" - "io/fs" - "path/filepath" - "runtime" - "sort" - "strings" + "log/slog" "sync" - "syscall" "github.com/dop251/goja" - "github.com/dop251/goja_nodejs/console" - "github.com/dop251/goja_nodejs/require" + "github.com/nucleuscloud/neosync/internal/benthos_slogger" + "github.com/nucleuscloud/neosync/internal/javascript" + javascript_vm "github.com/nucleuscloud/neosync/internal/javascript/vm" - "github.com/nucleuscloud/neosync/worker/pkg/benthos/javascript/ifs" "github.com/warpstreamlabs/bento/public/service" ) const ( - codeField = "code" - fileField = "file" - includeField = "global_folders" + codeField = "code" ) func javascriptProcessorConfig() *service.ConfigSpec { - functionsSlice := make([]string, 0, len(vmRunnerFunctionCtors)) - for k := range vmRunnerFunctionCtors { - functionsSlice = append(functionsSlice, k) - } - sort.Strings(functionsSlice) - - var description strings.Builder - for _, name := range functionsSlice { - _, _ = description.WriteString("\n") - _, _ = description.WriteString(vmRunnerFunctionCtors[name].String()) - } - return service.NewConfigSpec(). - Categories("Mapping"). - Version("4.14.0"). - Summary("Executes a provided JavaScript code block or file for each message."). - Description(` -The [execution engine](https://github.com/dop251/goja) behind this processor provides full ECMAScript 5.1 support (including regex and strict mode). Most of the ECMAScript 6 spec is implemented but this is a work in progress. - -Imports via `+"`require`"+` should work similarly to NodeJS, and access to the console is supported which will print via the Benthos logger. More caveats can be [found here](https://github.com/dop251/goja#known-incompatibilities-and-caveats). - -This processor is implemented using the [github.com/dop251/goja](https://github.com/dop251/goja) library.`). - Footnotes(` -## Runtime - -In order to optimize code execution JS runtimes are created on demand (in order to support parallel execution) and are reused across invocations. Therefore, it is important to understand that global state created by your programs will outlive individual invocations. In order for your programs to avoid failing after the first invocation ensure that you do not define variables at the global scope. - -Although technically possible, it is recommended that you do not rely on the global state for maintaining state across invocations as the pooling nature of the runtimes will prevent deterministic behavior. We aim to support deterministic strategies for mutating global state in the future. - -## Functions -`+description.String()+` -`). - Field(service.NewInterpolatedStringField(codeField). - Description("An inline JavaScript program to run. One of `"+codeField+"` or `"+fileField+"` must be defined."). - Optional()). - Field(service.NewInterpolatedStringField(fileField). - Description("A file containing a JavaScript program to run. One of `"+codeField+"` or `"+fileField+"` must be defined."). - Optional()). - Field(service.NewStringListField(includeField). - Description("List of folders that will be used to load modules from if the requested JS module is not found elsewhere."). - Default([]string{})). - LintRule(fmt.Sprintf(` -let codeLen = (this.%v | "").length() -let fileLen = (this.%v | "").length() -root = if $codeLen == 0 && $fileLen == 0 { - "either the code or file field must be specified" -} else if $codeLen > 0 && $fileLen > 0 { - "cannot specify both the code and file fields" -}`, codeField, fileField)). - Example( - `Simple mutation`, - `In this example we define a simple function that performs a basic mutation against messages, treating their contents as raw strings.`, - ` -pipeline: - processors: - - javascript: - code: 'benthos.v0_msg_set_string(benthos.v0_msg_as_string() + "hello world");' -`, - ). - Example( - `Structured mutation`, - `In this example we define a function that performs basic mutations against a structured message. Note that we encapsulate the logic within an anonymous function that is called for each invocation, this is required in order to avoid duplicate variable declarations in the global state.`, - ` -pipeline: - processors: - - javascript: - code: | - (() => { - let thing = benthos.v0_msg_as_structured(); - thing.num_keys = Object.keys(thing).length; - delete thing["b"]; - benthos.v0_msg_set_structured(thing); - })(); -`, - ) + Field(service.NewInterpolatedStringField(codeField)) } -// func RegisterNeosyncJavascriptProcessor() error { -// return service.RegisterBatchProcessor( -// "javascript", javascriptProcessorConfig(), -// func(conf *service.ParsedConfig, mgr *service.Resources) (service.BatchProcessor, error) { -// return newJavascriptProcessorFromConfig(conf, mgr) -// }) - -// } - -func init() { - err := service.RegisterBatchProcessor( +func RegisterNeosyncJavascriptProcessor(env *service.Environment) error { + return env.RegisterBatchProcessor( "neosync_javascript", javascriptProcessorConfig(), func(conf *service.ParsedConfig, mgr *service.Resources) (service.BatchProcessor, error) { return newJavascriptProcessorFromConfig(conf, mgr) }) - if err != nil { - panic(err) - } } -//------------------------------------------------------------------------------ - type javascriptProcessor struct { - program *goja.Program - requireRegistry *require.Registry - logger *service.Logger - vmPool sync.Pool -} - -func sourceLoader(serviceFS *service.FS) require.SourceLoader { - // Copy of `require.DefaultSourceLoader`: https://github.com/dop251/goja_nodejs/blob/e84d9a924c5ca9e541575e643b7efbca5705862f/require/module.go#L116-L141 - // with some slight adjustments because we need to use the Benthos manager filesystem for opening and reading files. - return func(filename string) ([]byte, error) { - fp := filepath.FromSlash(filename) - f, err := serviceFS.Open(fp) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - err = require.ModuleFileDoesNotExistError - } else if runtime.GOOS == "windows" { - if errors.Is(err, syscall.Errno(0x7b)) { // ERROR_INVALID_NAME, The filename, directory name, or volume label syntax is incorrect. - err = require.ModuleFileDoesNotExistError - } - } - return nil, err - } - - defer f.Close() - // On some systems (e.g. plan9 and FreeBSD) it is possible to use the standard read() call on directories - // which means we cannot rely on read() returning an error, we have to do stat() instead. - if fi, err := f.Stat(); err == nil { - if fi.IsDir() { - return nil, require.ModuleFileDoesNotExistError - } - } else { - return nil, err - } - - return io.ReadAll(f) - } + program *goja.Program + slogger *slog.Logger + vmPool sync.Pool } func newJavascriptProcessorFromConfig(conf *service.ParsedConfig, mgr *service.Resources) (*javascriptProcessor, error) { - code, _ := conf.FieldString(codeField) - file, _ := conf.FieldString(fileField) - if file == "" && code == "" { - return nil, fmt.Errorf("either a `%v` or `%v` must be specified", codeField, fileField) + code, err := conf.FieldString(codeField) + if err != nil { + return nil, err } filename := "main.js" - if file != "" { - // Open file and read code - codeBytes, err := ifs.ReadFile(mgr.FS(), file) - if err != nil { - return nil, fmt.Errorf("failed to open target file: %w", err) - } - filename = file - code = string(codeBytes) - } - program, err := goja.Compile(filename, code, false) if err != nil { return nil, fmt.Errorf("failed to compile javascript code: %v", err) } logger := mgr.Logger() - registryGlobalFolders, err := conf.FieldStringList(includeField) - if err != nil { - return nil, err - } - requireRegistry := require.NewRegistry( - require.WithGlobalFolders(registryGlobalFolders...), - require.WithLoader(sourceLoader(mgr.FS())), - ) - requireRegistry.RegisterNativeModule("console", console.RequireWithPrinter(&Logger{logger})) + slogger := benthos_slogger.NewSlogger(logger) return &javascriptProcessor{ - program: program, - requireRegistry: requireRegistry, - logger: logger, - vmPool: sync.Pool{}, + program: program, + slogger: slogger, + vmPool: sync.Pool{ + New: func() any { + val, err := newPoolItem(slogger) + if err != nil { + return err + } + return val + }, + }, }, nil } +type vmPoolItem struct { + runner *javascript_vm.Runner + valueApi *benthosValueApi +} + func (j *javascriptProcessor) ProcessBatch(ctx context.Context, batch service.MessageBatch) ([]service.MessageBatch, error) { - var vr *vmRunner - var err error - if vmRunnerPtr := j.vmPool.Get(); vmRunnerPtr != nil { - vr = vmRunnerPtr.(*vmRunner) - } else { - if vr, err = j.newVM(); err != nil { + var runner *javascript_vm.Runner + var valueApi *benthosValueApi + + switch poolItem := j.vmPool.Get().(type) { + case *vmPoolItem: + runner = poolItem.runner + valueApi = poolItem.valueApi + defer func() { + poolItem.valueApi.SetMessage(nil) // reset the message to nil + j.vmPool.Put(poolItem) + }() + case error: + return nil, poolItem + } + + var newBatch service.MessageBatch + + for i := range batch { + valueApi.SetMessage(batch[i]) + _, err := runner.Run(ctx, j.program) + if err != nil { return nil, err } + if newMsg := valueApi.Message(); newMsg != nil { + newBatch = append(newBatch, newMsg) + } } - defer func() { - // TODO: Decide whether to reset the program - j.vmPool.Put(vr) - }() - b, err := vr.Run(ctx, batch) - if err != nil { - return nil, err - } - return []service.MessageBatch{b}, nil + return []service.MessageBatch{newBatch}, nil } func (j *javascriptProcessor) Close(ctx context.Context) error { - for { - mr := j.vmPool.Get() - if mr == nil { - return nil - } - if err := mr.(*vmRunner).Close(ctx); err != nil { - return err - } + return nil +} + +func newPoolItem(logger *slog.Logger) (*vmPoolItem, error) { + valueApi := newBatchBenthosValueApi() + runner, err := javascript.NewDefaultValueRunner(valueApi, logger) + if err != nil { + return nil, err } + return &vmPoolItem{ + valueApi: valueApi, + runner: runner, + }, nil } diff --git a/worker/pkg/benthos/javascript/processor_test.go b/worker/pkg/benthos/javascript/processor_test.go index 4b4a1e3306..6d5c7c73d2 100644 --- a/worker/pkg/benthos/javascript/processor_test.go +++ b/worker/pkg/benthos/javascript/processor_test.go @@ -1,4 +1,4 @@ -package javascript +package javascript_processor import ( "bytes" @@ -7,8 +7,6 @@ import ( "io" "net/http" "net/http/httptest" - "os" - "path" "testing" "time" @@ -254,87 +252,6 @@ code: | require.NoError(t, proc.Close(bCtx)) } -func TestProcessorBasicFromFile(t *testing.T) { - tmpDir := t.TempDir() - require.NoError(t, os.WriteFile(path.Join(tmpDir, "foo.js"), []byte(` -(() => { - let foo = "hello world" - benthos.v0_msg_set_string(benthos.v0_msg_as_string() + foo); -})(); -`), 0o644)) - - conf, err := javascriptProcessorConfig().ParseYAML(fmt.Sprintf(` -file: %v -`, path.Join(tmpDir, "foo.js")), nil) - require.NoError(t, err) - - proc, err := newJavascriptProcessorFromConfig(conf, service.MockResources()) - require.NoError(t, err) - - bCtx, done := context.WithTimeout(context.Background(), time.Second*30) - defer done() - - resBatches, err := proc.ProcessBatch(bCtx, service.MessageBatch{ - service.NewMessage([]byte("first ")), - service.NewMessage([]byte("second ")), - }) - require.NoError(t, err) - require.Len(t, resBatches, 1) - require.Len(t, resBatches[0], 2) - - resBytes, err := resBatches[0][0].AsBytes() - require.NoError(t, err) - assert.Equal(t, "first hello world", string(resBytes)) - - resBytes, err = resBatches[0][1].AsBytes() - require.NoError(t, err) - assert.Equal(t, "second hello world", string(resBytes)) - - require.NoError(t, proc.Close(bCtx)) -} - -func TestProcessorBasicFromModule(t *testing.T) { - tmpDir := t.TempDir() - // The file must have the .js extension and be imported without it using `require('blobber')` - require.NoError(t, os.WriteFile(path.Join(tmpDir, "blobber.js"), []byte(` -function blobber() { - return 'blobber module'; -} - -module.exports = blobber; -`), 0o644)) - - conf, err := javascriptProcessorConfig().ParseYAML(fmt.Sprintf(` -code: | - (() => { - const blobber = require('blobber'); - - benthos.v0_msg_set_string(benthos.v0_msg_as_string() + blobber()); - })(); -global_folders: [ "%s" ] -`, tmpDir), nil) - require.NoError(t, err) - - proc, err := newJavascriptProcessorFromConfig(conf, service.MockResources()) - require.NoError(t, err) - - bCtx, done := context.WithTimeout(context.Background(), time.Second*30) - defer done() - - resBatches, err := proc.ProcessBatch(bCtx, service.MessageBatch{ - service.NewMessage([]byte("hello ")), - }) - require.NoError(t, err) - require.Len(t, resBatches, 1) - require.Len(t, resBatches[0], 1) - - resBytes, err := resBatches[0][0].AsBytes() - require.NoError(t, err) - assert.Equal(t, "hello blobber module", string(resBytes)) - - require.NoError(t, proc.Close(bCtx)) -} - func TestProcessorHTTPFetch(t *testing.T) { testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { bodyBytes, err := io.ReadAll(r.Body) diff --git a/worker/pkg/benthos/javascript/vm.go b/worker/pkg/benthos/javascript/vm.go deleted file mode 100644 index 9457012b4b..0000000000 --- a/worker/pkg/benthos/javascript/vm.go +++ /dev/null @@ -1,149 +0,0 @@ -package javascript - -import ( - "context" - "fmt" - - "github.com/dop251/goja" - "github.com/dop251/goja_nodejs/console" - - "github.com/warpstreamlabs/bento/public/service" -) - -type vmRunner struct { - vm *goja.Runtime - p *goja.Program - - logger *service.Logger - - runBatch service.MessageBatch - targetMessage *service.Message - targetIndex int -} - -func (j *javascriptProcessor) newVM() (*vmRunner, error) { - vm := goja.New() - - j.requireRegistry.Enable(vm) - console.Enable(vm) - - vr := &vmRunner{ - vm: vm, - logger: j.logger, - p: j.program, - } - - for name, fc := range vmRunnerFunctionCtors { - if err := setFunction(vr, fc.namespace, name, fc.ctor(vr)); err != nil { - return nil, err - } - } - return vr, nil -} - -// The namespace within all our function definitions -const benthosFnCtxName = "benthos" -const neosyncFnCtxName = "neosync" - -func setFunction(vr *vmRunner, namespace, name string, function jsFunction) error { - var targetObj *goja.Object - if targetObjValue := vr.vm.GlobalObject().Get(namespace); targetObjValue != nil { - targetObj = targetObjValue.ToObject(vr.vm) - } - if targetObj == nil { - if err := vr.vm.GlobalObject().Set(namespace, map[string]any{}); err != nil { - return fmt.Errorf("failed to set global %s object: %w", namespace, err) - } - targetObj = vr.vm.GlobalObject().Get(namespace).ToObject(vr.vm) - } - - if err := targetObj.Set(name, func(call goja.FunctionCall, rt *goja.Runtime) goja.Value { - l := vr.logger.With("function", name) - result, err := function(call, rt, l) - if err != nil { - panic(rt.ToValue(err.Error())) - } - return rt.ToValue(result) - }); err != nil { - return fmt.Errorf("failed to set global %s function %v: %w", namespace, name, err) - } - - return nil -} - -func parseArgs(call goja.FunctionCall, ptrs ...any) error { - if len(ptrs) < len(call.Arguments) { - return fmt.Errorf("have %d arguments, but only %d pointers to parse into", len(call.Arguments), len(ptrs)) - } - - for i := 0; i < len(call.Arguments); i++ { - arg, ptr := call.Argument(i), ptrs[i] - - if goja.IsUndefined(arg) { - return fmt.Errorf("argument at position %d is undefined", i) - } - - var err error - switch p := ptr.(type) { - case *string: - *p = arg.String() - case *int: - *p = int(arg.ToInteger()) - case *int64: - *p = arg.ToInteger() - case *float64: - *p = arg.ToFloat() - case *map[string]any: - *p, err = getMapFromValue(arg) - case *bool: - *p = arg.ToBoolean() - case *[]any: - *p, err = getSliceFromValue(arg) - case *[]map[string]any: - *p, err = getMapSliceFromValue(arg) - case *goja.Value: - *p = arg - case *any: - *p = arg.Export() - default: - return fmt.Errorf("encountered unhandled type %T while trying to parse %v into %v", arg.ExportType().String(), arg, p) - } - if err != nil { - return fmt.Errorf("could not parse %v (%s) into %v (%T): %v", arg, arg.ExportType().String(), ptr, ptr, err) - } - } - - return nil -} - -func (r *vmRunner) reset() { - r.runBatch = nil - r.targetMessage = nil - r.targetIndex = 0 -} - -func (r *vmRunner) Run(ctx context.Context, batch service.MessageBatch) (service.MessageBatch, error) { - defer r.reset() - - var newBatch service.MessageBatch - for i := range batch { - r.reset() - r.runBatch = batch - r.targetIndex = i - r.targetMessage = batch[i] - - _, err := r.vm.RunProgram(r.p) - if err != nil { - // TODO: Make this more granular, error could be message specific - return nil, err - } - if newMsg := r.targetMessage; newMsg != nil { - newBatch = append(newBatch, newMsg) - } - } - return newBatch, nil -} - -func (r *vmRunner) Close(ctx context.Context) error { - return nil -} diff --git a/worker/pkg/benthos/transformer_executor/anon_value_api.go b/worker/pkg/benthos/transformer_executor/anon_value_api.go new file mode 100644 index 0000000000..2274aa7962 --- /dev/null +++ b/worker/pkg/benthos/transformer_executor/anon_value_api.go @@ -0,0 +1,74 @@ +package transformer_executor + +import ( + "encoding/json" + "fmt" + + javascript_functions "github.com/nucleuscloud/neosync/internal/javascript/functions" + "github.com/warpstreamlabs/bento/public/service" +) + +type anonValueApi struct { + message *service.Message +} + +var _ javascript_functions.ValueApi = (*anonValueApi)(nil) + +func newAnonValueApi() *anonValueApi { + return &anonValueApi{} +} + +func (b *anonValueApi) SetMessage(message *service.Message) { + b.message = message +} + +func (b *anonValueApi) Message() *service.Message { + return b.message +} + +func (b *anonValueApi) SetBytes(bytes []byte) { + b.message.SetBytes(bytes) +} + +func (b *anonValueApi) AsBytes() ([]byte, error) { + return b.message.AsBytes() +} + +func (b *anonValueApi) SetStructured(value any) { + b.message.SetStructured(value) +} + +func (b *anonValueApi) AsStructured() (any, error) { + return b.message.AsStructured() +} + +func (b *anonValueApi) MetaGet(key string) (any, bool) { + return b.message.MetaGet(key) +} + +func (b *anonValueApi) MetaSetMut(key string, value any) { + b.message.MetaSetMut(key, value) +} + +func (b *anonValueApi) GetPropertyPathValue(propertyPath string) (any, error) { + if b.message == nil { + return nil, fmt.Errorf("message is nil") + } + structuredValue, err := b.message.AsStructured() + if err != nil { + return nil, fmt.Errorf("failed to get structured message: %w", err) + } + structuredValueMap, ok := structuredValue.(map[string]any) + if !ok { + return nil, fmt.Errorf("structured value is not a map[string]any") + } + return structuredValueMap[propertyPath], nil +} + +func NewMessage(input map[string]any) (*service.Message, error) { + bits, err := json.Marshal(input) + if err != nil { + return nil, fmt.Errorf("failed to marshal input map: %w", err) + } + return service.NewMessage(bits), nil +} diff --git a/worker/pkg/benthos/transformer_executor/anon_value_api_test.go b/worker/pkg/benthos/transformer_executor/anon_value_api_test.go new file mode 100644 index 0000000000..611c2a5aac --- /dev/null +++ b/worker/pkg/benthos/transformer_executor/anon_value_api_test.go @@ -0,0 +1,17 @@ +package transformer_executor + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_AnonValueApi_getPropertyPathValue(t *testing.T) { + message, err := NewMessage(map[string]any{"a": "b"}) + require.NoError(t, err) + api := newAnonValueApi() + api.SetMessage(message) + value, err := api.GetPropertyPathValue("a") + require.NoError(t, err) + require.Equal(t, "b", value) +} diff --git a/worker/pkg/benthos/transformers/transformer_initializer.go b/worker/pkg/benthos/transformer_executor/executor.go similarity index 67% rename from worker/pkg/benthos/transformers/transformer_initializer.go rename to worker/pkg/benthos/transformer_executor/executor.go index f8c2ddb93d..6efbbd56df 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer.go +++ b/worker/pkg/benthos/transformer_executor/executor.go @@ -1,4 +1,4 @@ -package transformers +package transformer_executor import ( "context" @@ -6,9 +6,13 @@ import ( "fmt" "log/slog" + "github.com/dop251/goja" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" presidioapi "github.com/nucleuscloud/neosync/internal/ee/presidio" ee_transformer_fns "github.com/nucleuscloud/neosync/internal/ee/transformers/functions" + "github.com/nucleuscloud/neosync/internal/javascript" + javascript_userland "github.com/nucleuscloud/neosync/internal/javascript/userland" + "github.com/nucleuscloud/neosync/worker/pkg/benthos/transformers" ) type TransformerExecutor struct { @@ -85,6 +89,80 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform return nil, err } return InitializeTransformerByConfigType(resolvedConfig, opts...) + case *mgmtv1alpha1.TransformerConfig_GenerateJavascriptConfig: + config := typedCfg.GenerateJavascriptConfig + if config == nil { + return nil, fmt.Errorf("generate javascript config is nil") + } + + valueApi := newAnonValueApi() + runner, err := javascript.NewDefaultValueRunner(valueApi, execCfg.logger) + if err != nil { + return nil, err + } + jsCode, propertyPath := javascript_userland.GetSingleGenerateFunction(config.GetCode()) + program, err := goja.Compile("main.js", jsCode, false) + if err != nil { + return nil, err + } + + return &TransformerExecutor{ + Opts: nil, + Mutate: func(value any, opts any) (any, error) { + inputMessage, err := NewMessage(map[string]any{}) + if err != nil { + return nil, fmt.Errorf("failed to create input message: %w", err) + } + valueApi.SetMessage(inputMessage) + _, err = runner.Run(context.Background(), program) + if err != nil { + return nil, fmt.Errorf("failed to run program: %w", err) + } + updatedValue, err := valueApi.GetPropertyPathValue(propertyPath) + if err != nil { + return nil, fmt.Errorf("failed to get property path value: %w", err) + } + return updatedValue, nil + }, + }, nil + case *mgmtv1alpha1.TransformerConfig_TransformJavascriptConfig: + config := typedCfg.TransformJavascriptConfig + if config == nil { + return nil, fmt.Errorf("transform javascript config is nil") + } + + valueApi := newAnonValueApi() + runner, err := javascript.NewDefaultValueRunner(valueApi, execCfg.logger) + if err != nil { + return nil, err + } + jsCode, propertyPath := javascript_userland.GetSingleTransformFunction(config.GetCode()) + program, err := goja.Compile("main.js", jsCode, false) + if err != nil { + return nil, err + } + + return &TransformerExecutor{ + Opts: nil, + Mutate: func(value any, opts any) (any, error) { + inputMessage, err := NewMessage(map[string]any{ + propertyPath: value, + }) + if err != nil { + return nil, fmt.Errorf("failed to create input message: %w", err) + } + valueApi.SetMessage(inputMessage) + _, err = runner.Run(context.Background(), program) + if err != nil { + return nil, fmt.Errorf("failed to run program: %w", err) + } + updatedValue, err := valueApi.GetPropertyPathValue(propertyPath) + if err != nil { + return nil, fmt.Errorf("failed to get property path value: %w", err) + } + return updatedValue, nil + }, + }, nil case *mgmtv1alpha1.TransformerConfig_PassthroughConfig: return &TransformerExecutor{ Opts: nil, @@ -94,11 +172,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateCategoricalConfig: config := transformerConfig.GetGenerateCategoricalConfig() - opts, err := NewGenerateCategoricalOptsFromConfig(config) + opts, err := transformers.NewGenerateCategoricalOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateCategorical().Generate + generate := transformers.NewGenerateCategorical().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -108,11 +186,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateBoolConfig: config := transformerConfig.GetGenerateBoolConfig() - opts, err := NewGenerateBoolOptsFromConfig(config) + opts, err := transformers.NewGenerateBoolOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateBool().Generate + generate := transformers.NewGenerateBool().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -123,11 +201,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformStringConfig: config := transformerConfig.GetTransformStringConfig() minLength := int64(3) // TODO: pull this value from the database schema - opts, err := NewTransformStringOptsFromConfig(config, &minLength, &maxLength) + opts, err := transformers.NewTransformStringOptsFromConfig(config, &minLength, &maxLength) if err != nil { return nil, err } - transform := NewTransformString().Transform + transform := transformers.NewTransformString().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -136,11 +214,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_TransformInt64Config: config := transformerConfig.GetTransformInt64Config() - opts, err := NewTransformInt64OptsFromConfig(config) + opts, err := transformers.NewTransformInt64OptsFromConfig(config) if err != nil { return nil, err } - transform := NewTransformInt64().Transform + transform := transformers.NewTransformInt64().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -150,11 +228,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFullNameConfig: config := transformerConfig.GetTransformFullNameConfig() - opts, err := NewTransformFullNameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewTransformFullNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - transform := NewTransformFullName().Transform + transform := transformers.NewTransformFullName().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -164,11 +242,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateEmailConfig: config := transformerConfig.GetGenerateEmailConfig() - opts, err := NewGenerateEmailOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateEmailOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateEmail().Generate + generate := transformers.NewGenerateEmail().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -178,11 +256,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformEmailConfig: config := transformerConfig.GetTransformEmailConfig() - opts, err := NewTransformEmailOptsFromConfig(config, &maxLength) + opts, err := transformers.NewTransformEmailOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - transform := NewTransformEmail().Transform + transform := transformers.NewTransformEmail().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -192,11 +270,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateCardNumberConfig: config := transformerConfig.GetGenerateCardNumberConfig() - opts, err := NewGenerateCardNumberOptsFromConfig(config) + opts, err := transformers.NewGenerateCardNumberOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateCardNumber().Generate + generate := transformers.NewGenerateCardNumber().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -206,11 +284,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateCityConfig: config := transformerConfig.GetGenerateCityConfig() - opts, err := NewGenerateCityOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateCityOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateCity().Generate + generate := transformers.NewGenerateCity().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -220,11 +298,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateE164PhoneNumberConfig: config := transformerConfig.GetGenerateE164PhoneNumberConfig() - opts, err := NewGenerateInternationalPhoneNumberOptsFromConfig(config) + opts, err := transformers.NewGenerateInternationalPhoneNumberOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateInternationalPhoneNumber().Generate + generate := transformers.NewGenerateInternationalPhoneNumber().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -233,11 +311,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_GenerateFirstNameConfig: config := transformerConfig.GetGenerateFirstNameConfig() - opts, err := NewGenerateFirstNameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateFirstNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateFirstName().Generate + generate := transformers.NewGenerateFirstName().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -247,11 +325,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateFloat64Config: config := transformerConfig.GetGenerateFloat64Config() - opts, err := NewGenerateFloat64OptsFromConfig(config, nil) + opts, err := transformers.NewGenerateFloat64OptsFromConfig(config, nil) if err != nil { return nil, err } - generate := NewGenerateFloat64().Generate + generate := transformers.NewGenerateFloat64().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -261,11 +339,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateFullAddressConfig: config := transformerConfig.GetGenerateFullAddressConfig() - opts, err := NewGenerateFullAddressOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateFullAddressOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateFullAddress().Generate + generate := transformers.NewGenerateFullAddress().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -275,11 +353,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateFullNameConfig: config := transformerConfig.GetGenerateFullNameConfig() - opts, err := NewGenerateFullNameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateFullNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateFullName().Generate + generate := transformers.NewGenerateFullName().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -289,11 +367,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateGenderConfig: config := transformerConfig.GetGenerateGenderConfig() - opts, err := NewGenerateGenderOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateGenderOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateGender().Generate + generate := transformers.NewGenerateGender().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -303,11 +381,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateInt64PhoneNumberConfig: config := transformerConfig.GetGenerateInt64PhoneNumberConfig() - opts, err := NewGenerateInt64PhoneNumberOptsFromConfig(config) + opts, err := transformers.NewGenerateInt64PhoneNumberOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateInt64PhoneNumber().Generate + generate := transformers.NewGenerateInt64PhoneNumber().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -317,11 +395,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateInt64Config: config := transformerConfig.GetGenerateInt64Config() - opts, err := NewGenerateInt64OptsFromConfig(config) + opts, err := transformers.NewGenerateInt64OptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateInt64().Generate + generate := transformers.NewGenerateInt64().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -331,11 +409,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateLastNameConfig: config := transformerConfig.GetGenerateLastNameConfig() - opts, err := NewGenerateLastNameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateLastNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateLastName().Generate + generate := transformers.NewGenerateLastName().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -345,11 +423,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateSha256HashConfig: config := transformerConfig.GetGenerateSha256HashConfig() - opts, err := NewGenerateSHA256HashOptsFromConfig(config) + opts, err := transformers.NewGenerateSHA256HashOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateSHA256Hash().Generate + generate := transformers.NewGenerateSHA256Hash().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -359,11 +437,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateSsnConfig: config := transformerConfig.GetGenerateSsnConfig() - opts, err := NewGenerateSSNOptsFromConfig(config) + opts, err := transformers.NewGenerateSSNOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateSSN().Generate + generate := transformers.NewGenerateSSN().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -373,11 +451,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStateConfig: config := transformerConfig.GetGenerateStateConfig() - opts, err := NewGenerateStateOptsFromConfig(config) + opts, err := transformers.NewGenerateStateOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateState().Generate + generate := transformers.NewGenerateState().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -387,11 +465,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStreetAddressConfig: config := transformerConfig.GetGenerateStreetAddressConfig() - opts, err := NewGenerateStreetAddressOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateStreetAddressOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateStreetAddress().Generate + generate := transformers.NewGenerateStreetAddress().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -401,11 +479,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringPhoneNumberConfig: config := transformerConfig.GetGenerateStringPhoneNumberConfig() - opts, err := NewGenerateStringPhoneNumberOptsFromConfig(config) + opts, err := transformers.NewGenerateStringPhoneNumberOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateStringPhoneNumber().Generate + generate := transformers.NewGenerateStringPhoneNumber().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -415,11 +493,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateStringConfig: config := transformerConfig.GetGenerateStringConfig() - opts, err := NewGenerateRandomStringOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateRandomStringOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateRandomString().Generate + generate := transformers.NewGenerateRandomString().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -429,11 +507,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateUnixtimestampConfig: config := transformerConfig.GetGenerateUnixtimestampConfig() - opts, err := NewGenerateUnixTimestampOptsFromConfig(config) + opts, err := transformers.NewGenerateUnixTimestampOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateUnixTimestamp().Generate + generate := transformers.NewGenerateUnixTimestamp().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -443,11 +521,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateUsernameConfig: config := transformerConfig.GetGenerateUsernameConfig() - opts, err := NewGenerateUsernameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateUsernameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateUsername().Generate + generate := transformers.NewGenerateUsername().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -457,11 +535,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateUtctimestampConfig: config := transformerConfig.GetGenerateUtctimestampConfig() - opts, err := NewGenerateUTCTimestampOptsFromConfig(config) + opts, err := transformers.NewGenerateUTCTimestampOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateUTCTimestamp().Generate + generate := transformers.NewGenerateUTCTimestamp().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -471,11 +549,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateUuidConfig: config := transformerConfig.GetGenerateUuidConfig() - opts, err := NewGenerateUUIDOptsFromConfig(config) + opts, err := transformers.NewGenerateUUIDOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateUUID().Generate + generate := transformers.NewGenerateUUID().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -485,11 +563,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateZipcodeConfig: config := transformerConfig.GetGenerateZipcodeConfig() - opts, err := NewGenerateZipcodeOptsFromConfig(config) + opts, err := transformers.NewGenerateZipcodeOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateZipcode().Generate + generate := transformers.NewGenerateZipcode().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -499,11 +577,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformE164PhoneNumberConfig: config := transformerConfig.GetTransformE164PhoneNumberConfig() - opts, err := NewTransformE164PhoneNumberOptsFromConfig(config, nil) + opts, err := transformers.NewTransformE164PhoneNumberOptsFromConfig(config, nil) if err != nil { return nil, err } - transform := NewTransformE164PhoneNumber().Transform + transform := transformers.NewTransformE164PhoneNumber().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -513,11 +591,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFirstNameConfig: config := transformerConfig.GetTransformFirstNameConfig() - opts, err := NewTransformFirstNameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewTransformFirstNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - transform := NewTransformFirstName().Transform + transform := transformers.NewTransformFirstName().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -527,11 +605,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformFloat64Config: config := transformerConfig.GetTransformFloat64Config() - opts, err := NewTransformFloat64OptsFromConfig(config, nil, nil) + opts, err := transformers.NewTransformFloat64OptsFromConfig(config, nil, nil) if err != nil { return nil, err } - transform := NewTransformFloat64().Transform + transform := transformers.NewTransformFloat64().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -541,11 +619,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformInt64PhoneNumberConfig: config := transformerConfig.GetTransformInt64PhoneNumberConfig() - opts, err := NewTransformInt64PhoneNumberOptsFromConfig(config) + opts, err := transformers.NewTransformInt64PhoneNumberOptsFromConfig(config) if err != nil { return nil, err } - transform := NewTransformInt64PhoneNumber().Transform + transform := transformers.NewTransformInt64PhoneNumber().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -555,11 +633,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformLastNameConfig: config := transformerConfig.GetTransformLastNameConfig() - opts, err := NewTransformLastNameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewTransformLastNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - transform := NewTransformLastName().Transform + transform := transformers.NewTransformLastName().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -569,11 +647,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformPhoneNumberConfig: config := transformerConfig.GetTransformPhoneNumberConfig() - opts, err := NewTransformStringPhoneNumberOptsFromConfig(config, &maxLength) + opts, err := transformers.NewTransformStringPhoneNumberOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - transform := NewTransformStringPhoneNumber().Transform + transform := transformers.NewTransformStringPhoneNumber().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -599,11 +677,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_TransformCharacterScrambleConfig: config := transformerConfig.GetTransformCharacterScrambleConfig() - opts, err := NewTransformCharacterScrambleOptsFromConfig(config) + opts, err := transformers.NewTransformCharacterScrambleOptsFromConfig(config) if err != nil { return nil, err } - transform := NewTransformCharacterScramble().Transform + transform := transformers.NewTransformCharacterScramble().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -613,11 +691,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateCountryConfig: config := transformerConfig.GetGenerateCountryConfig() - opts, err := NewGenerateCountryOptsFromConfig(config) + opts, err := transformers.NewGenerateCountryOptsFromConfig(config) if err != nil { return nil, err } - generate := NewGenerateCountry().Generate + generate := transformers.NewGenerateCountry().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -656,11 +734,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateBusinessNameConfig: config := transformerConfig.GetGenerateBusinessNameConfig() - opts, err := NewGenerateBusinessNameOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateBusinessNameOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateBusinessName().Generate + generate := transformers.NewGenerateBusinessName().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -670,11 +748,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform case *mgmtv1alpha1.TransformerConfig_GenerateIpAddressConfig: config := transformerConfig.GetGenerateIpAddressConfig() - opts, err := NewGenerateIpAddressOptsFromConfig(config, &maxLength) + opts, err := transformers.NewGenerateIpAddressOptsFromConfig(config, &maxLength) if err != nil { return nil, err } - generate := NewGenerateIpAddress().Generate + generate := transformers.NewGenerateIpAddress().Generate return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -683,11 +761,11 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil case *mgmtv1alpha1.TransformerConfig_TransformUuidConfig: config := transformerConfig.GetTransformUuidConfig() - opts, err := NewTransformUuidOptsFromConfig(config) + opts, err := transformers.NewTransformUuidOptsFromConfig(config) if err != nil { return nil, err } - transform := NewTransformUuid().Transform + transform := transformers.NewTransformUuid().Transform return &TransformerExecutor{ Opts: opts, Mutate: func(value any, opts any) (any, error) { @@ -696,6 +774,6 @@ func InitializeTransformerByConfigType(transformerConfig *mgmtv1alpha1.Transform }, nil default: - return nil, fmt.Errorf("unsupported transformer: %v", transformerConfig) + return nil, fmt.Errorf("unsupported transformerr: %T", typedCfg) } } diff --git a/worker/pkg/benthos/transformers/transformer_initializer_test.go b/worker/pkg/benthos/transformer_executor/executor_test.go similarity index 98% rename from worker/pkg/benthos/transformers/transformer_initializer_test.go rename to worker/pkg/benthos/transformer_executor/executor_test.go index 37cdbfdb8d..09eb140d6b 100644 --- a/worker/pkg/benthos/transformers/transformer_initializer_test.go +++ b/worker/pkg/benthos/transformer_executor/executor_test.go @@ -1,10 +1,11 @@ -package transformers +package transformer_executor import ( "strconv" "testing" "time" + "github.com/google/uuid" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" presidioapi "github.com/nucleuscloud/neosync/internal/ee/presidio" ee_transformer_fns "github.com/nucleuscloud/neosync/internal/ee/transformers/functions" @@ -37,6 +38,36 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { require.NoError(t, err) require.Equal(t, "A", result) }) + t.Run("TransformJavascriptConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_TransformJavascriptConfig{ + TransformJavascriptConfig: &mgmtv1alpha1.TransformJavascript{ + Code: "return value + ' world';", + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate("hello", nil) + require.NoError(t, err) + require.Equal(t, "hello world", result) + }) + t.Run("GenerateJavascriptConfig", func(t *testing.T) { + config := &mgmtv1alpha1.TransformerConfig{ + Config: &mgmtv1alpha1.TransformerConfig_GenerateJavascriptConfig{ + GenerateJavascriptConfig: &mgmtv1alpha1.GenerateJavascript{ + Code: "return 'hello world';", + }, + }, + } + executor, err := InitializeTransformerByConfigType(config) + require.NoError(t, err) + require.NotNil(t, executor) + result, err := executor.Mutate(nil, nil) + require.NoError(t, err) + require.Equal(t, "hello world", result) + }) t.Run("PassthroughConfig", func(t *testing.T) { config := &mgmtv1alpha1.TransformerConfig{ Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}, @@ -1713,7 +1744,7 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { executor, err := InitializeTransformerByConfigType(config) require.NoError(t, err) require.NotNil(t, executor) - originalValue := generateUuid(true) + originalValue := uuid.NewString() result, err := executor.Mutate(originalValue, executor.Opts) require.NoError(t, err) require.NotEqual(t, originalValue, result) @@ -1727,7 +1758,7 @@ func Test_InitializeTransformerByConfigType(t *testing.T) { executor, err := InitializeTransformerByConfigType(config) require.NoError(t, err) require.NotNil(t, executor) - originalValue := generateUuid(true) + originalValue := uuid.NewString() result, err := executor.Mutate(originalValue, executor.Opts) require.NoError(t, err) require.NotEqual(t, originalValue, result) diff --git a/worker/pkg/benthos/transformer_executor/mock_UserDefinedTransformerResolver.go b/worker/pkg/benthos/transformer_executor/mock_UserDefinedTransformerResolver.go new file mode 100644 index 0000000000..95dcfee154 --- /dev/null +++ b/worker/pkg/benthos/transformer_executor/mock_UserDefinedTransformerResolver.go @@ -0,0 +1,96 @@ +// Code generated by mockery. DO NOT EDIT. + +package transformer_executor + +import ( + context "context" + + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" + mock "github.com/stretchr/testify/mock" +) + +// MockUserDefinedTransformerResolver is an autogenerated mock type for the UserDefinedTransformerResolver type +type MockUserDefinedTransformerResolver struct { + mock.Mock +} + +type MockUserDefinedTransformerResolver_Expecter struct { + mock *mock.Mock +} + +func (_m *MockUserDefinedTransformerResolver) EXPECT() *MockUserDefinedTransformerResolver_Expecter { + return &MockUserDefinedTransformerResolver_Expecter{mock: &_m.Mock} +} + +// GetUserDefinedTransformer provides a mock function with given fields: ctx, id +func (_m *MockUserDefinedTransformerResolver) GetUserDefinedTransformer(ctx context.Context, id string) (*mgmtv1alpha1.TransformerConfig, error) { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for GetUserDefinedTransformer") + } + + var r0 *mgmtv1alpha1.TransformerConfig + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*mgmtv1alpha1.TransformerConfig, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *mgmtv1alpha1.TransformerConfig); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*mgmtv1alpha1.TransformerConfig) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetUserDefinedTransformer' +type MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call struct { + *mock.Call +} + +// GetUserDefinedTransformer is a helper method to define mock.On call +// - ctx context.Context +// - id string +func (_e *MockUserDefinedTransformerResolver_Expecter) GetUserDefinedTransformer(ctx interface{}, id interface{}) *MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call { + return &MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call{Call: _e.mock.On("GetUserDefinedTransformer", ctx, id)} +} + +func (_c *MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call) Run(run func(ctx context.Context, id string)) *MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call) Return(_a0 *mgmtv1alpha1.TransformerConfig, _a1 error) *MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call) RunAndReturn(run func(context.Context, string) (*mgmtv1alpha1.TransformerConfig, error)) *MockUserDefinedTransformerResolver_GetUserDefinedTransformer_Call { + _c.Call.Return(run) + return _c +} + +// NewMockUserDefinedTransformerResolver creates a new instance of MockUserDefinedTransformerResolver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockUserDefinedTransformerResolver(t interface { + mock.TestingT + Cleanup(func()) +}) *MockUserDefinedTransformerResolver { + mock := &MockUserDefinedTransformerResolver{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +}