Skip to content

Commit

Permalink
Deprecates Source in JobMappingTransformer config (#2869)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nickzelei
nickzelei authored Oct 28, 2024
1 parent 60f4f14 commit 208fe2b
Show file tree
Hide file tree
Showing 34 changed files with 490 additions and 470 deletions.
2 changes: 1 addition & 1 deletion backend/gen/go/db/models.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions backend/gen/go/db/transformers.sql.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions backend/gen/go/protos/mgmt/v1alpha1/job.pb.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions backend/protos/mgmt/v1alpha1/job.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,7 @@ message CreateJobResponse {
}

message JobMappingTransformer {
// @deprecated - This is no longer used in favor just providing the TransformerConfig
TransformerSource source = 1;
TransformerConfig config = 3;
}
Expand Down
93 changes: 31 additions & 62 deletions backend/services/mgmt/v1alpha1/job-service/jobs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,12 +403,10 @@ func Test_CreateJob(t *testing.T) {
},
Mappings: []*pg_models.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
{Schema: "schema-2", Table: "table-2", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
},
VirtualForeignKeys: []*pg_models.VirtualForeignConstraint{},
Expand Down Expand Up @@ -462,12 +460,10 @@ func Test_CreateJob(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "schema-2", Table: "table-2", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
WorkflowOptions: &mgmtv1alpha1.WorkflowOptions{},
Expand Down Expand Up @@ -548,12 +544,10 @@ func Test_CreateJob_Schedule_Creation_Error(t *testing.T) {
},
Mappings: []*pg_models.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
{Schema: "schema-2", Table: "table-2", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
},
VirtualForeignKeys: []*pg_models.VirtualForeignConstraint{},
Expand Down Expand Up @@ -609,12 +603,10 @@ func Test_CreateJob_Schedule_Creation_Error(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "schema-2", Table: "table-2", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
WorkflowOptions: &mgmtv1alpha1.WorkflowOptions{},
Expand Down Expand Up @@ -697,12 +689,10 @@ func Test_CreateJob_Schedule_Creation_Error_JobCleanup_Error(t *testing.T) {
},
Mappings: []*pg_models.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
{Schema: "schema-2", Table: "table-2", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
},
VirtualForeignKeys: []*pg_models.VirtualForeignConstraint{},
Expand Down Expand Up @@ -758,12 +748,10 @@ func Test_CreateJob_Schedule_Creation_Error_JobCleanup_Error(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "schema-2", Table: "table-2", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
WorkflowOptions: &mgmtv1alpha1.WorkflowOptions{},
Expand Down Expand Up @@ -1001,8 +989,7 @@ func Test_UpdateJobSourceConnection_Success(t *testing.T) {
ID: job.ID,
Mappings: []*pg_models.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
},
UpdatedByID: userUuid,
Expand Down Expand Up @@ -1047,8 +1034,7 @@ func Test_UpdateJobSourceConnection_Success(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
},
Expand Down Expand Up @@ -1089,8 +1075,7 @@ func Test_UpdateJobSourceConnection_GenerateSuccess(t *testing.T) {
ID: job.ID,
Mappings: []*pg_models.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", JobMappingTransformer: &pg_models.JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &pg_models.TransformerConfigs{},
Config: &pg_models.TransformerConfig{Passthrough: &pg_models.PassthroughConfig{}},
}},
},
UpdatedByID: userUuid,
Expand Down Expand Up @@ -1133,8 +1118,7 @@ func Test_UpdateJobSourceConnection_GenerateSuccess(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
},
Expand Down Expand Up @@ -1190,8 +1174,7 @@ func Test_UpdateJobSourceConnection_PgMismatchError(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
},
Expand Down Expand Up @@ -1248,8 +1231,7 @@ func Test_UpdateJobSourceConnection_MysqlMismatchError(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
},
Expand Down Expand Up @@ -1306,8 +1288,7 @@ func Test_UpdateJobSourceConnection_AwsS3MismatchError(t *testing.T) {
},
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "schema-1", Table: "table-1", Column: "col", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
},
Expand Down Expand Up @@ -1829,28 +1810,22 @@ func Test_ValidateJobMappings_NoValidationErrors(t *testing.T) {
ConnectionId: connId,
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "public", Table: "orders", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "public", Table: "users", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_1", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_1", Column: "table2_id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_2", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_2", Column: "table1_id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
},
Expand Down Expand Up @@ -1904,28 +1879,22 @@ func Test_ValidateJobMappings_ValidationErrors(t *testing.T) {
ConnectionId: connId,
Mappings: []*mgmtv1alpha1.JobMapping{
{Schema: "public", Table: "orders", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "account", Table: "accounts", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_1", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_1", Column: "table2_id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_2", Column: "id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
{Schema: "circle", Table: "table_2", Column: "table1_id", Transformer: &mgmtv1alpha1.JobMappingTransformer{
Source: mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH,
Config: &mgmtv1alpha1.TransformerConfig{},
Config: &mgmtv1alpha1.TransformerConfig{Config: &mgmtv1alpha1.TransformerConfig_PassthroughConfig{}},
}},
},
},
Expand Down
4 changes: 2 additions & 2 deletions backend/services/mgmt/v1alpha1/transformers-service/userdefined_transformers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func (s *Service) CreateUserDefinedTransformer(ctx context.Context, req *connect
AccountID: *accountUuid,
Name: req.Msg.Name,
Description: req.Msg.Description,
TransformerConfig: &pg_models.TransformerConfigs{},
TransformerConfig: &pg_models.TransformerConfig{},
Source: int32(req.Msg.Source),
CreatedByID: *userUuid,
UpdatedByID: *userUuid,
Expand Down Expand Up @@ -173,7 +173,7 @@ func (s *Service) UpdateUserDefinedTransformer(ctx context.Context, req *connect
updateParams := &db_queries.UpdateUserDefinedTransformerParams{
Name: req.Msg.Name,
Description: req.Msg.Description,
TransformerConfig: &pg_models.TransformerConfigs{},
TransformerConfig: &pg_models.TransformerConfig{},
UpdatedByID: *userUuid,
ID: tUuid,
}
Expand Down
12 changes: 4 additions & 8 deletions backend/sql/postgresql/models/models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -942,26 +942,22 @@ func (s *DynamoDBSourceOptions) ToDto() *mgmtv1alpha1.DynamoDBSourceConnectionOp
if s.UnmappedTransforms == nil {
s.UnmappedTransforms = &DynamoDBSourceUnmappedTransformConfig{
B: &JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &TransformerConfigs{
Config: &TransformerConfig{
Passthrough: &PassthroughConfig{},
},
},
Boolean: &JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_BOOL),
Config: &TransformerConfigs{
Config: &TransformerConfig{
GenerateBool: &GenerateBoolConfig{},
},
},
N: &JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_PASSTHROUGH),
Config: &TransformerConfigs{
Config: &TransformerConfig{
Passthrough: &PassthroughConfig{},
},
},
S: &JobMappingTransformerModel{
Source: int32(mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_GENERATE_RANDOM_STRING),
Config: &TransformerConfigs{
Config: &TransformerConfig{
GenerateString: &GenerateStringConfig{},
},
},
Expand Down
32 changes: 10 additions & 22 deletions backend/sql/postgresql/models/transformers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ import (
)

type JobMappingTransformerModel struct {
Source int32 `json:"source"`
Config *TransformerConfigs `json:"config,omitempty"`
Config *TransformerConfig `json:"config,omitempty"`
}

type TransformerConfigs struct {
type TransformerConfig struct {
GenerateEmail *GenerateEmailConfig `json:"generateEmailConfig,omitempty"`
TransformEmail *TransformEmailConfig `json:"transformEmail,omitempty"`
GenerateBool *GenerateBoolConfig `json:"generateBool,omitempty"`
Expand Down Expand Up @@ -205,22 +204,20 @@ type GenerateCountryConfig struct {
GenerateFullName *bool `json:"generateFullName,omitempty"`
}

// from API -> DB
func (t *JobMappingTransformerModel) FromTransformerDto(tr *mgmtv1alpha1.JobMappingTransformer) error {
t.Source = int32(tr.Source)

config := &TransformerConfigs{}
if tr == nil {
tr = &mgmtv1alpha1.JobMappingTransformer{}
}

config := &TransformerConfig{}
if err := config.FromTransformerConfigDto(tr.GetConfig()); err != nil {
return err
}

t.Config = config

return nil
}

func (t *TransformerConfigs) FromTransformerConfigDto(tr *mgmtv1alpha1.TransformerConfig) error {
func (t *TransformerConfig) FromTransformerConfigDto(tr *mgmtv1alpha1.TransformerConfig) error {
if tr == nil {
tr = &mgmtv1alpha1.TransformerConfig{}
}
Expand Down Expand Up @@ -378,31 +375,22 @@ func (t *TransformerConfigs) FromTransformerConfigDto(tr *mgmtv1alpha1.Transform
GenerateFullName: tr.GetGenerateCountryConfig().GenerateFullName,
}
default:
t = &TransformerConfigs{}
t = &TransformerConfig{}
}

return nil
}

// DB -> API
func (t *JobMappingTransformerModel) ToTransformerDto() *mgmtv1alpha1.JobMappingTransformer {
_, ok := mgmtv1alpha1.TransformerSource_name[t.Source]
var source mgmtv1alpha1.TransformerSource
if !ok {
source = mgmtv1alpha1.TransformerSource_TRANSFORMER_SOURCE_UNSPECIFIED
} else {
source = mgmtv1alpha1.TransformerSource(t.Source)
}
if t.Config == nil {
t.Config = &TransformerConfigs{}
t.Config = &TransformerConfig{}
}
return &mgmtv1alpha1.JobMappingTransformer{
Source: source,
Config: t.Config.ToTransformerConfigDto(),
}
}

func (t *TransformerConfigs) ToTransformerConfigDto() *mgmtv1alpha1.TransformerConfig {
func (t *TransformerConfig) ToTransformerConfigDto() *mgmtv1alpha1.TransformerConfig {
switch {
case t.GenerateEmail != nil:
return &mgmtv1alpha1.TransformerConfig{
Expand Down
Loading

0 comments on commit 208fe2b

Please sign in to comment.