diff --git a/cmd/dev/app/datasource/datasource.go b/cmd/dev/app/datasource/datasource.go new file mode 100644 index 0000000000..0c8a7c0d55 --- /dev/null +++ b/cmd/dev/app/datasource/datasource.go @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package datasource provides the root command for the datasource subcommands +package datasource + +import "github.com/spf13/cobra" + +// CmdDataSource is the root command for the datasource subcommands +func CmdDataSource() *cobra.Command { + var rtCmd = &cobra.Command{ + Use: "datasource", + Short: "datasource provides utilities for testing and working with data sources", + } + + rtCmd.AddCommand(CmdGenerate()) + + return rtCmd +} diff --git a/cmd/dev/app/datasource/generate.go b/cmd/dev/app/datasource/generate.go new file mode 100644 index 0000000000..8f23e92370 --- /dev/null +++ b/cmd/dev/app/datasource/generate.go @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +package datasource + +import ( + "fmt" + "net/http" + "net/url" + "os" + "regexp" + "strings" + + "buf.build/go/protoyaml" + "github.com/go-openapi/loads" + "github.com/go-openapi/spec" + "github.com/spf13/cobra" + "google.golang.org/protobuf/types/known/structpb" + + minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1" +) + +// CmdGenerate returns a cobra command for the 'datasource generate' subcommand. +func CmdGenerate() *cobra.Command { + var generateCmd = &cobra.Command{ + Use: "generate", + Aliases: []string{"gen"}, + Short: "generate datasource code from an OpenAPI specification", + Long: `The 'datasource generate' subcommand allows you to generate datasource code from an OpenAPI +specification`, + RunE: generateCmdRun, + Args: cobra.ExactArgs(1), + SilenceUsage: true, + } + + return generateCmd +} + +// parseOpenAPI parses an OpenAPI specification from a byte slice. +func parseOpenAPI(filepath string) (*spec.Swagger, error) { + doc, err := loads.Spec(filepath) + if err != nil { + return nil, fmt.Errorf("failed to parse OpenAPI spec: %w", err) + } + + return doc.Spec(), nil +} + +func initDataSourceStruct(name string) *minderv1.DataSource { + return &minderv1.DataSource{ + Version: minderv1.VersionV1, + Type: "data-source", + Name: name, + Context: &minderv1.ContextV2{}, + } +} + +func initDriverStruct() *minderv1.RestDataSource { + return &minderv1.RestDataSource{ + Def: make(map[string]*minderv1.RestDataSource_Def), + } +} + +// conver the title to a valid datasource name. It should only contain alphanumeric characters and dashes. +func swaggerTitleToDataSourceName(title string) string { + re := regexp.MustCompile("[^a-zA-Z0-9-]+") + return re.ReplaceAllString(title, "-") +} + +// swaggerToDataSource generates datasource code from an OpenAPI specification. +func swaggerToDataSource(cmd *cobra.Command, swagger *spec.Swagger) error { + if swagger.Info == nil { + return fmt.Errorf("info section is required in OpenAPI spec") + } + + ds := initDataSourceStruct(swaggerTitleToDataSourceName(swagger.Info.Title)) + drv := initDriverStruct() + ds.Driver = &minderv1.DataSource_Rest{Rest: drv} + + // Add the OpenAPI specification to the DataSource + basepath := swagger.BasePath + if basepath == "" { + return fmt.Errorf("base path is required in OpenAPI spec") + } + + for path, pathItem := range swagger.Paths.Paths { + p, err := url.JoinPath(basepath, path) + if err != nil { + cmd.PrintErrf("error joining path %s and basepath %s: %v\n Skipping", path, basepath, err) + continue + } + + for method, op := range operations(pathItem) { + opName := generateOpName(method, path) + // Create a new REST DataSource definition + def := &minderv1.RestDataSource_Def{ + Method: method, + Endpoint: p, + // TODO: Make this configurable + Parse: "json", + } + + is := paramsToInputSchema(op.Parameters) + + if requiresMsgBody(method) { + def.Body = &minderv1.RestDataSource_Def_BodyFromField{ + BodyFromField: "body", + } + + // Add the `body` field to the input schema + is = inputSchemaForBody(is) + } + + pbs, err := structpb.NewStruct(is) + if err != nil { + return fmt.Errorf("error creating input schema: %w", err) + } + + def.InputSchema = pbs + + // Add the operation to the DataSource + drv.Def[opName] = def + } + } + + return writeDataSourceToFile(ds) +} + +// Generates an operation name for a data source. Note that these names +// must be unique within a data source. They also should be only alphanumeric +// characters and underscores +func generateOpName(method, path string) string { + // Replace all non-alphanumeric characters with underscores + re := regexp.MustCompile("[^a-zA-Z0-9]+") + return re.ReplaceAllString(fmt.Sprintf("%s_%s", strings.ToLower(method), strings.ToLower(path)), "_") +} + +func operations(p spec.PathItem) map[string]*spec.Operation { + out := make(map[string]*spec.Operation) + for mstr, op := range map[string]*spec.Operation{ + http.MethodGet: p.Get, + http.MethodPut: p.Put, + http.MethodPost: p.Post, + http.MethodDelete: p.Delete, + http.MethodOptions: p.Options, + http.MethodHead: p.Head, + http.MethodPatch: p.Patch, + } { + if op != nil { + out[mstr] = op + } + } + + return out +} + +func requiresMsgBody(method string) bool { + return method == http.MethodPost || method == http.MethodPut || method == http.MethodPatch +} + +func paramsToInputSchema(params []spec.Parameter) map[string]any { + if len(params) == 0 { + return nil + } + + is := map[string]any{ + "type": "object", + "properties": make(map[string]any), + } + + for _, p := range params { + is["properties"].(map[string]any)[p.Name] = map[string]any{ + // TODO: Add support for more types + "type": "string", + } + + if p.Required { + if _, ok := is["required"]; !ok { + is["required"] = make([]string, 0) + } + + is["required"] = append(is["required"].([]string), p.Name) + } + } + + return is +} + +func inputSchemaForBody(is map[string]any) map[string]any { + if is == nil { + is = map[string]any{ + "type": "object", + "properties": make(map[string]any), + } + } + + is["properties"].(map[string]any)["body"] = map[string]any{ + "type": "object", + } + + return is +} + +func writeDataSourceToFile(ds *minderv1.DataSource) error { + // Convert the DataSource to YAML + dsYAML, err := protoyaml.MarshalOptions{ + Indent: 2, + }.Marshal(ds) + if err != nil { + return fmt.Errorf("error marshalling DataSource to YAML: %w", err) + } + + // Write the YAML to a file + if _, err := os.Stdout.Write(dsYAML); err != nil { + return fmt.Errorf("error writing DataSource to file: %w", err) + } + + return nil +} + +// generateCmdRun is the entry point for the 'datasource generate' command. +func generateCmdRun(cmd *cobra.Command, args []string) error { + // We've already validated that there is exactly one argument via the cobra.ExactArgs(1) call + filePath := args[0] + + // Parse the OpenAPI specification + swagger, err := parseOpenAPI(filePath) + if err != nil { + return fmt.Errorf("error parsing OpenAPI spec: %w", err) + } + + return swaggerToDataSource(cmd, swagger) +} diff --git a/cmd/dev/app/root.go b/cmd/dev/app/root.go index c6ccabfe05..1875f41ddb 100644 --- a/cmd/dev/app/root.go +++ b/cmd/dev/app/root.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "github.com/mindersec/minder/cmd/dev/app/bundles" + "github.com/mindersec/minder/cmd/dev/app/datasource" "github.com/mindersec/minder/cmd/dev/app/image" "github.com/mindersec/minder/cmd/dev/app/rule_type" "github.com/mindersec/minder/cmd/dev/app/testserver" @@ -29,6 +30,7 @@ https://docs.stacklok.com/minder`, cmd.AddCommand(image.CmdImage()) cmd.AddCommand(testserver.CmdTestServer()) cmd.AddCommand(bundles.CmdBundle()) + cmd.AddCommand(datasource.CmdDataSource()) return cmd } diff --git a/go.mod b/go.mod index 6ed96e963a..3c2eed4651 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.23.4 require ( buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1 + buf.build/go/protoyaml v0.3.1 github.com/ThreeDotsLabs/watermill v1.4.2 github.com/ThreeDotsLabs/watermill-sql/v3 v3.1.0 github.com/alexdrl/zerowater v0.0.3 @@ -308,7 +309,7 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/loads v0.22.0 // indirect github.com/go-openapi/runtime v0.28.0 // indirect - github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/spec v0.21.0 github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-openapi/validate v0.24.0 // indirect diff --git a/go.sum b/go.sum index c378296957..acdf27a8e9 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1 h1:BICM6du/XzvEgeorNo4xgohK3nMTmEPViGyd5t7xVqk= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1/go.mod h1:JnMVLi3qrNYPODVpEKG7UjHLl/d2zR221e66YCSmP2Q= +buf.build/go/protoyaml v0.3.1 h1:ucyzE7DRnjX+mQ6AH4JzN0Kg50ByHHu+yrSKbgQn2D4= +buf.build/go/protoyaml v0.3.1/go.mod h1:0TzNpFQDXhwbkXb/ajLvxIijqbve+vMQvWY/b3/Dzxg= cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=