From 06b9a8d4300c5bd55b7d8ab60f340bf2f59f669e Mon Sep 17 00:00:00 2001 From: Chris Seto Date: Fri, 4 Oct 2024 17:41:01 -0400 Subject: [PATCH] gotohelm: subcharting PoC This is not complete by any means. Just wanted to see where blockers where, if any. Was delighted to discover that `.Subcharts` did 90% of the hard work for us and very few changes to gotohelm would be required. --- Taskfile.yaml | 2 +- charts/redpanda/console.tpl.go | 52 ++++++- charts/redpanda/templates/_console.go.tpl | 18 ++- .../console/configmap-and-deployment.yaml | 33 +---- charts/redpanda/values.go | 4 + cmd/gotohelm/main.go | 13 +- pkg/gotohelm/helmette/helm.go | 7 +- pkg/gotohelm/transpiler.go | 130 ++++++++++-------- 8 files changed, 160 insertions(+), 99 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 42dc7a7b75..42b2e90ed3 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -112,7 +112,7 @@ tasks: # Generate the values JSON schema from the Values struct - go run ./cmd/genschema redpanda > charts/redpanda/values.schema.json # Generate helm templates from Go definitions - - go run ./cmd/gotohelm -write ./charts/redpanda/templates ./charts/redpanda + - go run ./cmd/gotohelm -write ./charts/redpanda/templates ./charts/redpanda ./charts/... chart:generate:connectors: desc: "Generate files for the connectors Helm chart" diff --git a/charts/redpanda/console.tpl.go b/charts/redpanda/console.tpl.go index 910ea8f3f3..0d36dc63fb 100644 --- a/charts/redpanda/console.tpl.go +++ b/charts/redpanda/console.tpl.go @@ -20,9 +20,59 @@ import ( "fmt" "github.com/redpanda-data/console/backend/pkg/config" + "github.com/redpanda-data/helm-charts/charts/connectors" + "github.com/redpanda-data/helm-charts/charts/console" "github.com/redpanda-data/helm-charts/pkg/gotohelm/helmette" ) +func ConsoleIntegration(dot *helmette.Dot) []any { + values := helmette.Unwrap[Values](dot.Values) + + var manifests []any + + // {{/* Secret */}} + // {{ $secretConfig := dict ( dict + // "create" $.Values.console.secret.create + // ) + // }} + // {{ if and .Values.console.enabled (not .Values.console.secret.create) }} + // {{ $licenseKey := ( include "enterprise-license" . ) }} + // # before license changes, this was not printing a secret, so we gather in which case to print + // # for now only if we have a license do we print, however, this may be an issue for some + // # since if we do include a license we MUST also print all secret items. + // {{ if ( not (empty $licenseKey ) ) }} + // {{/* License and license are set twice here as a work around to a bug in the post-go console chart. */}} + // {{ $secretConfig = ( dict + // "create" true + // "enterprise" ( dict "license" $licenseKey "License" $licenseKey) + // ) + // }} + // + // {{ $config := dict + // "Values" (dict + // "secret" $secretConfig + // )}} + + // if the console chart has the creation of the secret disabled, create it here instead if needed + if values.Console.Enabled && !values.Console.Secret.Create { + consoleDot := helmette.MergeTo[*helmette.Dot]( + dot.Subcharts["console"], + map[string]any{ + "Values": map[string]any{ + "secret": map[string]any{ + "create": true, + // TODO enterprise license. + }, + }, + }, + ) + + manifests = append(manifests, console.Secret(consoleDot)) + } + + return manifests +} + func ConsoleConfig(dot *helmette.Dot) any { values := helmette.Unwrap[Values](dot.Values) @@ -79,7 +129,7 @@ func ConsoleConfig(dot *helmette.Dot) any { } connectorsURL := fmt.Sprintf("http://%s.%s.svc.%s:%d", - ConnectorsFullName(dot), + connectors.Fullname(dot.Subcharts["connectors"]), dot.Release.Namespace, helmette.TrimSuffix(".", values.ClusterDomain), p) diff --git a/charts/redpanda/templates/_console.go.tpl b/charts/redpanda/templates/_console.go.tpl index f8498e9986..9bae229f86 100644 --- a/charts/redpanda/templates/_console.go.tpl +++ b/charts/redpanda/templates/_console.go.tpl @@ -1,5 +1,21 @@ {{- /* Generated from "console.tpl.go" */ -}} +{{- define "redpanda.ConsoleIntegration" -}} +{{- $dot := (index .a 0) -}} +{{- range $_ := (list 1) -}} +{{- $_is_returning := false -}} +{{- $values := $dot.Values.AsMap -}} +{{- $manifests := (coalesce nil) -}} +{{- if (and $values.console.enabled (not $values.console.secret.create)) -}} +{{- $consoleDot := (merge (dict ) (index $dot.Subcharts "console") (dict "Values" (dict "secret" (dict "create" true ) ) )) -}} +{{- $manifests = (concat (default (list ) $manifests) (list (get (fromJson (include "console.Secret" (dict "a" (list $consoleDot) ))) "r"))) -}} +{{- end -}} +{{- $_is_returning = true -}} +{{- (dict "r" $manifests) | toJson -}} +{{- break -}} +{{- end -}} +{{- end -}} + {{- define "redpanda.ConsoleConfig" -}} {{- $dot := (index .a 0) -}} {{- range $_ := (list 1) -}} @@ -33,7 +49,7 @@ {{- (dict "r" $c) | toJson -}} {{- break -}} {{- end -}} -{{- $connectorsURL := (printf "http://%s.%s.svc.%s:%d" (get (fromJson (include "redpanda.ConnectorsFullName" (dict "a" (list $dot) ))) "r") $dot.Release.Namespace (trimSuffix "." $values.clusterDomain) $p) -}} +{{- $connectorsURL := (printf "http://%s.%s.svc.%s:%d" (get (fromJson (include "connectors.Fullname" (dict "a" (list (index $dot.Subcharts "connectors")) ))) "r") $dot.Release.Namespace (trimSuffix "." $values.clusterDomain) $p) -}} {{- $_ := (set $c "connect" (mustMergeOverwrite (dict "enabled" false "clusters" (coalesce nil) "connectTimeout" 0 "readTimeout" 0 "requestTimeout" 0 ) (dict "enabled" $values.connectors.enabled "clusters" (list (mustMergeOverwrite (dict "name" "" "url" "" "tls" (dict "enabled" false "caFilepath" "" "certFilepath" "" "keyFilepath" "" "insecureSkipTlsVerify" false ) "username" "" "password" "" "token" "" ) (dict "name" "connectors" "url" $connectorsURL "tls" (mustMergeOverwrite (dict "enabled" false "caFilepath" "" "certFilepath" "" "keyFilepath" "" "insecureSkipTlsVerify" false ) (dict "enabled" false "caFilepath" "" "certFilepath" "" "keyFilepath" "" "insecureSkipTlsVerify" false )) "username" "" "password" "" "token" "" ))) ))) -}} {{- end -}} {{- $_is_returning = true -}} diff --git a/charts/redpanda/templates/console/configmap-and-deployment.yaml b/charts/redpanda/templates/console/configmap-and-deployment.yaml index a03229aae2..552ff34d17 100644 --- a/charts/redpanda/templates/console/configmap-and-deployment.yaml +++ b/charts/redpanda/templates/console/configmap-and-deployment.yaml @@ -15,36 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{/* Secret */}} -{{ $secretConfig := dict ( dict - "create" $.Values.console.secret.create - ) -}} -{{/* if the console chart has the creation of the secret disabled, create it here instead if needed */}} -{{ if and .Values.console.enabled (not .Values.console.secret.create) }} -{{ $licenseKey := ( include "enterprise-license" . ) }} -# before license changes, this was not printing a secret, so we gather in which case to print -# for now only if we have a license do we print, however, this may be an issue for some -# since if we do include a license we MUST also print all secret items. - {{ if ( not (empty $licenseKey ) ) }} -{{/* License and license are set twice here as a work around to a bug in the post-go console chart. */}} -{{ $secretConfig = ( dict - "create" true - "enterprise" ( dict "license" $licenseKey "License" $licenseKey) - ) -}} - -{{ $config := dict - "Values" (dict - "secret" $secretConfig - )}} - -{{ $secretValues := merge $config .Subcharts.console }} -{{ $wrappedSecretValues := (dict "Chart" .Subcharts.console.Chart "Release" .Release "Values" (dict "AsMap" $secretValues.Values)) }} ---- -{{- include "_shims.render-manifest" (list "console.Secret" $wrappedSecretValues) -}} - {{ end }} -{{ end }} +{{- include "_shims.render-manifest" (list "redpanda.ConsoleIntegration" .) -}} {{ $configmap := dict }} {{/* if the console chart has the creation of the configmap disabled, create it here instead */}} @@ -57,7 +28,6 @@ limitations under the License. "Values" (dict "console" (dict "config" $consoleConfig) "configmap" $consoleConfigmap - "secret" $secretConfig ) }} @@ -192,7 +162,6 @@ limitations under the License. "extraVolumes" $extraVolumes "extraVolumeMounts" $extraVolumeMounts "extraEnv" $extraEnv - "secret" $secretConfig "enterprise" $enterprise "image" $.Values.console.image "autoscaling" .Values.console.autoscaling diff --git a/charts/redpanda/values.go b/charts/redpanda/values.go index b3ef14eaf0..cd400dbab5 100644 --- a/charts/redpanda/values.go +++ b/charts/redpanda/values.go @@ -87,6 +87,10 @@ type Console struct { Console struct { Config map[string]any `json:"config"` } `json:"console"` + + Secret struct { + Create bool `json:"create"` + } `json:"secret"` } type Connectors struct { diff --git a/cmd/gotohelm/main.go b/cmd/gotohelm/main.go index 2e0db2f6e2..6a3b904b14 100644 --- a/cmd/gotohelm/main.go +++ b/cmd/gotohelm/main.go @@ -19,13 +19,22 @@ func main() { pkgs, err := gotohelm.LoadPackages(&packages.Config{ Dir: cwd, - }, flag.Args()...) + }, flag.Args()[0]) + if err != nil { + panic(err) + } + + // falling back to golist might be better here? + // Might make the mapping consistent? + deps, err := gotohelm.LoadPackages(&packages.Config{ + Dir: cwd, + }, flag.Args()[1:]...) if err != nil { panic(err) } for _, pkg := range pkgs { - chart, err := gotohelm.Transpile(pkg) + chart, err := gotohelm.Transpile(pkg, deps...) if err != nil { fmt.Printf("Failed to transpile %q: %s\n", pkg.Name, err) continue diff --git a/pkg/gotohelm/helmette/helm.go b/pkg/gotohelm/helmette/helm.go index 6e7da25fe2..becbd11597 100644 --- a/pkg/gotohelm/helmette/helm.go +++ b/pkg/gotohelm/helmette/helm.go @@ -13,9 +13,10 @@ import ( // of a helm template. // See also: https://github.com/helm/helm/blob/3764b483b385a12e7d3765bff38eced840362049/pkg/chartutil/values.go#L137-L166 type Dot struct { - Values Values - Release Release - Chart Chart + Values Values + Release Release + Chart Chart + Subcharts map[string]*Dot // Capabilities // KubeConfig is a hacked in value to allow `Lookup` to not rely on global diff --git a/pkg/gotohelm/transpiler.go b/pkg/gotohelm/transpiler.go index 8271933872..06bc01f146 100644 --- a/pkg/gotohelm/transpiler.go +++ b/pkg/gotohelm/transpiler.go @@ -45,7 +45,7 @@ type Chart struct { Files []*File } -func Transpile(pkg *packages.Package) (_ *Chart, err error) { +func Transpile(pkg *packages.Package, deps ...*packages.Package) (_ *Chart, err error) { defer func() { switch v := recover().(type) { case nil: @@ -56,15 +56,21 @@ func Transpile(pkg *packages.Package) (_ *Chart, err error) { } }() + dependencies := map[string]struct{}{} + for _, pkg := range append(deps, pkg) { + dependencies[pkg.PkgPath] = struct{}{} + } + t := &Transpiler{ Package: pkg, Fset: pkg.Fset, TypesInfo: pkg.TypesInfo, Files: pkg.Syntax, - packages: mkPkgTree(pkg), - namespaces: map[*types.Package]string{}, - names: map[*types.Func]string{}, + packages: mkPkgTree(pkg), + namespaces: map[*types.Package]string{}, + dependencies: dependencies, + names: map[*types.Func]string{}, builtins: map[string]string{ "fmt.Sprintf": "printf", "golang.org/x/exp/maps.Keys": "keys", @@ -91,6 +97,8 @@ type Transpiler struct { // their builtin equivalent. builtins map[string]string packages map[string]*packages.Package + // todo + dependencies map[string]struct{} // namespaces is a cache for holding the namespace package directive. It's // exclusively used by `namespaceFor`. namespaces map[*types.Package]string @@ -981,60 +989,7 @@ func (t *Transpiler) transpileCallExpr(n *ast.CallExpr) Node { }) } - // Call to function within the same package. A-Okay. It's transpiled. NB: - // This is intentionally after the builtins check to support our bootstrap - // package's builtin bindings. - if callee.Pkg().Path() == t.Package.PkgPath { - var call Node - - // Method call. - if r := callee.Type().(*types.Signature).Recv(); r != nil { - typ := r.Type() - - mutable := false - switch t := typ.(type) { - case *types.Pointer: - typ = t.Elem() - mutable = true - } - - if _, ok := typ.(*types.Named); !ok { - panic(&Unsupported{Fset: t.Fset, Node: n, Msg: "method calls with not pointer type with named type"}) - } - var receiverArg Node - - // When receiver is a pointer then dictionary can be passed as is. - // When receiver is not a pointer then dictionary is a deep copied. - receiverArg = &BuiltInCall{FuncName: "deepCopy", Arguments: []Node{t.transpileExpr(n.Fun.(*ast.SelectorExpr).X)}} - if mutable { - receiverArg = t.transpileExpr(n.Fun.(*ast.SelectorExpr).X) - } - - call = &Call{ - FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))), - // Method calls come in as a "top level" CallExpr where .Fun is the - // selector up to that call. e.g. `Foo.Bar.Baz()` will be a `CallExpr`. - // It's `.Fun` is a `SelectorExpr` where `.X` is `Foo.Bar`, the receiver, - // and `.Sel` is `Baz`, the method name. - Arguments: append([]Node{receiverArg}, args...), - } - } else { - call = &Call{FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))), Arguments: args} - } - - // If there's only a single return value, we'll possibly want to wrap - // the value in a cast for safety. If there are multiple return values, - // any casting will be handled by the transpilation of selector - // expressions. - if signature.Results().Len() == 1 { - return t.maybeCast(call, signature.Results().At(0).Type()) - } - return call - } - - // Finally, we fall to calls to any other functions. We'll handle any - // special cases that require a bit of extra fiddling to make work falling - // back to a not supported message. + // Big ol' switch to handle various special cases. switch id { case "sort.Strings": @@ -1147,10 +1102,67 @@ func (t *Transpiler) transpileCallExpr(n *ast.CallExpr) Node { // functionally equivalent. It has the added benefit of normalizing the // string form of the Quantity. return &Call{FuncName: "_shims.resource_MustParse", Arguments: []Node{reciever}} + } - default: + // All special cases have been handled. We've got a call into an external package. + // If it's not in our dependencies list, the call can't be handled by gotohelm. + // If it is, that means we're calling to either a method in the same + // package OR a method in a subchart. Either is fine, we'll just transpile + // the call. + + if _, ok := t.dependencies[callee.Pkg().Path()]; !ok { panic(fmt.Sprintf("unsupported function %q", id)) } + + // Make a comment here about things. + var call Node + + // Method call. + if r := callee.Type().(*types.Signature).Recv(); r == nil { + // Easy case: if there's no receiver, this is just a function call. + call = &Call{FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))), Arguments: args} + } else { + // Otherwise, if there is a receiver, we need to emulate a method call. + typ := r.Type() + + mutable := false + switch t := typ.(type) { + case *types.Pointer: + typ = t.Elem() + mutable = true + } + + if _, ok := typ.(*types.Named); !ok { + panic(&Unsupported{Fset: t.Fset, Node: n, Msg: "method calls with not pointer type with named type"}) + } + var receiverArg Node + + // When receiver is a pointer then dictionary can be passed as is. + // When receiver is not a pointer then dictionary is deep copied to emulate immutability. + receiverArg = &BuiltInCall{FuncName: "deepCopy", Arguments: []Node{t.transpileExpr(n.Fun.(*ast.SelectorExpr).X)}} + if mutable { + receiverArg = t.transpileExpr(n.Fun.(*ast.SelectorExpr).X) + } + + call = &Call{ + FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))), + // Method calls come in as a "top level" CallExpr where .Fun is the + // selector up to that call. e.g. `Foo.Bar.Baz()` will be a `CallExpr`. + // It's `.Fun` is a `SelectorExpr` where `.X` is `Foo.Bar`, the receiver, + // and `.Sel` is `Baz`, the method name. + Arguments: append([]Node{receiverArg}, args...), + } + } + + // If there's only a single return value, we'll possibly want to wrap + // the value in a cast for safety. If there are multiple return values, + // any casting will be handled by the transpilation of selector + // expressions. + if signature.Results().Len() == 1 { + return t.maybeCast(call, signature.Results().At(0).Type()) + } + + return call } func (t *Transpiler) transpileTypeRepr(typ types.Type) Node {