gotohelm: subcharting PoC

This is not complete by any means. Just wanted to see where blockers where, if
any. Was delighted to discover that `.Subcharts` did 90% of the hard work for
us and very few changes to gotohelm would be required.

Taskfile.yaml

@@ -112,7 +112,7 @@ tasks:
       # Generate the values JSON schema from the Values struct
       - go run ./cmd/genschema redpanda > charts/redpanda/values.schema.json
       # Generate helm templates from Go definitions
-      - go run ./cmd/gotohelm -write ./charts/redpanda/templates ./charts/redpanda 
+      - go run ./cmd/gotohelm -write ./charts/redpanda/templates ./charts/redpanda ./charts/...
 
   chart:generate:connectors:
     desc: "Generate files for the connectors Helm chart"

charts/redpanda/console.tpl.go

@@ -20,9 +20,59 @@ import (
 	"fmt"
 
 	"github.com/redpanda-data/console/backend/pkg/config"
+	"github.com/redpanda-data/helm-charts/charts/connectors"
+	"github.com/redpanda-data/helm-charts/charts/console"
 	"github.com/redpanda-data/helm-charts/pkg/gotohelm/helmette"
 )
 
+func ConsoleIntegration(dot *helmette.Dot) []any {
+	values := helmette.Unwrap[Values](dot.Values)
+
+	var manifests []any
+
+	// {{/* Secret */}}
+	// {{ $secretConfig := dict ( dict
+	//   "create" $.Values.console.secret.create
+	//   )
+	// }}
+	// {{ if and .Values.console.enabled (not .Values.console.secret.create) }}
+	// {{ $licenseKey := ( include "enterprise-license" .  ) }}
+	// # before license changes, this was not printing a secret, so we gather in which case to print
+	// # for now only if we have a license do we print, however, this may be an issue for some
+	// # since if we do include a license we MUST also print all secret items.
+	//   {{ if ( not (empty $licenseKey ) ) }}
+	// {{/* License and license are set twice here as a work around to a bug in the post-go console chart. */}}
+	// {{ $secretConfig = ( dict
+	//   "create" true
+	//   "enterprise" ( dict "license" $licenseKey "License" $licenseKey)
+	//   )
+	// }}
+	//
+	// {{ $config := dict
+	//   "Values" (dict
+	//   "secret" $secretConfig
+	//   )}}
+
+	// if the console chart has the creation of the secret disabled, create it here instead if needed
+	if values.Console.Enabled && !values.Console.Secret.Create {
+		consoleDot := helmette.MergeTo[*helmette.Dot](
+			dot.Subcharts["console"],
+			map[string]any{
+				"Values": map[string]any{
+					"secret": map[string]any{
+						"create": true,
+						// TODO enterprise license.
+					},
+				},
+			},
+		)
+
+		manifests = append(manifests, console.Secret(consoleDot))
+	}
+
+	return manifests
+}
+
 func ConsoleConfig(dot *helmette.Dot) any {
 	values := helmette.Unwrap[Values](dot.Values)
 
@@ -79,7 +129,7 @@ func ConsoleConfig(dot *helmette.Dot) any {
 		}
 
 		connectorsURL := fmt.Sprintf("http://%s.%s.svc.%s:%d",
-			ConnectorsFullName(dot),
+			connectors.Fullname(dot.Subcharts["connectors"]),
 			dot.Release.Namespace,
 			helmette.TrimSuffix(".", values.ClusterDomain),
 			p)

charts/redpanda/templates/_console.go.tpl

@@ -1,5 +1,21 @@
 {{- /* Generated from "console.tpl.go" */ -}}
 
+{{- define "redpanda.ConsoleIntegration" -}}
+{{- $dot := (index .a 0) -}}
+{{- range $_ := (list 1) -}}
+{{- $_is_returning := false -}}
+{{- $values := $dot.Values.AsMap -}}
+{{- $manifests := (coalesce nil) -}}
+{{- if (and $values.console.enabled (not $values.console.secret.create)) -}}
+{{- $consoleDot := (merge (dict ) (index $dot.Subcharts "console") (dict "Values" (dict "secret" (dict "create" true ) ) )) -}}
+{{- $manifests = (concat (default (list ) $manifests) (list (get (fromJson (include "console.Secret" (dict "a" (list $consoleDot) ))) "r"))) -}}
+{{- end -}}
+{{- $_is_returning = true -}}
+{{- (dict "r" $manifests) | toJson -}}
+{{- break -}}
+{{- end -}}
+{{- end -}}
+
 {{- define "redpanda.ConsoleConfig" -}}
 {{- $dot := (index .a 0) -}}
 {{- range $_ := (list 1) -}}
@@ -33,7 +49,7 @@
 {{- (dict "r" $c) | toJson -}}
 {{- break -}}
 {{- end -}}
-{{- $connectorsURL := (printf "http://%s.%s.svc.%s:%d" (get (fromJson (include "redpanda.ConnectorsFullName" (dict "a" (list $dot) ))) "r") $dot.Release.Namespace (trimSuffix "." $values.clusterDomain) $p) -}}
+{{- $connectorsURL := (printf "http://%s.%s.svc.%s:%d" (get (fromJson (include "connectors.Fullname" (dict "a" (list (index $dot.Subcharts "connectors")) ))) "r") $dot.Release.Namespace (trimSuffix "." $values.clusterDomain) $p) -}}
 {{- $_ := (set $c "connect" (mustMergeOverwrite (dict "enabled" false "clusters" (coalesce nil) "connectTimeout" 0 "readTimeout" 0 "requestTimeout" 0 ) (dict "enabled" $values.connectors.enabled "clusters" (list (mustMergeOverwrite (dict "name" "" "url" "" "tls" (dict "enabled" false "caFilepath" "" "certFilepath" "" "keyFilepath" "" "insecureSkipTlsVerify" false ) "username" "" "password" "" "token" "" ) (dict "name" "connectors" "url" $connectorsURL "tls" (mustMergeOverwrite (dict "enabled" false "caFilepath" "" "certFilepath" "" "keyFilepath" "" "insecureSkipTlsVerify" false ) (dict "enabled" false "caFilepath" "" "certFilepath" "" "keyFilepath" "" "insecureSkipTlsVerify" false )) "username" "" "password" "" "token" "" ))) ))) -}}
 {{- end -}}
 {{- $_is_returning = true -}}

charts/redpanda/templates/console/configmap-and-deployment.yaml

@@ -15,36 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{/* Secret */}}
-{{ $secretConfig := dict ( dict
-  "create" $.Values.console.secret.create
-  )
-}}
-{{/* if the console chart has the creation of the secret disabled, create it here instead if needed */}}
-{{ if and .Values.console.enabled (not .Values.console.secret.create) }}
-{{ $licenseKey := ( include "enterprise-license" .  ) }}
-# before license changes, this was not printing a secret, so we gather in which case to print
-# for now only if we have a license do we print, however, this may be an issue for some
-# since if we do include a license we MUST also print all secret items.
-  {{ if ( not (empty $licenseKey ) ) }}
-{{/* License and license are set twice here as a work around to a bug in the post-go console chart. */}}
-{{ $secretConfig = ( dict
-  "create" true
-  "enterprise" ( dict "license" $licenseKey "License" $licenseKey)
-  )
-}}
-
-{{ $config := dict
-  "Values" (dict
-  "secret" $secretConfig
-  )}}
-
-{{ $secretValues := merge $config .Subcharts.console }}
-{{ $wrappedSecretValues := (dict "Chart" .Subcharts.console.Chart "Release" .Release "Values" (dict "AsMap" $secretValues.Values)) }}
----
-{{- include "_shims.render-manifest" (list "console.Secret" $wrappedSecretValues) -}}
-  {{ end }}
-{{ end }}
+{{- include "_shims.render-manifest" (list "redpanda.ConsoleIntegration" .) -}}
 
 {{ $configmap := dict }}
 {{/* if the console chart has the creation of the configmap disabled, create it here instead */}}
@@ -57,7 +28,6 @@ limitations under the License.
     "Values" (dict
       "console" (dict "config" $consoleConfig)
       "configmap" $consoleConfigmap
-      "secret" $secretConfig
     )
 }}
 
@@ -192,7 +162,6 @@ limitations under the License.
   "extraVolumes" $extraVolumes
   "extraVolumeMounts" $extraVolumeMounts
   "extraEnv" $extraEnv
-  "secret" $secretConfig
   "enterprise" $enterprise
   "image" $.Values.console.image
   "autoscaling" .Values.console.autoscaling

charts/redpanda/values.go

@@ -87,6 +87,10 @@ type Console struct {
 	Console struct {
 		Config map[string]any `json:"config"`
 	} `json:"console"`
+
+	Secret struct {
+		Create bool `json:"create"`
+	} `json:"secret"`
 }
 
 type Connectors struct {

cmd/gotohelm/main.go

@@ -19,13 +19,22 @@ func main() {
 
 	pkgs, err := gotohelm.LoadPackages(&packages.Config{
 		Dir: cwd,
-	}, flag.Args()...)
+	}, flag.Args()[0])
+	if err != nil {
+		panic(err)
+	}
+
+	// falling back to golist might be better here?
+	// Might make the mapping consistent?
+	deps, err := gotohelm.LoadPackages(&packages.Config{
+		Dir: cwd,
+	}, flag.Args()[1:]...)
 	if err != nil {
 		panic(err)
 	}
 
 	for _, pkg := range pkgs {
-		chart, err := gotohelm.Transpile(pkg)
+		chart, err := gotohelm.Transpile(pkg, deps...)
 		if err != nil {
 			fmt.Printf("Failed to transpile %q: %s\n", pkg.Name, err)
 			continue

pkg/gotohelm/helmette/helm.go

@@ -13,9 +13,10 @@ import (
 // of a helm template.
 // See also: https://github.com/helm/helm/blob/3764b483b385a12e7d3765bff38eced840362049/pkg/chartutil/values.go#L137-L166
 type Dot struct {
-	Values  Values
-	Release Release
-	Chart   Chart
+	Values    Values
+	Release   Release
+	Chart     Chart
+	Subcharts map[string]*Dot
 	// Capabilities
 
 	// KubeConfig is a hacked in value to allow `Lookup` to not rely on global

pkg/gotohelm/transpiler.go

@@ -45,7 +45,7 @@ type Chart struct {
 	Files []*File
 }
 
-func Transpile(pkg *packages.Package) (_ *Chart, err error) {
+func Transpile(pkg *packages.Package, deps ...*packages.Package) (_ *Chart, err error) {
 	defer func() {
 		switch v := recover().(type) {
 		case nil:
@@ -56,15 +56,21 @@ func Transpile(pkg *packages.Package) (_ *Chart, err error) {
 		}
 	}()
 
+	dependencies := map[string]struct{}{}
+	for _, pkg := range append(deps, pkg) {
+		dependencies[pkg.PkgPath] = struct{}{}
+	}
+
 	t := &Transpiler{
 		Package:   pkg,
 		Fset:      pkg.Fset,
 		TypesInfo: pkg.TypesInfo,
 		Files:     pkg.Syntax,
 
-		packages:   mkPkgTree(pkg),
-		namespaces: map[*types.Package]string{},
-		names:      map[*types.Func]string{},
+		packages:     mkPkgTree(pkg),
+		namespaces:   map[*types.Package]string{},
+		dependencies: dependencies,
+		names:        map[*types.Func]string{},
 		builtins: map[string]string{
 			"fmt.Sprintf":                "printf",
 			"golang.org/x/exp/maps.Keys": "keys",
@@ -91,6 +97,8 @@ type Transpiler struct {
 	// their builtin equivalent.
 	builtins map[string]string
 	packages map[string]*packages.Package
+	// todo
+	dependencies map[string]struct{}
 	// namespaces is a cache for holding the namespace package directive. It's
 	// exclusively used by `namespaceFor`.
 	namespaces map[*types.Package]string
@@ -981,60 +989,7 @@ func (t *Transpiler) transpileCallExpr(n *ast.CallExpr) Node {
 		})
 	}
 
-	// Call to function within the same package. A-Okay. It's transpiled. NB:
-	// This is intentionally after the builtins check to support our bootstrap
-	// package's builtin bindings.
-	if callee.Pkg().Path() == t.Package.PkgPath {
-		var call Node
-
-		// Method call.
-		if r := callee.Type().(*types.Signature).Recv(); r != nil {
-			typ := r.Type()
-
-			mutable := false
-			switch t := typ.(type) {
-			case *types.Pointer:
-				typ = t.Elem()
-				mutable = true
-			}
-
-			if _, ok := typ.(*types.Named); !ok {
-				panic(&Unsupported{Fset: t.Fset, Node: n, Msg: "method calls with not pointer type with named type"})
-			}
-			var receiverArg Node
-
-			// When receiver is a pointer then dictionary can be passed as is.
-			// When receiver is not a pointer then dictionary is a deep copied.
-			receiverArg = &BuiltInCall{FuncName: "deepCopy", Arguments: []Node{t.transpileExpr(n.Fun.(*ast.SelectorExpr).X)}}
-			if mutable {
-				receiverArg = t.transpileExpr(n.Fun.(*ast.SelectorExpr).X)
-			}
-
-			call = &Call{
-				FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))),
-				// Method calls come in as a "top level" CallExpr where .Fun is the
-				// selector up to that call. e.g. `Foo.Bar.Baz()` will be a `CallExpr`.
-				// It's `.Fun` is a `SelectorExpr` where `.X` is `Foo.Bar`, the receiver,
-				// and `.Sel` is `Baz`, the method name.
-				Arguments: append([]Node{receiverArg}, args...),
-			}
-		} else {
-			call = &Call{FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))), Arguments: args}
-		}
-
-		// If there's only a single return value, we'll possibly want to wrap
-		// the value in a cast for safety. If there are multiple return values,
-		// any casting will be handled by the transpilation of selector
-		// expressions.
-		if signature.Results().Len() == 1 {
-			return t.maybeCast(call, signature.Results().At(0).Type())
-		}
-		return call
-	}
-
-	// Finally, we fall to calls to any other functions. We'll handle any
-	// special cases that require a bit of extra fiddling to make work falling
-	// back to a not supported message.
+	// Big ol' switch to handle various special cases.
 
 	switch id {
 	case "sort.Strings":
@@ -1147,10 +1102,67 @@ func (t *Transpiler) transpileCallExpr(n *ast.CallExpr) Node {
 		// functionally equivalent. It has the added benefit of normalizing the
 		// string form of the Quantity.
 		return &Call{FuncName: "_shims.resource_MustParse", Arguments: []Node{reciever}}
+	}
 
-	default:
+	// All special cases have been handled. We've got a call into an external package.
+	// If it's not in our dependencies list, the call can't be handled by gotohelm.
+	// If it is, that means we're calling to either a method in the same
+	// package OR a method in a subchart. Either is fine, we'll just transpile
+	// the call.
+
+	if _, ok := t.dependencies[callee.Pkg().Path()]; !ok {
 		panic(fmt.Sprintf("unsupported function %q", id))
 	}
+
+	// Make a comment here about things.
+	var call Node
+
+	// Method call.
+	if r := callee.Type().(*types.Signature).Recv(); r == nil {
+		// Easy case: if there's no receiver, this is just a function call.
+		call = &Call{FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))), Arguments: args}
+	} else {
+		// Otherwise, if there is a receiver, we need to emulate a method call.
+		typ := r.Type()
+
+		mutable := false
+		switch t := typ.(type) {
+		case *types.Pointer:
+			typ = t.Elem()
+			mutable = true
+		}
+
+		if _, ok := typ.(*types.Named); !ok {
+			panic(&Unsupported{Fset: t.Fset, Node: n, Msg: "method calls with not pointer type with named type"})
+		}
+		var receiverArg Node
+
+		// When receiver is a pointer then dictionary can be passed as is.
+		// When receiver is not a pointer then dictionary is deep copied to emulate immutability.
+		receiverArg = &BuiltInCall{FuncName: "deepCopy", Arguments: []Node{t.transpileExpr(n.Fun.(*ast.SelectorExpr).X)}}
+		if mutable {
+			receiverArg = t.transpileExpr(n.Fun.(*ast.SelectorExpr).X)
+		}
+
+		call = &Call{
+			FuncName: fmt.Sprintf("%s.%s", t.namespaceFor(callee.Pkg()), t.funcNameFor(callee.(*types.Func))),
+			// Method calls come in as a "top level" CallExpr where .Fun is the
+			// selector up to that call. e.g. `Foo.Bar.Baz()` will be a `CallExpr`.
+			// It's `.Fun` is a `SelectorExpr` where `.X` is `Foo.Bar`, the receiver,
+			// and `.Sel` is `Baz`, the method name.
+			Arguments: append([]Node{receiverArg}, args...),
+		}
+	}
+
+	// If there's only a single return value, we'll possibly want to wrap
+	// the value in a cast for safety. If there are multiple return values,
+	// any casting will be handled by the transpilation of selector
+	// expressions.
+	if signature.Results().Len() == 1 {
+		return t.maybeCast(call, signature.Results().At(0).Type())
+	}
+
+	return call
 }
 
 func (t *Transpiler) transpileTypeRepr(typ types.Type) Node {