From 37ea4ccc1d96f4d74a34a7fd71e88e6425600f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 27 Jun 2024 16:22:35 +0200 Subject: [PATCH] Add js.Batch Fixes #12626 Closes #7499 Closes #12874 --- common/herrors/errors.go | 15 + common/maps/scratch.go | 14 + media/mediaType.go | 12 +- resources/resource.go | 6 + resources/resource/resourcetypes.go | 7 + resources/resource_transformers/js/build.go | 131 ++- resources/resource_transformers/js/options.go | 163 +++- .../resource_transformers/js/options_test.go | 102 ++- .../resource_transformers/js/transform.go | 71 ++ resources/transform.go | 6 + .../go_templates/texttemplate/exec.go | 5 +- .../texttemplate/hugo_template.go | 34 +- tpl/js/batch-esm-callback.gotmpl | 15 + tpl/js/batch.go | 835 ++++++++++++++++++ tpl/js/batch_integration_test.go | 154 ++++ tpl/js/js.go | 24 +- 16 files changed, 1433 insertions(+), 161 deletions(-) create mode 100644 resources/resource_transformers/js/transform.go create mode 100644 tpl/js/batch-esm-callback.gotmpl create mode 100644 tpl/js/batch.go create mode 100644 tpl/js/batch_integration_test.go diff --git a/common/herrors/errors.go b/common/herrors/errors.go index e7f91462e31..67ca1d1d4da 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -133,6 +133,21 @@ func IsNotExist(err error) bool { return false } +// IsExist returns true if the error is a file exists error. +// Unlike os.IsExist, this also considers wrapped errors. +func IsExist(err error) bool { + if os.IsExist(err) { + return true + } + + // os.IsExist does not consider wrapped errors. + if os.IsExist(errors.Unwrap(err)) { + return true + } + + return false +} + var nilPointerErrRe = regexp.MustCompile(`at <(.*)>: error calling (.*?): runtime error: invalid memory address or nil pointer dereference`) const deferredPrefix = "__hdeferred/" diff --git a/common/maps/scratch.go b/common/maps/scratch.go index e9f412540b2..3bb160ae037 100644 --- a/common/maps/scratch.go +++ b/common/maps/scratch.go @@ -107,6 +107,20 @@ func (c *Scratch) Get(key string) any { return val } +// GetOrCreate returns the value for the given key if it exists, or creates it +// using the given func and stores that value in the map. +// For internal use. +func (c *Scratch) GetOrCreate(key string, create func() any) any { + c.mu.Lock() + defer c.mu.Unlock() + if val, found := c.values[key]; found { + return val + } + val := create() + c.values[key] = val + return val +} + // Values returns the raw backing map. Note that you should just use // this method on the locally scoped Scratch instances you obtain via newScratch, not // .Page.Scratch etc., as that will lead to concurrency issues. diff --git a/media/mediaType.go b/media/mediaType.go index a7ba1309a7d..97b10879c07 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -273,9 +273,13 @@ func (t Types) GetByType(tp string) (Type, bool) { return Type{}, false } +func (t Types) normalizeSuffix(s string) string { + return strings.ToLower(strings.TrimPrefix(s, ".")) +} + // BySuffix will return all media types matching a suffix. func (t Types) BySuffix(suffix string) []Type { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) var types []Type for _, tt := range t { if tt.hasSuffix(suffix) { @@ -287,7 +291,7 @@ func (t Types) BySuffix(suffix string) []Type { // GetFirstBySuffix will return the first type matching the given suffix. func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt, SuffixInfo{ @@ -304,7 +308,7 @@ func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { // is ambiguous. // The lookup is case insensitive. func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { if found { @@ -324,7 +328,7 @@ func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { } func (t Types) IsTextSuffix(suffix string) bool { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt.IsText() diff --git a/resources/resource.go b/resources/resource.go index cc7008e5a88..b32cb0baed1 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -47,6 +47,7 @@ var ( _ resource.Cloner = (*genericResource)(nil) _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) _ resource.Identifier = (*genericResource)(nil) + _ resource.PathProvider = (*genericResource)(nil) _ identity.IdentityGroupProvider = (*genericResource)(nil) _ identity.DependencyManagerProvider = (*genericResource)(nil) _ identity.Identity = (*genericResource)(nil) @@ -463,6 +464,11 @@ func (l *genericResource) Key() string { return key } +// TODO1 test and document this. Consider adding it to the Resource interface. +func (l *genericResource) Path() string { + return l.paths.TargetPath() +} + func (l *genericResource) MediaType() media.Type { return l.sd.MediaType } diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 0fb87f37137..8fcc3b70b9f 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -108,6 +108,13 @@ type MediaTypeProvider interface { MediaType() media.Type } +type PathProvider interface { + // Path is the relative path to this resource. + // In most cases this will be the same as the RelPermalink(), + // but it will not trigger any lazy publishing. + Path() string +} + type ResourceLinksProvider interface { // Permalink represents the absolute link to this resource. Permalink() string diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go index cc68d225335..b97548a44ff 100644 --- a/resources/resource_transformers/js/build.go +++ b/resources/resource_transformers/js/build.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,25 +16,20 @@ package js import ( "errors" "fmt" - "io" "os" "path" "path/filepath" "regexp" "strings" - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/media" - + "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/text" - + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/identity" + "github.com/spf13/afero" - "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" ) @@ -53,54 +48,55 @@ func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) *Client { } } -type buildTransformation struct { - optsm map[string]any - c *Client +// ProcessExernal processes a resource with the user provided options. +func (c *Client) ProcessExernal(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { + return res.Transform( + &buildTransformation{c: c, optsm: opts}, + ) } -func (t *buildTransformation) Key() internal.ResourceTransformationKey { - return internal.NewResourceTransformationKey("jsbuild", t.optsm) +// ProcessExernal processes a resource with the given options. +func (c *Client) ProcessInternal(res resources.ResourceTransformer, opts Options) (resource.Resource, error) { + return res.Transform( + &buildTransformation{c: c, opts: opts}, + ) } -func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - ctx.OutMediaType = media.Builtin.JavascriptType +func (c *Client) BuildBundle(opts Options) (api.BuildResult, error) { + return c.build(opts, nil) +} - opts, err := decodeOptions(t.optsm) - if err != nil { - return err +// Note that transformCtx may be nil. +func (c *Client) build(opts Options, transformCtx *resources.ResourceTransformationCtx) (api.BuildResult, error) { + dependencyManager := opts.DependencyManager + if transformCtx != nil { + dependencyManager = transformCtx.DependencyManager // TODO1 } - - if opts.TargetPath != "" { - ctx.OutPath = opts.TargetPath - } else { - ctx.ReplaceOutPathExtension(".js") + if dependencyManager == nil { + dependencyManager = identity.NopManager } - src, err := io.ReadAll(ctx.From) - if err != nil { - return err - } + opts.ResolveDir = c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved + opts.TsConfig = c.rs.ResolveJSConfigFile("tsconfig.json") - opts.sourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) - opts.resolveDir = t.c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved - opts.contents = string(src) - opts.mediaType = ctx.InMediaType - opts.tsConfig = t.c.rs.ResolveJSConfigFile("tsconfig.json") + if err := opts.validate(); err != nil { + return api.BuildResult{}, err + } buildOptions, err := toBuildOptions(opts) if err != nil { - return err + return api.BuildResult{}, err } - buildOptions.Plugins, err = createBuildPlugins(ctx.DependencyManager, t.c, opts) + buildOptions.Plugins, err = createBuildPlugins(c, dependencyManager, opts) if err != nil { - return err + return api.BuildResult{}, err } if buildOptions.Sourcemap == api.SourceMapExternal && buildOptions.Outdir == "" { buildOptions.Outdir, err = os.MkdirTemp(os.TempDir(), "compileOutput") if err != nil { - return err + return api.BuildResult{}, err } defer os.Remove(buildOptions.Outdir) } @@ -110,13 +106,13 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx for i, ext := range opts.Inject { impPath := filepath.FromSlash(ext) if filepath.IsAbs(impPath) { - return fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") + return api.BuildResult{}, fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") } - m := resolveComponentInAssets(t.c.rs.Assets.Fs, impPath) + m := resolveComponentInAssets(c.rs.Assets.Fs, impPath) if m == nil { - return fmt.Errorf("inject: file %q not found", ext) + return api.BuildResult{}, fmt.Errorf("inject: file %q not found", ext) } opts.Inject[i] = m.Filename @@ -138,7 +134,7 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx } path := loc.File if path == stdinImporter { - path = ctx.SourcePath + path = transformCtx.SourcePath } errorMessage := msg.Text @@ -154,7 +150,7 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx f, err = hugofs.Os.Open(path) } else { var fi os.FileInfo - fi, err = t.c.sfs.Fs.Stat(path) + fi, err = c.sfs.Fs.Stat(path) if err == nil { m := fi.(hugofs.FileMetaInfo).Meta() path = m.Filename @@ -185,38 +181,37 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx // Return 1, log the rest. for i, err := range errors { if i > 0 { - t.c.rs.Logger.Errorf("js.Build failed: %s", err) + c.rs.Logger.Errorf("js.Build failed: %s", err) } } - return errors[0] + return result, errors[0] } - if buildOptions.Sourcemap == api.SourceMapExternal { - content := string(result.OutputFiles[1].Contents) - symPath := path.Base(ctx.OutPath) + ".map" - re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) - content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") + // TODO1 option etc. fmt.Printf("%s", api.AnalyzeMetafile(result.Metafile, api.AnalyzeMetafileOptions{})) + + if transformCtx != nil { + if buildOptions.Sourcemap == api.SourceMapExternal { + content := string(result.OutputFiles[1].Contents) + symPath := path.Base(transformCtx.OutPath) + ".map" + re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) + content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") + + if err = transformCtx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { + return result, err + } + _, err := transformCtx.To.Write([]byte(content)) + if err != nil { + return result, err + } + } else { + _, err := transformCtx.To.Write(result.OutputFiles[0].Contents) + if err != nil { + return result, err + } - if err = ctx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { - return err - } - _, err := ctx.To.Write([]byte(content)) - if err != nil { - return err - } - } else { - _, err := ctx.To.Write(result.OutputFiles[0].Contents) - if err != nil { - return err } } - return nil -} -// Process process esbuild transform -func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { - return res.Transform( - &buildTransformation{c: c, optsm: opts}, - ) + return result, nil } diff --git a/resources/resource_transformers/js/options.go b/resources/resource_transformers/js/options.go index 8c271d032d7..03741d2d454 100644 --- a/resources/resource_transformers/js/options.go +++ b/resources/resource_transformers/js/options.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "os" + "path" "path/filepath" "strings" @@ -39,8 +40,44 @@ const ( stdinImporter = "" ) -// Options esbuild configuration type Options struct { + ExternalOptions + InternalOptions +} + +func (opts *Options) validate() error { + if opts.ImportOnResolveFunc != nil && opts.ImportOnLoadFunc == nil { + return fmt.Errorf("ImportOnLoadFunc must be set if ImportOnResolveFunc is set") + } + if opts.ImportOnResolveFunc == nil && opts.ImportOnLoadFunc != nil { + return fmt.Errorf("ImportOnResolveFunc must be set if ImportOnLoadFunc is set") + } + return nil +} + +// InternalOptions holds internal options for the js.Build template function. +type InternalOptions struct { + MediaType media.Type + OutDir string + Contents string + SourceDir string + ResolveDir string + + DependencyManager identity.Manager + + // TODO1 + Write bool // Set to false to write to memory. + AllowOverwrite bool + Splitting bool + TsConfig string + EntryPoints []string + ImportOnResolveFunc func(string) string + ImportOnLoadFunc func(string) string + Stdin bool +} + +// ExternalOptions holds user facing options for the js.Build template function. +type ExternalOptions struct { // If not set, the source path will be used as the base target path. // Note that the target path's extension may change if the target MIME type // is different, e.g. when the source is TypeScript. @@ -105,17 +142,10 @@ type Options struct { // Deprecated: This no longer have any effect and will be removed. // TODO(bep) remove. See https://github.com/evanw/esbuild/commit/869e8117b499ca1dbfc5b3021938a53ffe934dba AvoidTDZ bool - - mediaType media.Type - outDir string - contents string - sourceDir string - resolveDir string - tsConfig string } -func decodeOptions(m map[string]any) (Options, error) { - var opts Options +func decodeOptions(m map[string]any) (ExternalOptions, error) { + var opts ExternalOptions if err := mapstructure.WeakDecode(m, &opts); err != nil { return opts, err @@ -212,7 +242,7 @@ func resolveComponentInAssets(fs afero.Fs, impPath string) *hugofs.FileMeta { return m } -func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ([]api.Plugin, error) { +func createBuildPlugins(c *Client, depsManager identity.Manager, opts Options) ([]api.Plugin, error) { fs := c.rs.Assets resolveImport := func(args api.OnResolveArgs) (api.OnResolveResult, error) { @@ -223,26 +253,49 @@ func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ( impPath = override } } - isStdin := args.Importer == stdinImporter - var relDir string - if !isStdin { - rel, found := fs.MakePathRelative(args.Importer, true) - if !found { - // Not in any of the /assets folders. - // This is an import from a node_modules, let - // ESBuild resolve this. - return api.OnResolveResult{}, nil + + if opts.ImportOnResolveFunc != nil { + if s := opts.ImportOnResolveFunc(impPath); s != "" { + return api.OnResolveResult{Path: s, Namespace: nsImportHugo}, nil } + } - relDir = filepath.Dir(rel) - } else { - relDir = opts.sourceDir + dodebug := strings.Contains(impPath, "css") + if dodebug { + // impPath = strings.TrimPrefix(impPath, "/") } + importer := args.Importer + // TODO1 todelido. + if false && dodebug { + impPath = path.Join("js/hugoheadlessui/components", impPath) + } else { + isStdin := importer == stdinImporter + var relDir string + if !isStdin { + if strings.HasPrefix(importer, "@hugo-virtual") { + // TODO1 constants. + relDir = filepath.Dir(strings.TrimPrefix(importer, "@hugo-virtual")) + } else { + rel, found := fs.MakePathRelative(importer, true) + + if !found { + // Not in any of the /assets folders. + // This is an import from a node_modules, let + // ESBuild resolve this. + return api.OnResolveResult{}, nil + } + + relDir = filepath.Dir(rel) + } + } else { + relDir = opts.SourceDir + } - // Imports not starting with a "." is assumed to live relative to /assets. - // Hugo makes no assumptions about the directory structure below /assets. - if relDir != "" && strings.HasPrefix(impPath, ".") { - impPath = filepath.Join(relDir, impPath) + // Imports not starting with a "." is assumed to live relative to /assets. + // Hugo makes no assumptions about the directory structure below /assets. + if relDir != "" && strings.HasPrefix(impPath, ".") { + impPath = filepath.Join(relDir, impPath) + } } m := resolveComponentInAssets(fs.Fs, impPath) @@ -272,16 +325,26 @@ func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ( }) build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsImportHugo}, func(args api.OnLoadArgs) (api.OnLoadResult, error) { - b, err := os.ReadFile(args.Path) - if err != nil { - return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + var c string + if opts.ImportOnLoadFunc != nil { + if s := opts.ImportOnLoadFunc(args.Path); s != "" { + c = s + } + } + + if c == "" { + b, err := os.ReadFile(args.Path) + if err != nil { + return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + } + c = string(b) } - c := string(b) + return api.OnLoadResult{ // See https://github.com/evanw/esbuild/issues/502 // This allows all modules to resolve dependencies // in the main project's node_modules. - ResolveDir: opts.resolveDir, + ResolveDir: opts.ResolveDir, Contents: &c, Loader: loaderFromFilename(args.Path), }, nil @@ -353,7 +416,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { return } - mediaType := opts.mediaType + mediaType := opts.MediaType if mediaType.IsZero() { mediaType = media.Builtin.JavascriptType } @@ -371,7 +434,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { case media.Builtin.JSXType.SubType: loader = api.LoaderJSX default: - err = fmt.Errorf("unsupported Media Type: %q", opts.mediaType) + err = fmt.Errorf("unsupported Media Type: %q", opts.MediaType) return } @@ -408,7 +471,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { } // By default we only need to specify outDir and no outFile - outDir := opts.outDir + outDir := opts.OutDir outFile := "" var sourceMap api.SourceMap switch opts.SourceMap { @@ -424,8 +487,9 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { } buildOptions = api.BuildOptions{ - Outfile: outFile, - Bundle: true, + Outfile: outFile, + Bundle: true, + Metafile: true, Target: target, Format: format, @@ -435,9 +499,12 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { MinifyIdentifiers: opts.Minify, MinifySyntax: opts.Minify, - Outdir: outDir, - Define: defines, + Outdir: outDir, + Write: opts.Write, + AllowOverwrite: opts.AllowOverwrite, + Splitting: opts.Splitting, + Define: defines, External: opts.Externals, JSXFactory: opts.JSXFactory, @@ -446,16 +513,18 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { JSX: jsx, JSXImportSource: opts.JSXImportSource, - Tsconfig: opts.tsConfig, + Tsconfig: opts.TsConfig, + + EntryPoints: opts.EntryPoints, + } - // Note: We're not passing Sourcefile to ESBuild. - // This makes ESBuild pass `stdin` as the Importer to the import - // resolver, which is what we need/expect. - Stdin: &api.StdinOptions{ - Contents: opts.contents, - ResolveDir: opts.resolveDir, + if opts.Stdin { + // This makes ESBuild pass `stdin` as the Importer to the import. + buildOptions.Stdin = &api.StdinOptions{ + Contents: opts.Contents, + ResolveDir: opts.ResolveDir, Loader: loader, - }, + } } return } diff --git a/resources/resource_transformers/js/options_test.go b/resources/resource_transformers/js/options_test.go index 53aa9b6bbff..18a208d1b31 100644 --- a/resources/resource_transformers/js/options_test.go +++ b/resources/resource_transformers/js/options_test.go @@ -50,7 +50,11 @@ func TestOptionKey(t *testing.T) { func TestToBuildOptions(t *testing.T) { c := qt.New(t) - opts, err := toBuildOptions(Options{mediaType: media.Builtin.JavascriptType}) + opts, err := toBuildOptions(Options{ + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ @@ -62,13 +66,19 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", - Format: "cjs", - Minify: true, - mediaType: media.Builtin.JavascriptType, - AvoidTDZ: true, - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", + Format: "cjs", + Minify: true, + AvoidTDZ: true, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -82,10 +92,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -100,10 +117,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -118,10 +142,18 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "external", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "external", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -136,10 +168,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - mediaType: media.Builtin.JavascriptType, - JSX: "automatic", JSXImportSource: "preact", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + JSX: "automatic", JSXImportSource: "preact", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -173,10 +212,17 @@ func TestToBuildOptionsTarget(t *testing.T) { {"esnext", api.ESNext}, } { c.Run(test.target, func(c *qt.C) { - opts, err := toBuildOptions(Options{ - Target: test.target, - mediaType: media.Builtin.JavascriptType, - }) + opts, err := toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: test.target, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts.Target, qt.Equals, test.expect) }) diff --git a/resources/resource_transformers/js/transform.go b/resources/resource_transformers/js/transform.go new file mode 100644 index 00000000000..4e6912bacab --- /dev/null +++ b/resources/resource_transformers/js/transform.go @@ -0,0 +1,71 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "io" + "path" + "path/filepath" + + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" +) + +type buildTransformation struct { + optsm map[string]any + opts Options + c *Client +} + +func (t *buildTransformation) Key() internal.ResourceTransformationKey { + // Pick the most stable key source. + var v any = t.optsm + if v == nil { + v = t.opts + } + return internal.NewResourceTransformationKey("jsbuild", v) +} + +func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { + ctx.OutMediaType = media.Builtin.JavascriptType + + if t.optsm != nil { + optsExt, err := decodeOptions(t.optsm) + if err != nil { + return err + } + t.opts.ExternalOptions = optsExt + } + + if t.opts.TargetPath != "" { + ctx.OutPath = t.opts.TargetPath + } else { + ctx.ReplaceOutPathExtension(".js") + } + + src, err := io.ReadAll(ctx.From) + if err != nil { + return err + } + + t.opts.SourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) + t.opts.Contents = string(src) + t.opts.MediaType = ctx.InMediaType + t.opts.Stdin = true + + _, err = t.c.build(t.opts, ctx) + + return err +} diff --git a/resources/transform.go b/resources/transform.go index 336495e6d07..9781ea6c31c 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -52,6 +52,7 @@ var ( _ identity.IdentityGroupProvider = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) + _ resource.PathProvider = (*resourceAdapter)(nil) _ resource.ResourceNameTitleProvider = (*resourceAdapter)(nil) _ resource.WithResourceMetaProvider = (*resourceAdapter)(nil) _ identity.DependencyManagerProvider = (*resourceAdapter)(nil) @@ -277,6 +278,11 @@ func (r *resourceAdapter) Key() string { return r.target.(resource.Identifier).Key() } +func (r *resourceAdapter) Path() string { + r.init(false, false) + return r.target.(resource.PathProvider).Path() +} + func (r *resourceAdapter) MediaType() media.Type { r.init(false, false) return r.target.MediaType() diff --git a/tpl/internal/go_templates/texttemplate/exec.go b/tpl/internal/go_templates/texttemplate/exec.go index bd8c82bd705..f7bffad7aae 100644 --- a/tpl/internal/go_templates/texttemplate/exec.go +++ b/tpl/internal/go_templates/texttemplate/exec.go @@ -305,7 +305,10 @@ func (s *state) walkIfOrWith(typ parse.NodeType, dot reflect.Value, pipe *parse. } if truth { if typ == parse.NodeWith { - s.walk(val, list) + func() { + defer s.pushWithValue(val)() + s.walk(val, list) + }() } else { s.walk(dot, list) } diff --git a/tpl/internal/go_templates/texttemplate/hugo_template.go b/tpl/internal/go_templates/texttemplate/hugo_template.go index 12dbe041217..2be75eadda0 100644 --- a/tpl/internal/go_templates/texttemplate/hugo_template.go +++ b/tpl/internal/go_templates/texttemplate/hugo_template.go @@ -19,6 +19,7 @@ import ( "reflect" "github.com/gohugoio/hugo/common/hreflect" + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" ) @@ -110,14 +111,31 @@ func (t *Template) executeWithState(state *state, value reflect.Value) (err erro // template so that multiple executions of the same template // can execute in parallel. type state struct { - tmpl *Template - ctx context.Context // Added for Hugo. The original data context. - prep Preparer // Added for Hugo. - helper ExecHelper // Added for Hugo. - wr io.Writer - node parse.Node // current node, for errors - vars []variable // push-down stack of variable values. - depth int // the height of the stack of executing templates. + tmpl *Template + ctx context.Context // Added for Hugo. The original data context. + prep Preparer // Added for Hugo. + helper ExecHelper // Added for Hugo. + withValues []reflect.Value // Added for Hugo. Push-down stack of values. + + wr io.Writer + node parse.Node // current node, for errors + vars []variable // push-down stack of variable values. + depth int // the height of the stack of executing templates. +} + +func (s *state) pushWithValue(value reflect.Value) func() { + s.withValues = append(s.withValues, value) + return func() { + // TODO1 integrate with GO 1.23. + v, _ := indirect(s.withValues[len(s.withValues)-1]) + if hreflect.IsValid(v) { + if closer, ok := v.Interface().(types.Closer); ok { + closer.Close() + } + } + + s.withValues = s.withValues[:len(s.withValues)-1] + } } func (s *state) evalFunction(dot reflect.Value, node *parse.IdentifierNode, cmd parse.Node, args []parse.Node, final reflect.Value) reflect.Value { diff --git a/tpl/js/batch-esm-callback.gotmpl b/tpl/js/batch-esm-callback.gotmpl new file mode 100644 index 00000000000..aa198719a8b --- /dev/null +++ b/tpl/js/batch-esm-callback.gotmpl @@ -0,0 +1,15 @@ +{{ range $i, $e := .Modules -}} + import { default as {{ printf "Mod%d" $i }} } from "{{ .ImportPath }}"; +{{ end -}} +{{ with .CallbackImportPath }} + import { default as Callback } from "{{ . }}"; +{{ end }} +{{/* */}} +let mods = []; +{{ range $i, $e := .Modules -}} + mods.push({{ .CallbackJSON $i }}); +{{ end -}} +{{/* */}} +{{ if .CallbackImportPath }} + Callback(mods); +{{ end }} diff --git a/tpl/js/batch.go b/tpl/js/batch.go new file mode 100644 index 00000000000..e21d53a647b --- /dev/null +++ b/tpl/js/batch.go @@ -0,0 +1,835 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package js + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/gohugoio/hugo/cache/dynacache" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_transformers/js" + template "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate" + "github.com/mitchellh/mapstructure" + "github.com/spf13/cast" +) + +type Batcher interface { + UseScript(id string) BatcherScript + UseScriptGroup(id string) BatcherScriptMany + Build() (*Package, error) +} + +type BatcherScript interface { + BatcherScriptOps + types.Closer +} + +type BatcherScriptOps interface { + ResourceGetSetter + AddInstance(id string, opts any) string +} + +type BatcherScriptMany interface { + BatcherScriptManyOps + types.Closer +} + +type BatcherScriptManyOps interface { + CallbackGetSetter + UseScript(id string) BatcherScript +} + +type CallbackGetSetter interface { + GetCallback() resource.Resource + SetCallback(r resource.Resource) string +} + +type ResourceGetSetter interface { + GetResource() resource.Resource + SetResource(r resource.Resource) string +} + +func (ns *Namespace) Batch(id string, store *maps.Scratch) (Batcher, error) { + key := path.Join(nsBundle, id) + b := store.GetOrCreate(key, func() any { + return &batcher{id: id, scriptOnes: make(map[string]*scriptOne), scriptManys: make(map[string]*scriptMany), client: ns} + }) + return b.(*batcher), nil +} + +func (b *batcher) UseScript(id string) BatcherScript { + b.mu.Lock() + + one, found := b.scriptOnes[id] + if !found { + one = &scriptOne{ + id: id, + instances: make(map[string]scriptInstance), + client: b.client, + } + b.scriptOnes[id] = one + } + + b.mu.Unlock() + one.mu.Lock() + + // This will be auto closed if used in a with statement. + // But the caller may also call Close, so make sure we only do it once. + var closeOnce sync.Once + + return struct { + BatcherScriptOps + types.Closer + }{ + one, + close(func() error { + closeOnce.Do(func() { + one.mu.Unlock() + }) + return nil + }), + } +} + +func (b *batcher) UseScriptGroup(id string) BatcherScriptMany { + b.mu.Lock() + + many, found := b.scriptManys[id] + if !found { + many = &scriptMany{id: id, client: b.client, items: make(map[string]*scriptManyItem)} + b.scriptManys[id] = many + } + + b.mu.Unlock() + many.mu.Lock() + + // This will be auto closed if used in a with statement. + // But the caller may also call Close, so make sure we only do it once. + var closeOnce sync.Once + + return struct { + BatcherScriptManyOps + types.Closer + }{ + many, + close(func() error { + closeOnce.Do(func() { + many.mu.Unlock() + }) + return nil + }), + } +} + +type close func() error + +func (c close) Close() error { + return c() +} + +var ( + _ Batcher = (*batcher)(nil) + _ BatcherScriptOps = (*scriptOne)(nil) + _ BatcherScriptManyOps = (*scriptMany)(nil) +) + +func (b *scriptOne) AddInstance(id string, opts any) string { + if b.r == nil { + panic("resource not set") + } + if id == "" { + panic("id not set") + } + + b.instances[id] = decodeScriptInstance(opts) + return "" +} + +func decodeScriptInstance(opts any) scriptInstance { + var inst scriptInstance + if err := mapstructure.WeakDecode(opts, &inst); err != nil { + panic(err) + } + return inst +} + +func (b *scriptManyItem) AddInstance(id string, opts any) string { + b.instances[id] = decodeScriptInstance(opts) + return "" +} + +func (b *scriptMany) GetCallback() resource.Resource { + if resource.StaleVersion(b.callback) > 0 { + // Allow the client to set a new resource. + return nil + } + return b.callback +} + +func (b *scriptMany) SetCallback(r resource.Resource) string { + if r == nil { + // TODO1 apply this to all Setters. + panic("resource not set") + } + b.callback = r + return "" +} + +func (b *scriptMany) UseScript(id string) BatcherScript { + item, found := b.items[id] + if !found { + item = &scriptManyItem{ + id: id, + instances: make(map[string]scriptInstance), + client: b.client, + } + b.items[id] = item + } + + item.mu.Lock() + + // This will be auto closed if used in a with statement. + // But the caller may also call Close, so make sure we only do it once. + var closeOnce sync.Once + + return struct { + BatcherScriptOps + types.Closer + }{ + item, + close(func() error { + closeOnce.Do(func() { + item.mu.Unlock() + }) + return nil + }), + } +} + +type batchTemplateContext struct { + keyPath string + ID string + CallbackImportPath string + Modules []batchTemplateExecutionsContext +} + +type batchTemplateExecutionsContext struct { + ID string `json:"id"` + ImportPath string `json:"importPath"` + Instances []batchTemplateExecution `json:"instances"` + + r resource.Resource +} + +func (b batchTemplateExecutionsContext) CallbackJSON(i int) string { + mod := fmt.Sprintf("Mod%d", i) + + v := struct { + Mod string `json:"mod"` + batchTemplateExecutionsContext + }{ + mod, + b, + } + + bb, err := json.Marshal(v) + if err != nil { + panic(err) + } + s := string(bb) + + s = strings.ReplaceAll(s, fmt.Sprintf("%q", mod), mod) + + return s +} + +type batchTemplateExecution struct { + ID string `json:"id"` + Params any `json:"params"` +} + +type batchBuildOpts struct { + Callback resource.Resource + js.ExternalOptions `mapstructure:",squash"` +} + +type scriptsOne struct { + mu sync.Mutex + + id string + batches map[string]*scriptOne + + client *Namespace +} + +type scriptsMany struct { + mu sync.Mutex + + id string + batches map[string]*scriptMany + + client *Namespace +} + +type scriptOne struct { + mu sync.Mutex + id string + + resourceGetSet + instances map[string]scriptInstance + + client *Namespace +} + +type scriptInstance struct { + Params map[string]any +} + +type scriptManyItem struct { + mu sync.Mutex + id string + + resourceGetSet + instances map[string]scriptInstance + + client *Namespace +} + +type scriptMany struct { + mu sync.Mutex + id string + callback resource.Resource + + items map[string]*scriptManyItem + + client *Namespace +} + +type batcher struct { + mu sync.Mutex + id string + scriptOnes map[string]*scriptOne + scriptManys map[string]*scriptMany + + client *Namespace +} + +type resourceGetSet struct { + r resource.Resource +} + +func (r *resourceGetSet) Dir() string { + return path.Dir(r.r.(resource.PathProvider).Path()) +} + +func (r *resourceGetSet) GetResource() resource.Resource { + if resource.StaleVersion(r.r) > 0 { + // Allow the client to set a new resource. + return nil + } + return r.r +} + +func (r *resourceGetSet) SetResource(res resource.Resource) string { + if res == nil { + panic("resource not set") + } + r.r = res + return "" +} + +var ( + _ resource.StaleInfo = (*Package)(nil) + _ identity.IsProbablyDependencyProvider = (*Package)(nil) + _ identity.Identity = (*Package)(nil) +) + +// TODO1 names. +type Package struct { + outDir string + id string + staleVersion uint32 + b *batcher + Groups map[string]resource.Resources +} + +func (p *Package) IdentifierBase() string { + return p.id +} + +func (p *Package) StaleVersion() uint32 { + p.b.mu.Lock() + defer p.b.mu.Unlock() + if p.staleVersion == 0 { + p.staleVersion = p.calculateStaleVersion() + } + return p.staleVersion +} + +func (p *Package) IsProbablyDependency(other identity.Identity) bool { + depsFinder := identity.NewFinder(identity.FinderConfig{}) + var b bool + p.forEeachResource(func(rr resource.Resource) bool { + identity.WalkIdentitiesShallow(other, func(level int, left identity.Identity) bool { + identity.WalkIdentitiesShallow(rr, func(level int, right identity.Identity) bool { + if i := depsFinder.Contains(left, right, -1); i > 0 { + b = true + } + return b + }) + return b + }) + return b + }) + + // TODO1 why is this called twice on change? + + return b +} + +func (p *Package) forEeachResource(f func(r resource.Resource) bool) { + for _, v := range p.b.scriptManys { + if b := func() bool { + v.mu.Lock() + defer v.mu.Unlock() + if v.callback != nil { + if f(v.callback) { + return true + } + } + for _, vv := range v.items { + vv.mu.Lock() + defer vv.mu.Unlock() + if f(vv.r) { + return true + } + } + return false + }(); b { + return + } + } + + for _, v := range p.b.scriptOnes { + if b := func() bool { + v.mu.Lock() + defer v.mu.Unlock() + if f(v.r) { + return true + } + return false + }(); b { + return + } + } +} + +func (p *Package) calculateStaleVersion() uint32 { + // Return the first 0 zero value of the resources in this bundle. + var i uint32 + p.forEeachResource(func(r resource.Resource) bool { + if i = resource.StaleVersion(r); i > 0 { + return true + } + return false + }) + + return i +} + +func (b *batcher) Build() (*Package, error) { + key := dynacache.CleanKey(b.id + ".js") + p, err := b.client.bundlesCache.GetOrCreate(key, func(string) (*Package, error) { + return b.build() + }) + if err != nil { + return nil, err + } + + if p.b != b { + panic("bundler mismatch") + } + + return p, nil +} + +func (b *batcher) build() (*Package, error) { + defer herrors.Recover() // TODO1 + b.mu.Lock() + defer b.mu.Unlock() + + keyPath := b.id + + importResource := make(map[string]resource.Resource) + resultResource := make(map[string]resource.Resource) + pathGroup := make(map[string]string) + var entryPoints []string + addResource := func(group, pth string, r resource.Resource, isResult bool) { + fmt.Println("addEntryPoint", group, pth, isResult) + pathGroup[pth] = group + importResource[pth] = r + if isResult { + resultResource[pth] = r + } + entryPoints = append(entryPoints, pth) + } + + if len(b.scriptOnes) > 0 { + for k, v := range b.scriptOnes { + if v.r == nil { + return nil, fmt.Errorf("resource not set for %q", k) + } + keyPath := keyPath + "_" + k + resourcePath := paths.AddLeadingSlash(keyPath + v.r.MediaType().FirstSuffix.FullSuffix) + addResource(k, resourcePath, v.r, true) + + } + } + + if len(b.scriptManys) > 0 { + for k, v := range b.scriptManys { + keyPath := keyPath + "_" + k + + bopts := batchBuildOpts{ + Callback: v.callback, + } + var callbackImpPath string + if bopts.Callback != nil { + callbackImpPath = paths.AddLeadingSlash(keyPath + "_callback" + bopts.Callback.MediaType().FirstSuffix.FullSuffix) + addResource(k, callbackImpPath, bopts.Callback, false) + } + + t := &batchTemplateContext{ + keyPath: keyPath, + ID: v.id, + CallbackImportPath: callbackImpPath, + } + + for kk, vv := range v.items { + if vv.r == nil { + // TODO1 others. + return nil, fmt.Errorf("resource not set for %q", kk) + } + keyPath := keyPath + "_" + kk + const namespace = "@hugo-virtual" + impPath := path.Join(namespace, vv.Dir(), keyPath+vv.r.MediaType().FirstSuffix.FullSuffix) + bt := batchTemplateExecutionsContext{ + ID: kk, + r: vv.r, + ImportPath: impPath, + } + importResource[bt.ImportPath] = vv.r + for kkk, vvv := range vv.instances { + bt.Instances = append(bt.Instances, batchTemplateExecution{ID: kkk, Params: vvv.Params}) + sort.Slice(bt.Instances, func(i, j int) bool { + return bt.Instances[i].ID < bt.Instances[j].ID + }) + } + t.Modules = append(t.Modules, bt) + } + sort.Slice(t.Modules, func(i, j int) bool { + return t.Modules[i].ID < t.Modules[j].ID + }) + + r, s, err := b.client.buildBatch(t) + if err != nil { + return nil, err + } + addResource(v.id, s, r, true) + } + } + + target := "es2018" + + conf := b.client.d.Conf + absPublishDir := b.client.d.AbsPublishDir + mediaTypes := b.client.d.ResourceSpec.MediaTypes() + cssMt, _, _ := mediaTypes.GetFirstBySuffix("css") + + // TODO1 remove on close? + fmt.Println(conf.Dirs().CacheDir) + cacheDir := filepath.Join(b.client.d.SourceSpec.Cfg.Dirs().CacheDir, "_jsbatch") + if err := os.Mkdir(cacheDir, 0o777); err != nil && !herrors.IsExist(err) { + return nil, err + } + outDir, err := os.MkdirTemp(cacheDir, "jsbatch") + if err != nil { + return nil, err + } + + jopts := js.Options{ + ExternalOptions: js.ExternalOptions{ + Format: "esm", + Target: target, + Defines: map[string]any{ + //"process.env.NODE_ENV": `"development"`, + }, + }, + InternalOptions: js.InternalOptions{ + OutDir: outDir, + Write: true, + AllowOverwrite: true, + Splitting: true, + ImportOnResolveFunc: func(imp string) string { + if _, found := importResource[imp]; found { + return imp + } + return "" + }, + ImportOnLoadFunc: func(imp string) string { + if r, found := importResource[imp]; found { + content, err := r.(resource.ContentProvider).Content(context.Background()) // TODO1 + if err != nil { + panic(err) + } + return cast.ToString(content) + } + + return "" + }, + EntryPoints: entryPoints, + }, + } + + result, err := b.client.client.BuildBundle(jopts) + if err != nil { + return nil, err + } + + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + m := fromJSONToMeta(cwd, result.Metafile) + + groups := make(map[string]resource.Resources) + + // TODO1 + addFoo := func(filename, targetPath, group string, mt media.Type) error { + rd := resources.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return os.Open(filename) + }, + MediaType: mt, + TargetPath: targetPath, + // DependencyManager: idm, TODO1 + } + r, err := b.client.d.ResourceSpec.NewResource(rd) + if err != nil { + return err + } + + groups[group] = append(groups[group], r) + + return nil + } + + createAndAddResource3 := func(o esBuildResultMetaOutput) (bool, error) { + p := filepath.ToSlash(strings.TrimPrefix(o.filename, outDir)) + ext := path.Ext(p) + mt, _, found := mediaTypes.GetBySuffix(ext) + if !found { + return false, nil + } + groupPath := p + group, found := pathGroup[groupPath] + + if !found { + return false, nil + } + + if err := addFoo(o.filename, p, group, mt); err != nil { + return false, err + } + + if o.CSSBundle != "" { + p := filepath.ToSlash(strings.TrimPrefix(o.CSSBundle, outDir)) + if err := addFoo(o.CSSBundle, p, group, cssMt); err != nil { + return false, err + } + } + + return true, nil + } + + for _, o := range m.Outputs { + handled, err := createAndAddResource3(o) + if err != nil { + return nil, err + } + if !handled { + // Copy to destination. + p := strings.TrimPrefix(o.filename, outDir) + if err := hugio.CopyFile(hugofs.Os, o.filename, filepath.Join(absPublishDir, p)); err != nil { + return nil, fmt.Errorf("failed to copy %q to %q: %w", o.filename, absPublishDir, err) + } + } + } + + return &Package{ + outDir: outDir, + b: b, + id: path.Join(nsBundle, b.id), + Groups: groups, + }, nil +} + +type bundleResource string + +func (b bundleResource) Name() string { + return path.Base(string(b)) +} + +func (b bundleResource) Title() string { + return b.Name() +} + +func (b bundleResource) RelPermalink() string { + return "/js/bundles/mybundle" + string(b) +} + +func (b bundleResource) Permalink() string { + panic("not implemented") +} + +func (b bundleResource) ResourceType() string { + panic("not implemented") +} + +func (b bundleResource) MediaType() media.Type { + panic("not implemented") +} + +func (b bundleResource) Data() any { + panic("not implemented") +} + +func (b bundleResource) Err() resource.ResourceError { + return nil +} + +func (b bundleResource) Params() maps.Params { + panic("not implemented") +} + +const nsBundle = "__hugo-js-bundle" + +func (ns *Namespace) buildBatch(t *batchTemplateContext) (resource.Resource, string, error) { + var buf bytes.Buffer + if err := batchEsmCallbackTemplate.Execute(&buf, t); err != nil { + return nil, "", err + } + + s := paths.AddLeadingSlash(t.keyPath + ".js") + r, err := ns.createClient.FromString(s, buf.String()) + if err != nil { + return nil, "", err + } + + return r, s, nil +} + +//go:embed batch-esm-callback.gotmpl +var batchEsmCallbackTemplateString string +var batchEsmCallbackTemplate *template.Template + +func init() { + batchEsmCallbackTemplate = template.Must(template.New("batch-esm-callback").Parse(batchEsmCallbackTemplateString)) +} + +func fromJSONToMeta(cwd, s string) esBuildResultMeta { + var m esBuildResultMeta + if err := json.Unmarshal([]byte(s), &m); err != nil { + panic(err) + } + if err := m.Compile(cwd); err != nil { + panic(err) + } + + return m +} + +type esBuildResultMeta struct { + Outputs map[string]esBuildResultMetaOutput + + // Compiled values. + cssBundleEntryPoint map[string]esBuildResultMetaOutput +} + +func (e *esBuildResultMeta) Compile(cwd string) error { + // Rewrite the paths to be absolute. + // See https://github.com/evanw/esbuild/issues/338 + outputs := make(map[string]esBuildResultMetaOutput) + for k, v := range e.Outputs { + filename := filepath.Join(cwd, k) + if err := v.Compile(filename); err != nil { + return err + } + if v.CSSBundle != "" { + v.CSSBundle = filepath.Join(cwd, v.CSSBundle) + } + outputs[filename] = v + } + e.Outputs = outputs + + e.cssBundleEntryPoint = make(map[string]esBuildResultMetaOutput) + for _, v := range e.Outputs { + if v.CSSBundle != "" { + e.cssBundleEntryPoint[v.CSSBundle] = v + } + } + return nil +} + +type esBuildResultMetaOutput struct { + Bytes int64 + Exports []string + Imports []esBuildResultMetaOutputImport + EntryPoint string + CSSBundle string + + // compiled values. + filename string +} + +func (e *esBuildResultMetaOutput) Compile(filename string) error { + e.filename = filename + return nil +} + +type esBuildResultMetaOutputImport struct { + Path string + Kind string +} diff --git a/tpl/js/batch_integration_test.go b/tpl/js/batch_integration_test.go new file mode 100644 index 00000000000..1fb9291c3cf --- /dev/null +++ b/tpl/js/batch_integration_test.go @@ -0,0 +1,154 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package js_test + +import ( + "testing" + + "github.com/gohugoio/hugo/hugolib" +) + +func TestBatch(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "page"] +baseURL = "https://example.com" +-- package.json -- +{ + "devDependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + } +} +-- assets/js/reactcallback.js -- +import * as ReactDOM from 'react-dom/client'; +import * as React from 'react'; + +export default function Callback(modules) { + for (const module of modules) { + for (const instance of module.instances) { + /* This is a convention in this project. */ + let elId = §§${module.id}-${instance.id}§§; + let el = document.getElementById(elId); + if (!el) { + console.warn(§§Element with id ${elId} not found§§); + continue; + } + const root = ReactDOM.createRoot(el); + const reactEl = React.createElement(module.mod, instance.params); + root.render(reactEl); + } + } +} + +-- assets/js/button.css -- +button { + background-color: red; +} +-- assets/js/react1.jsx -- +import * as React from "react"; +import './button.css' + + +window.React1 = React; + +let text = 'Click me' + +export default function MyButton() { + return ( + + ) +} +-- assets/js/react2.jsx -- +import * as React from "react"; + +window.React2 = React; + +let text = 'Click me, too!' + +export default function MyOtherButton() { + return ( + + ) +} +-- assets/js/main1.js -- +import * as React from "react"; + +console.log('main1.React', React) + +-- assets/js/main2.js -- +import * as React from "react"; + +console.log('main2.React', React) + +-- layouts/index.html -- +Home. +{{ $bundle := (js.Batch "mybundle" .Store) }} +{{ with $bundle.UseScript "main1" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/main1.js") }} + {{ end }} + {{ .AddInstance "foo" (dict "title" "Main1 Instance") }} +{{ end }} + {{ with $bundle.UseScript "main2" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/main2.js") }} + {{ end }} + {{ .AddInstance "foo" (dict "title" "Main2 Instance") }} +{{ end }} +{{ with $bundle.UseScriptGroup "reactbatch" }} + {{ if not .GetCallback }} + {{ .SetCallback (resources.Get "js/reactcallback.js") }} + {{ end }} + {{ with .UseScript "r1" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/react1.jsx") }} + {{ end }} + {{ .AddInstance "i1" (dict "title" "Instance 1") }} + {{ .AddInstance "i2" (dict "title" "Instance 2") }} + {{ end }} + {{ with .UseScript "r2" }} + {{ if not .GetResource }} + {{ .SetResource (resources.Get "js/react2.jsx") }} + {{ end }} + {{ .AddInstance "i1" (dict "title" "Instance 2-1") }} + {{ end }} +{{ end }} +{{ range $k, $v := $bundle.Build.Groups }} + {{ range . }} + {{ $k }}: {{ .RelPermalink }} +{{ end }} +{{ end }}} +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + NeedsOsFS: true, + NeedsNpmInstall: true, + TxtarString: files, + // PrintAndKeepTempDir: true, + }).Build() + + // b.AssertPublishDir("sadf") + + b.AssertFileContent("public/index.html", ` +asdfsdf + + + `) +} + +// TODO1 make instance into a map with params as only key (for now) diff --git a/tpl/js/js.go b/tpl/js/js.go index c68e0af9272..95744432cdb 100644 --- a/tpl/js/js.go +++ b/tpl/js/js.go @@ -17,9 +17,11 @@ package js import ( "errors" + "github.com/gohugoio/hugo/cache/dynacache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/resources/resource_transformers/babel" "github.com/gohugoio/hugo/resources/resource_transformers/js" "github.com/gohugoio/hugo/tpl/internal/resourcehelpers" @@ -31,15 +33,27 @@ func New(deps *deps.Deps) *Namespace { return &Namespace{} } return &Namespace{ - client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), - babelClient: babel.New(deps.ResourceSpec), + d: deps, + bundlesCache: dynacache.GetOrCreatePartition[string, *Package]( + deps.MemCache, + "/jsb1", + dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 10}, + ), + client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), + createClient: create.New(deps.ResourceSpec), + babelClient: babel.New(deps.ResourceSpec), } } // Namespace provides template functions for the "js" namespace. type Namespace struct { - client *js.Client - babelClient *babel.Client + d *deps.Deps + + client *js.Client + createClient *create.Client + babelClient *babel.Client + + bundlesCache *dynacache.Partition[string, *Package] } // Build processes the given Resource with ESBuild. @@ -65,7 +79,7 @@ func (ns *Namespace) Build(args ...any) (resource.Resource, error) { m = map[string]any{"targetPath": targetPath} } - return ns.client.Process(r, m) + return ns.client.ProcessExernal(r, m) } // Babel processes the given Resource with Babel.