From f4b8d633ea8b3c0677751bd523eac680f16d310f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 30 Oct 2024 18:10:09 +0100 Subject: [PATCH 1/2] Fix some RenderShortcodes error cases This issue fixes two cases where `{{__hugo_ctx` artifacts were left in the rendered output: 1. Inclusion when `.RenderShortcodes` is wrapped in HTML. 2. Inclusion of Markdown file without a trailing newline in some cases. Closes #12854 Updates #12998 --- common/constants/constants.go | 1 + hugolib/collections_test.go | 16 +-- hugolib/integrationtest_builder.go | 23 ++++ hugolib/menu_test.go | 124 +++--------------- hugolib/page.go | 19 ++- hugolib/rendershortcodes_test.go | 112 ++++++++++++++++ hugolib/shortcode_page.go | 4 + markup/goldmark/convert.go | 2 +- markup/goldmark/goldmark_integration_test.go | 2 +- markup/goldmark/hugocontext/hugocontext.go | 111 +++++++++++++--- .../goldmark/hugocontext/hugocontext_test.go | 2 +- resources/page/page.go | 2 + .../page/page_markup_integration_test.go | 6 +- 13 files changed, 283 insertions(+), 141 deletions(-) diff --git a/common/constants/constants.go b/common/constants/constants.go index f8f057e053b..752aef72c3c 100644 --- a/common/constants/constants.go +++ b/common/constants/constants.go @@ -21,6 +21,7 @@ const ( ErrRemoteGetCSV = "error-remote-getcsv" WarnFrontMatterParamsOverrides = "warning-frontmatter-params-overrides" + WarnRenderShortcodesInHTML = "warning-rendershortcodes-in-html" ) // Field/method names with special meaning. diff --git a/hugolib/collections_test.go b/hugolib/collections_test.go index fff57337f78..a8c817becac 100644 --- a/hugolib/collections_test.go +++ b/hugolib/collections_test.go @@ -71,9 +71,9 @@ tags_weight: %d {{ $pageGroups := slice $cool $blue }} {{ $weighted := slice $wp1 $wp2 }} -{{ printf "pages:%d:%T:%v/%v" (len $pages) $pages (index $pages 0) (index $pages 1) }} -{{ printf "pageGroups:%d:%T:%v/%v" (len $pageGroups) $pageGroups (index (index $pageGroups 0).Pages 0) (index (index $pageGroups 1).Pages 0)}} -{{ printf "weightedPages:%d::%T:%v" (len $weighted) $weighted $weighted | safeHTML }} +{{ printf "pages:%d:%T:%s|%s" (len $pages) $pages (index $pages 0).Path (index $pages 1).Path }} +{{ printf "pageGroups:%d:%T:%s|%s" (len $pageGroups) $pageGroups (index (index $pageGroups 0).Pages 0).Path (index (index $pageGroups 1).Pages 0).Path}} +{{ printf "weightedPages:%d:%T" (len $weighted) $weighted | safeHTML }} `) b.CreateSites().Build(BuildCfg{}) @@ -82,9 +82,9 @@ tags_weight: %d c.Assert(len(b.H.Sites[0].RegularPages()), qt.Equals, 2) b.AssertFileContent("public/index.html", - "pages:2:page.Pages:Page(/page1)/Page(/page2)", - "pageGroups:2:page.PagesGroup:Page(/page1)/Page(/page2)", - `weightedPages:2::page.WeightedPages:[WeightedPage(10,"Page") WeightedPage(20,"Page")]`) + "pages:2:page.Pages:/page1|/page2", + "pageGroups:2:page.PagesGroup:/page1|/page2", + `weightedPages:2:page.WeightedPages`) } func TestUnionFunc(t *testing.T) { @@ -189,7 +189,7 @@ tags_weight: %d {{ $appendStrings := slice "a" "b" | append "c" "d" "e" }} {{ $appendStringsSlice := slice "a" "b" "c" | append (slice "c" "d") }} -{{ printf "pages:%d:%T:%v/%v" (len $pages) $pages (index $pages 0) (index $pages 1) }} +{{ printf "pages:%d:%T:%s|%s" (len $pages) $pages (index $pages 0).Path (index $pages 1).Path }} {{ printf "appendPages:%d:%T:%v/%v" (len $appendPages) $appendPages (index $appendPages 0).Kind (index $appendPages 8).Kind }} {{ printf "appendStrings:%T:%v" $appendStrings $appendStrings }} {{ printf "appendStringsSlice:%T:%v" $appendStringsSlice $appendStringsSlice }} @@ -207,7 +207,7 @@ tags_weight: %d c.Assert(len(b.H.Sites[0].RegularPages()), qt.Equals, 2) b.AssertFileContent("public/index.html", - "pages:2:page.Pages:Page(/page2)/Page(/page1)", + "pages:2:page.Pages:/page2|/page1", "appendPages:9:page.Pages:home/page", "appendStrings:[]string:[a b c d e]", "appendStringsSlice:[]string:[a b c c d]", diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index b806ad7c143..5dc13592ff2 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -2,6 +2,7 @@ package hugolib import ( "bytes" + "context" "encoding/base64" "errors" "fmt" @@ -32,6 +33,7 @@ import ( "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/hugofs" "github.com/spf13/afero" + "github.com/spf13/cast" "golang.org/x/text/unicode/norm" "golang.org/x/tools/txtar" ) @@ -294,6 +296,12 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s } } +func (s *IntegrationTestBuilder) AssertFileContentEquals(filename string, match string) { + s.Helper() + content := s.FileContent(filename) + s.Assert(content, qt.Equals, match, qt.Commentf(match)) +} + func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches ...string) { s.Helper() content := s.FileContent(filename) @@ -302,6 +310,16 @@ func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches } } +func (s *IntegrationTestBuilder) AssertNoRenderShortcodesArtifacts() { + s.Helper() + for _, p := range s.H.Pages() { + content, err := p.Content(context.Background()) + s.Assert(err, qt.IsNil) + comment := qt.Commentf("Page: %s\n%s", p.Path(), content) + s.Assert(strings.Contains(cast.ToString(content), "__hugo_ctx"), qt.IsFalse, comment) + } +} + func (s *IntegrationTestBuilder) AssertPublishDir(matches ...string) { s.AssertFs(s.fs.PublishDir, matches...) } @@ -835,6 +853,11 @@ type IntegrationTestConfig struct { // The files to use on txtar format, see // https://pkg.go.dev/golang.org/x/exp/cmd/txtar + // There are some conentions used in this test setup. + // - §§§ can be used to wrap code fences. + // - §§ can be used to wrap multiline strings. + // - filenames prefixed with sourcefilename: will be read from the file system relative to the current dir. + // - filenames with a .png or .jpg extension will be treated as binary and base64 decoded. TxtarString string // COnfig to use as the base. We will also read the config from the txtar. diff --git a/hugolib/menu_test.go b/hugolib/menu_test.go index 304b4fbf4ea..6ee62771b7c 100644 --- a/hugolib/menu_test.go +++ b/hugolib/menu_test.go @@ -105,94 +105,6 @@ Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, "/sect3/|Sect3s|Sect3s|0|-|-|") } -// related issue #7594 -func TestMenusSort(t *testing.T) { - b := newTestSitesBuilder(t).WithSimpleConfigFile() - - b.WithTemplatesAdded("index.html", ` -{{ range $k, $v := .Site.Menus.main }} -Default1|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }} -{{ range $k, $v := .Site.Menus.main.ByWeight }} -ByWeight|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }} -{{ range $k, $v := (.Site.Menus.main.ByWeight).Reverse }} -Reverse|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }} -{{ range $k, $v := .Site.Menus.main }} -Default2|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }} -{{ range $k, $v := .Site.Menus.main.ByWeight }} -ByWeight|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }} -{{ range $k, $v := .Site.Menus.main }} -Default3|{{ $k }}|{{ $v.Weight }}|{{ $v.Name }}|{{ .URL }}|{{ $v.Page }}{{ end }} -`) - - b.WithContent("_index.md", ` ---- -title: Home -menu: - main: - weight: 100 ----`) - - b.WithContent("blog/A.md", ` ---- -title: "A" -menu: - main: - weight: 10 ---- -`) - - b.WithContent("blog/B.md", ` ---- -title: "B" -menu: - main: - weight: 20 ---- -`) - b.WithContent("blog/C.md", ` ---- -title: "C" -menu: - main: - weight: 30 ---- -`) - - b.Build(BuildCfg{}) - - b.AssertFileContent("public/index.html", - `Default1|0|10|A|/blog/a/|Page(/blog/a) - Default1|1|20|B|/blog/b/|Page(/blog/b) - Default1|2|30|C|/blog/c/|Page(/blog/c) - Default1|3|100|Home|/|Page(/) - - ByWeight|0|10|A|/blog/a/|Page(/blog/a) - ByWeight|1|20|B|/blog/b/|Page(/blog/b) - ByWeight|2|30|C|/blog/c/|Page(/blog/c) - ByWeight|3|100|Home|/|Page(/) - - Reverse|0|100|Home|/|Page(/) - Reverse|1|30|C|/blog/c/|Page(/blog/c) - Reverse|2|20|B|/blog/b/|Page(/blog/b) - Reverse|3|10|A|/blog/a/|Page(/blog/a) - - Default2|0|10|A|/blog/a/|Page(/blog/a) - Default2|1|20|B|/blog/b/|Page(/blog/b) - Default2|2|30|C|/blog/c/|Page(/blog/c) - Default2|3|100|Home|/|Page(/) - - ByWeight|0|10|A|/blog/a/|Page(/blog/a) - ByWeight|1|20|B|/blog/b/|Page(/blog/b) - ByWeight|2|30|C|/blog/c/|Page(/blog/c) - ByWeight|3|100|Home|/|Page(/) - - Default3|0|10|A|/blog/a/|Page(/blog/a) - Default3|1|20|B|/blog/b/|Page(/blog/b) - Default3|2|30|C|/blog/c/|Page(/blog/c) - Default3|3|100|Home|/|Page(/)`, - ) -} - func TestMenusFrontMatter(t *testing.T) { b := newTestSitesBuilder(t).WithSimpleConfigFile() @@ -437,8 +349,8 @@ url = "/blog/post3" commonTempl := ` Main: {{ len .Site.Menus.main }} {{ range .Site.Menus.main }} -{{ .Title }}|HasMenuCurrent: {{ $.HasMenuCurrent "main" . }}|Page: {{ .Page }} -{{ .Title }}|IsMenuCurrent: {{ $.IsMenuCurrent "main" . }}|Page: {{ .Page }} +{{ .Title }}|HasMenuCurrent: {{ $.HasMenuCurrent "main" . }}|Page: {{ .Page.Path }} +{{ .Title }}|IsMenuCurrent: {{ $.IsMenuCurrent "main" . }}|Page: {{ .Page.Path }} {{ end }} ` @@ -494,34 +406,34 @@ title: "Contact: With No Menu Defined" b.AssertFileContent("public/index.html", ` Main: 5 -Home|HasMenuCurrent: false|Page: Page(/) -Blog|HasMenuCurrent: false|Page: Page(/blog) -My Post 2: With Menu Defined|HasMenuCurrent: false|Page: Page(/blog/post2) -My Post 3|HasMenuCurrent: false|Page: Page(/blog/post3) -Contact Us|HasMenuCurrent: false|Page: Page(/contact) +Home|HasMenuCurrent: false|Page: / +Blog|HasMenuCurrent: false|Page: /blog +My Post 2: With Menu Defined|HasMenuCurrent: false|Page: /blog/post2 +My Post 3|HasMenuCurrent: false|Page: /blog/post3 +Contact Us|HasMenuCurrent: false|Page: /contact `) b.AssertFileContent("public/blog/post1/index.html", ` -Home|HasMenuCurrent: false|Page: Page(/) -Blog|HasMenuCurrent: true|Page: Page(/blog) +Home|HasMenuCurrent: false|Page: / +Blog|HasMenuCurrent: true|Page: /blog `) b.AssertFileContent("public/blog/post2/index.html", ` -Home|HasMenuCurrent: false|Page: Page(/) -Blog|HasMenuCurrent: true|Page: Page(/blog) -Blog|IsMenuCurrent: false|Page: Page(/blog) +Home|HasMenuCurrent: false|Page: / +Blog|HasMenuCurrent: true|Page: /blog +Blog|IsMenuCurrent: false|Page: /blog `) b.AssertFileContent("public/blog/post3/index.html", ` -Home|HasMenuCurrent: false|Page: Page(/) -Blog|HasMenuCurrent: true|Page: Page(/blog) +Home|HasMenuCurrent: false|Page: / +Blog|HasMenuCurrent: true|Page: /blog `) b.AssertFileContent("public/contact/index.html", ` -Contact Us|HasMenuCurrent: false|Page: Page(/contact) -Contact Us|IsMenuCurrent: true|Page: Page(/contact) -Blog|HasMenuCurrent: false|Page: Page(/blog) -Blog|IsMenuCurrent: false|Page: Page(/blog) +Contact Us|HasMenuCurrent: false|Page: /contact +Contact Us|IsMenuCurrent: true|Page: /contact +Blog|HasMenuCurrent: false|Page: /blog +Blog|IsMenuCurrent: false|Page: /blog `) } diff --git a/hugolib/page.go b/hugolib/page.go index 7525ab67267..2bc1da044b4 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -16,7 +16,9 @@ package hugolib import ( "context" "fmt" + "path/filepath" "strconv" + "strings" "sync" "sync/atomic" @@ -358,7 +360,22 @@ func (p *pageState) Site() page.Site { } func (p *pageState) String() string { - return fmt.Sprintf("Page(%s)", p.Path()) + var sb strings.Builder + if p.File() != nil { + // The forward slashes even on Windows is motivated by + // getting stable tests. + // This information is meant for getting positional information in logs, + // so the direction of the slashes should not matter. + sb.WriteString(filepath.ToSlash(p.File().Filename())) + if p.File().IsContentAdapter() { + // Also include the path. + sb.WriteString(":") + sb.WriteString(p.Path()) + } + } else { + sb.WriteString(p.Path()) + } + return sb.String() } // IsTranslated returns whether this content file is translated to diff --git a/hugolib/rendershortcodes_test.go b/hugolib/rendershortcodes_test.go index 313c80a73fc..9a31b6536dc 100644 --- a/hugolib/rendershortcodes_test.go +++ b/hugolib/rendershortcodes_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "path/filepath" "strings" "testing" ) @@ -69,6 +70,7 @@ Content: {{ .Content }}| b := Test(t, files) + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/p1/index.html", "Fragments: [p1-h1 p2-h1 p2-h2 p2-h3 p2-withmarkdown p3-h1 p3-h2 p3-withmarkdown]|", "HasShortcode Level 1: true|", @@ -115,6 +117,7 @@ JSON: {{ .Content }} b := Test(t, files) + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/p1/index.html", "Myshort HTML") b.AssertFileContent("public/p1/index.json", "Myshort JSON") } @@ -147,9 +150,11 @@ Myshort Original. {{ .Content }} ` b := TestRunning(t, files) + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/p1/index.html", "Myshort Original.") b.EditFileReplaceAll("layouts/shortcodes/myshort.html", "Original", "Edited").Build() + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/p1/index.html", "Myshort Edited.") } @@ -192,12 +197,14 @@ Myshort Original. }, ).Build() + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/p1/index.html", "Original") b.EditFileReplaceFunc("content/p2.md", func(s string) string { return strings.Replace(s, "Original", "Edited", 1) }) b.Build() + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/p1/index.html", "Edited") } @@ -233,8 +240,10 @@ Myshort Original. ` b := TestRunning(t, files) + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/mysection/index.html", "p1-h1") b.EditFileReplaceAll("content/mysection/_index.md", "p1-h1", "p1-h1 Edited").Build() + b.AssertNoRenderShortcodesArtifacts() b.AssertFileContent("public/mysection/index.html", "p1-h1 Edited") } @@ -314,6 +323,8 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA b := Test(t, files) + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContent("public/markdown/index.html", // Images. "Image: /posts/p1/pixel1.png|\nImage: /posts/p1/pixel2.png|\n|\nImage: /markdown/pixel3.png|

\n|", @@ -333,3 +344,104 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA b.AssertFileContent("public/html/index.html", "! hugo_ctx") } + +// Issue 12854. +func TestRenderShortcodesWithHTML(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableLiveReload = true +disableKinds = ["home", "taxonomy", "term"] +markup.goldmark.renderer.unsafe = true +-- content/p1.md -- +--- +title: "p1" +--- +{{% include "p2" %}} +-- content/p2.md -- +--- +title: "p2" +--- +Hello world. Some **bold** text. Some Unicode: 神真美好. +-- layouts/shortcodes/include.html -- +{{ with site.GetPage (.Get 0) }} +
{{ .RenderShortcodes }}
+{{ end }} +-- layouts/_default/single.html -- +{{ .Content }} +` + + b := TestRunning(t, files, TestOptWarn()) + + b.AssertNoRenderShortcodesArtifacts() + b.AssertLogContains(filepath.ToSlash("WARN .RenderShortcodes detected inside HTML block in \"/content/p1.md\"; this may not be what you intended, see https://gohugo.io/methods/page/rendershortcodes/#limitations\nYou can suppress this warning by adding the following to your site configuration:\nignoreLogs = ['warning-rendershortcodes-in-html']")) + b.AssertFileContent("public/p1/index.html", "
Hello world. Some **bold** text. Some Unicode: 神真美好.\n
") + b.EditFileReplaceAll("content/p2.md", "Hello", "Hello Edited").Build() + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContent("public/p1/index.html", "
Hello Edited world. Some **bold** text. Some Unicode: 神真美好.\n
") +} + +func TestRenderShortcodesIncludeMarkdownFileWithoutTrailingNewline(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableLiveReload = true +disableKinds = ["home", "taxonomy", "term"] +markup.goldmark.renderer.unsafe = true +-- content/p1.md -- +--- +title: "p1" +--- +Content p1 id-1000.{{% include "p2" %}}{{% include "p3" %}} + +§§§ go +code_p1 +§§§ +§§§ go +code_p1_2 +§§§ + +§§§ go +code_p1_3 +§§§ +-- content/p2.md -- +--- +title: "p2" +--- +§§§ bash +code_p2 +§§§ +Foo. +-- content/p3.md -- +--- +title: "p3" +--- +§§§ php +code_p3 +§§§ +-- layouts/shortcodes/include.html -- +{{ with site.GetPage (.Get 0) -}} +{{ .RenderShortcodes -}} +{{ end -}} +-- layouts/_default/single.html -- +{{ .Content }} +-- layouts/_default/_markup/render-codeblock.html -- +{{ .Inner | safeHTML }} +` + + b := TestRunning(t, files, TestOptWarn()) + + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContentEquals("public/p1/index.html", "

Content p1 id-1000.

\ncode_p2

Foo.\n

\ncode_p3

\ncode_p1code_p1_2code_p1_3") + b.EditFileReplaceAll("content/p1.md", "id-1000.", "id-100.").Build() + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContentEquals("public/p1/index.html", "

Content p1 id-100.

\ncode_p2

Foo.\n

\ncode_p3

\ncode_p1code_p1_2code_p1_3") + b.EditFileReplaceAll("content/p2.md", "code_p2", "codep2").Build() + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContentEquals("public/p1/index.html", "

Content p1 id-100.

\ncodep2

Foo.\n

\ncode_p3

\ncode_p1code_p1_2code_p1_3") + b.EditFileReplaceAll("content/p3.md", "code_p3", "code_p3_edited").Build() + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContentEquals("public/p1/index.html", "

Content p1 id-100.

\ncodep2

Foo.\n

\ncode_p3_edited

\ncode_p1code_p1_2code_p1_3") +} diff --git a/hugolib/shortcode_page.go b/hugolib/shortcode_page.go index 8030b02851a..3d27cc93ca3 100644 --- a/hugolib/shortcode_page.go +++ b/hugolib/shortcode_page.go @@ -125,3 +125,7 @@ func newPageForRenderHook(p *pageState) page.Page { func (p *pageForRenderHooks) Unwrapv() any { return p.p } + +func (p *pageForRenderHooks) String() string { + return p.p.String() +} diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go index 5c31eee40d9..ea3bbc4ae9b 100644 --- a/markup/goldmark/convert.go +++ b/markup/goldmark/convert.go @@ -106,7 +106,7 @@ func newMarkdown(pcfg converter.ProviderConfig) goldmark.Markdown { renderer.WithNodeRenderers(util.Prioritized(emoji.NewHTMLRenderer(), 200))) var ( extensions = []goldmark.Extender{ - hugocontext.New(), + hugocontext.New(pcfg.Logger), newLinks(cfg), newTocExtension(tocRendererOptions), blockquotes.New(), diff --git a/markup/goldmark/goldmark_integration_test.go b/markup/goldmark/goldmark_integration_test.go index c691435eee7..19b18692e28 100644 --- a/markup/goldmark/goldmark_integration_test.go +++ b/markup/goldmark/goldmark_integration_test.go @@ -575,7 +575,7 @@ sc3_begin|{{ .Inner }}|sc3_end // Issue #7332 ":x:\n", // Issue #11587 - "

✔️

", + "

✔️\n

", // Should not be converted to emoji "sc1_begin|:smiley:|sc1_end", // Should be converted to emoji diff --git a/markup/goldmark/hugocontext/hugocontext.go b/markup/goldmark/hugocontext/hugocontext.go index b9c548dac5e..223c30c912e 100644 --- a/markup/goldmark/hugocontext/hugocontext.go +++ b/markup/goldmark/hugocontext/hugocontext.go @@ -16,20 +16,24 @@ package hugocontext import ( "bytes" "fmt" + "regexp" "strconv" "github.com/gohugoio/hugo/bufferpool" + "github.com/gohugoio/hugo/common/constants" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/markup/goldmark/internal/render" "github.com/yuin/goldmark" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/parser" "github.com/yuin/goldmark/renderer" + "github.com/yuin/goldmark/renderer/html" "github.com/yuin/goldmark/text" "github.com/yuin/goldmark/util" ) -func New() goldmark.Extender { - return &hugoContextExtension{} +func New(logger loggers.Logger) goldmark.Extender { + return &hugoContextExtension{logger: logger} } // Wrap wraps the given byte slice in a Hugo context that used to determine the correct Page @@ -37,14 +41,19 @@ func New() goldmark.Extender { func Wrap(b []byte, pid uint64) string { buf := bufferpool.GetBuffer() defer bufferpool.PutBuffer(buf) - buf.Write(prefix) + buf.Write(hugoCtxPrefix) buf.WriteString(" pid=") buf.WriteString(strconv.FormatUint(pid, 10)) - buf.Write(endDelim) + buf.Write(hugoCtxEndDelim) buf.WriteByte('\n') buf.Write(b) - buf.Write(prefix) - buf.Write(closingDelimAndNewline) + // To make sure that we're able to parse it, make sure it ends with a newline. + if len(b) > 0 && b[len(b)-1] != '\n' { + buf.WriteByte('\n') + } + buf.Write(hugoCtxPrefix) + buf.Write(hugoCtxClosingDelim) + buf.WriteByte('\n') return buf.String() } @@ -89,45 +98,100 @@ func (h *HugoContext) Kind() ast.NodeKind { } var ( - prefix = []byte("{{__hugo_ctx") - endDelim = []byte("}}") - closingDelimAndNewline = []byte("/}}\n") + hugoCtxPrefix = []byte("{{__hugo_ctx") + hugoCtxEndDelim = []byte("}}") + hugoCtxClosingDelim = []byte("/}}") + hugoCtxRe = regexp.MustCompile(`{{__hugo_ctx( pid=\d+)?/?}}\n?`) ) var _ parser.InlineParser = (*hugoContextParser)(nil) type hugoContextParser struct{} -func (s *hugoContextParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { - line, _ := block.PeekLine() - if !bytes.HasPrefix(line, prefix) { +func (a *hugoContextParser) Trigger() []byte { + return []byte{'{'} +} + +func (s *hugoContextParser) Parse(parent ast.Node, reader text.Reader, pc parser.Context) ast.Node { + line, _ := reader.PeekLine() + if !bytes.HasPrefix(line, hugoCtxPrefix) { return nil } - end := bytes.Index(line, endDelim) + end := bytes.Index(line, hugoCtxEndDelim) if end == -1 { return nil } - block.Advance(end + len(endDelim) + 1) // +1 for the newline + reader.Advance(end + len(hugoCtxEndDelim) + 1) // +1 for the newline if line[end-1] == '/' { return &HugoContext{Closing: true} } - attrBytes := line[len(prefix)+1 : end] + attrBytes := line[len(hugoCtxPrefix)+1 : end] h := &HugoContext{} h.parseAttrs(attrBytes) return h } -func (a *hugoContextParser) Trigger() []byte { - return []byte{'{'} +type hugoContextRenderer struct { + logger loggers.Logger + html.Config } -type hugoContextRenderer struct{} +func (r *hugoContextRenderer) SetOption(name renderer.OptionName, value any) { + r.Config.SetOption(name, value) +} func (r *hugoContextRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) { reg.Register(kindHugoContext, r.handleHugoContext) + reg.Register(ast.KindHTMLBlock, r.renderHTMLBlock) +} + +func (r *hugoContextRenderer) stripHugoCtx(b []byte) ([]byte, bool) { + if !bytes.Contains(b, hugoCtxPrefix) { + return b, false + } + return hugoCtxRe.ReplaceAll(b, nil), true +} + +func (r *hugoContextRenderer) renderHTMLBlock( + w util.BufWriter, source []byte, node ast.Node, entering bool, +) (ast.WalkStatus, error) { + n := node.(*ast.HTMLBlock) + if entering { + if r.Unsafe { + l := n.Lines().Len() + for i := 0; i < l; i++ { + line := n.Lines().At(i) + linev := line.Value(source) + var stripped bool + linev, stripped = r.stripHugoCtx(linev) + if stripped { + var p any + ctx, ok := w.(*render.Context) + if ok { + p, _ = render.GetPageAndPageInner(ctx) + } + r.logger.Warnidf(constants.WarnRenderShortcodesInHTML, ".RenderShortcodes detected inside HTML block in %q; this may not be what you intended, see https://gohugo.io/methods/page/rendershortcodes/#limitations", p) + } + + r.Writer.SecureWrite(w, linev) + } + } else { + _, _ = w.WriteString("\n") + } + } else { + if n.HasClosure() { + if r.Unsafe { + closure := n.ClosureLine + r.Writer.SecureWrite(w, closure.Value(source)) + } else { + _, _ = w.WriteString("\n") + } + } + } + return ast.WalkContinue, nil } func (r *hugoContextRenderer) handleHugoContext(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) { @@ -148,7 +212,9 @@ func (r *hugoContextRenderer) handleHugoContext(w util.BufWriter, source []byte, return ast.WalkContinue, nil } -type hugoContextExtension struct{} +type hugoContextExtension struct { + logger loggers.Logger +} func (a *hugoContextExtension) Extend(m goldmark.Markdown) { m.Parser().AddOptions( @@ -159,7 +225,12 @@ func (a *hugoContextExtension) Extend(m goldmark.Markdown) { m.Renderer().AddOptions( renderer.WithNodeRenderers( - util.Prioritized(&hugoContextRenderer{}, 50), + util.Prioritized(&hugoContextRenderer{ + logger: a.logger, + Config: html.Config{ + Writer: html.DefaultWriter, + }, + }, 50), ), ) } diff --git a/markup/goldmark/hugocontext/hugocontext_test.go b/markup/goldmark/hugocontext/hugocontext_test.go index 4a6eb80f597..62769f4d0c3 100644 --- a/markup/goldmark/hugocontext/hugocontext_test.go +++ b/markup/goldmark/hugocontext/hugocontext_test.go @@ -24,7 +24,7 @@ func TestWrap(t *testing.T) { b := []byte("test") - c.Assert(Wrap(b, 42), qt.Equals, "{{__hugo_ctx pid=42}}\ntest{{__hugo_ctx/}}\n") + c.Assert(Wrap(b, 42), qt.Equals, "{{__hugo_ctx pid=42}}\ntest\n{{__hugo_ctx/}}\n") } func BenchmarkWrap(b *testing.B) { diff --git a/resources/page/page.go b/resources/page/page.go index 4cda8d31fce..20525669c1e 100644 --- a/resources/page/page.go +++ b/resources/page/page.go @@ -17,6 +17,7 @@ package page import ( "context" + "fmt" "html/template" "github.com/gohugoio/hugo/markup/converter" @@ -180,6 +181,7 @@ type Page interface { ContentProvider TableOfContentsProvider PageWithoutContent + fmt.Stringer } type PageFragment interface { diff --git a/resources/page/page_markup_integration_test.go b/resources/page/page_markup_integration_test.go index 010a9d729a5..42509921586 100644 --- a/resources/page/page_markup_integration_test.go +++ b/resources/page/page_markup_integration_test.go @@ -161,13 +161,13 @@ includecontent: {{ hugo.Context.MarkupScope }}|{{ $p.Markup.Render.Content }}| b := hugolib.Test(t, files) - b.AssertFileContent("public/p1/index.html", "Render heading: title: P1 scope: |", "Foo scope: |") + b.AssertFileContentExact("public/p1/index.html", "Render heading: title: P1 scope: |", "Foo scope: |") - b.AssertFileContent("public/index.html", + b.AssertFileContentExact("public/index.html", + "Begin:\nincludecontent: home|Render heading: title: P3 scope: home|Foo scope: home|\n|\n:End", "Render heading: title: P1 scope: home|", "Foo scope: home|", "Begin:\nincluderendershortcodes: home|

\nRender heading: title: P2 scope: home|

|:End", - "Begin:\nincludecontent: home|Render heading: title: P3 scope: home|Foo scope: home|\n|\n:End", ) } From ef6b27fdaab76193f3cf69c112db27b7d66125a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 3 Nov 2024 10:41:34 +0100 Subject: [PATCH 2/2] Fix stale pages on rebuilds in GetPage with short refs Fixes #13004 --- hugolib/content_map_page.go | 12 +++++++---- hugolib/rendershortcodes_test.go | 35 ++++++++++++++++++++++++++++++++ hugolib/site.go | 1 + lazy/init.go | 2 +- lazy/once.go | 12 +++++------ 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index c3f06a592ba..5e8646b21c4 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -37,6 +37,7 @@ import ( "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/pagesfromdata" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/resources" @@ -109,6 +110,11 @@ type pageMap struct { cfg contentMapConfig } +// Invoked on rebuilds. +func (m *pageMap) Reset() { + m.pageReverseIndex.Reset() +} + // pageTrees holds pages and resources in a tree structure for all sites/languages. // Each site gets its own tree set via the Shape method. type pageTrees struct { @@ -958,9 +964,7 @@ type contentTreeReverseIndex struct { } func (c *contentTreeReverseIndex) Reset() { - c.contentTreeReverseIndexMap = &contentTreeReverseIndexMap{ - m: make(map[any]contentNodeI), - } + c.init.ResetWithLock().Unlock() } func (c *contentTreeReverseIndex) Get(key any) contentNodeI { @@ -972,7 +976,7 @@ func (c *contentTreeReverseIndex) Get(key any) contentNodeI { } type contentTreeReverseIndexMap struct { - init sync.Once + init lazy.OnceMore m map[any]contentNodeI } diff --git a/hugolib/rendershortcodes_test.go b/hugolib/rendershortcodes_test.go index 9a31b6536dc..0eebf46eb3e 100644 --- a/hugolib/rendershortcodes_test.go +++ b/hugolib/rendershortcodes_test.go @@ -445,3 +445,38 @@ code_p3 b.AssertNoRenderShortcodesArtifacts() b.AssertFileContentEquals("public/p1/index.html", "

Content p1 id-100.

\ncodep2

Foo.\n

\ncode_p3_edited

\ncode_p1code_p1_2code_p1_3") } + +// Issue 13004. +func TestRenderShortcodesIncludeShortRefEdit(t *testing.T) { + t.Parallel() + + files := ` +-- hugo.toml -- +disableLiveReload = true +disableKinds = ["home", "taxonomy", "term", "section", "rss", "sitemap", "robotsTXT", "404"] +-- content/first/p1.md -- +--- +title: "p1" +--- +## p1-h1 +{{% include "p2" %}} +-- content/second/p2.md -- +--- +title: "p2" +--- +### p2-h1 + +This is some **markup**. +-- layouts/shortcodes/include.html -- +{{ $p := site.GetPage (.Get 0) -}} +{{ $p.RenderShortcodes -}} +-- layouts/_default/single.html -- +{{ .Content }} +` + b := TestRunning(t, files) + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContentEquals("public/first/p1/index.html", "

p1-h1

\n

\n

p2-h1

\n

This is some markup.\n

\n") + b.EditFileReplaceAll("content/second/p2.md", "p2-h1", "p2-h1-edited").Build() + b.AssertNoRenderShortcodesArtifacts() + b.AssertFileContentEquals("public/first/p1/index.html", "

p1-h1

\n

\n

p2-h1-edited

\n

This is some markup.\n

\n") +} diff --git a/hugolib/site.go b/hugolib/site.go index 24ee5dcc506..c5a4956e228 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1351,6 +1351,7 @@ func (s *Site) getLanguagePermalinkLang(alwaysInSubDir bool) string { func (s *Site) resetBuildState(sourceChanged bool) { s.relatedDocsHandler = s.relatedDocsHandler.Clone() s.init.Reset() + s.pageMap.Reset() } func (s *Site) errorCollator(results <-chan error, errs chan<- error) { diff --git a/lazy/init.go b/lazy/init.go index 7b88a53518b..bef3867a9a0 100644 --- a/lazy/init.go +++ b/lazy/init.go @@ -36,7 +36,7 @@ type Init struct { prev *Init children []*Init - init onceMore + init OnceMore out any err error f func(context.Context) (any, error) diff --git a/lazy/once.go b/lazy/once.go index c6abcd88493..dac689df393 100644 --- a/lazy/once.go +++ b/lazy/once.go @@ -18,19 +18,19 @@ import ( "sync/atomic" ) -// onceMore is similar to sync.Once. +// OnceMore is similar to sync.Once. // // Additional features are: // * it can be reset, so the action can be repeated if needed // * it has methods to check if it's done or in progress -type onceMore struct { +type OnceMore struct { mu sync.Mutex lock uint32 done uint32 } -func (t *onceMore) Do(f func()) { +func (t *OnceMore) Do(f func()) { if atomic.LoadUint32(&t.done) == 1 { return } @@ -53,15 +53,15 @@ func (t *onceMore) Do(f func()) { f() } -func (t *onceMore) InProgress() bool { +func (t *OnceMore) InProgress() bool { return atomic.LoadUint32(&t.lock) == 1 } -func (t *onceMore) Done() bool { +func (t *OnceMore) Done() bool { return atomic.LoadUint32(&t.done) == 1 } -func (t *onceMore) ResetWithLock() *sync.Mutex { +func (t *OnceMore) ResetWithLock() *sync.Mutex { t.mu.Lock() defer atomic.StoreUint32(&t.done, 0) return &t.mu