Fix some RenderShortcodes error cases

This issue fixes two cases where `{{__hugo_ctx` artifacts were left in the rendered output:

1. Inclusion when `.RenderShortcodes` is wrapped in HTML.
2. Inclusion of Markdown file without a trailing newline in some cases.

Closes #12854

common/constants/constants.go

@@ -21,6 +21,7 @@ const (
 	ErrRemoteGetCSV  = "error-remote-getcsv"
 
 	WarnFrontMatterParamsOverrides = "warning-frontmatter-params-overrides"
+	WarnRenderShortcodesInHTML     = "warning-rendershortcodes-in-html"
 )
 
 // Field/method names with special meaning.

hugolib/integrationtest_builder.go

@@ -2,6 +2,7 @@ package hugolib
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -32,6 +33,7 @@ import (
 	"github.com/gohugoio/hugo/htesting"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/afero"
+	"github.com/spf13/cast"
 	"golang.org/x/text/unicode/norm"
 	"golang.org/x/tools/txtar"
 )
@@ -294,6 +296,12 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s
 	}
 }
 
+func (s *IntegrationTestBuilder) AssertFileContentEquals(filename string, match string) {
+	s.Helper()
+	content := s.FileContent(filename)
+	s.Assert(content, qt.Equals, match, qt.Commentf(match))
+}
+
 func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches ...string) {
 	s.Helper()
 	content := s.FileContent(filename)
@@ -302,6 +310,16 @@ func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches
 	}
 }
 
+func (s *IntegrationTestBuilder) AssertNoRenderShortcodesArtifacts() {
+	s.Helper()
+	for _, p := range s.H.Pages() {
+		content, err := p.Content(context.Background())
+		s.Assert(err, qt.IsNil)
+		comment := qt.Commentf("Page: %s", p.Path())
+		s.Assert(strings.Contains(cast.ToString(content), "__hugo_ctx"), qt.IsFalse, comment)
+	}
+}
+
 func (s *IntegrationTestBuilder) AssertPublishDir(matches ...string) {
 	s.AssertFs(s.fs.PublishDir, matches...)
 }
@@ -835,6 +853,11 @@ type IntegrationTestConfig struct {
 
 	// The files to use on txtar format, see
 	// https://pkg.go.dev/golang.org/x/exp/cmd/txtar
+	// There are some conentions used in this test setup.
+	// - §§§ can be used to wrap code fences.
+	// - §§ can be used to wrap multiline strings.
+	// - filenames prefixed with sourcefilename: will be read from the file system relative to the current dir.
+	// - filenames with a .png or .jpg extension will be treated as binary and base64 decoded.
 	TxtarString string
 
 	// COnfig to use as the base. We will also read the config from the txtar.

hugolib/rendershortcodes_test.go

@@ -69,6 +69,7 @@ Content: {{ .Content }}|
 
 	b := Test(t, files)
 
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/p1/index.html",
 		"Fragments: [p1-h1 p2-h1 p2-h2 p2-h3 p2-withmarkdown p3-h1 p3-h2 p3-withmarkdown]|",
 		"HasShortcode Level 1: true|",
@@ -115,6 +116,7 @@ JSON: {{ .Content }}
 
 	b := Test(t, files)
 
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/p1/index.html", "Myshort HTML")
 	b.AssertFileContent("public/p1/index.json", "Myshort JSON")
 }
@@ -147,9 +149,11 @@ Myshort Original.
  {{ .Content }}
 `
 	b := TestRunning(t, files)
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/p1/index.html", "Myshort Original.")
 
 	b.EditFileReplaceAll("layouts/shortcodes/myshort.html", "Original", "Edited").Build()
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/p1/index.html", "Myshort Edited.")
 }
 
@@ -192,12 +196,14 @@ Myshort Original.
 		},
 	).Build()
 
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/p1/index.html", "Original")
 
 	b.EditFileReplaceFunc("content/p2.md", func(s string) string {
 		return strings.Replace(s, "Original", "Edited", 1)
 	})
 	b.Build()
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/p1/index.html", "Edited")
 }
 
@@ -233,8 +239,10 @@ Myshort Original.
 `
 	b := TestRunning(t, files)
 
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/mysection/index.html", "p1-h1")
 	b.EditFileReplaceAll("content/mysection/_index.md", "p1-h1", "p1-h1 Edited").Build()
+	b.AssertNoRenderShortcodesArtifacts()
 	b.AssertFileContent("public/mysection/index.html", "p1-h1 Edited")
 }
 
@@ -314,6 +322,8 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
 
 	b := Test(t, files)
 
+	b.AssertNoRenderShortcodesArtifacts()
+
 	b.AssertFileContent("public/markdown/index.html",
 		// Images.
 		"Image: /posts/p1/pixel1.png|\nImage: /posts/p1/pixel2.png|\n|\nImage: /markdown/pixel3.png|</p>\n|",
@@ -333,3 +343,96 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA
 
 	b.AssertFileContent("public/html/index.html", "! hugo_ctx")
 }
+
+// Issue 12854.
+func TestRenderShortcodesWithHTML(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableLiveReload = true
+disableKinds = ["home", "taxonomy", "term"]
+markup.goldmark.renderer.unsafe = true
+-- content/p1.md --
+---
+title: "p1"
+---
+{{% include "p2" %}}
+-- content/p2.md --
+---
+title: "p2"
+---
+Hello <b>world</b>. Some **bold** text. Some Unicode: 神真美好.
+-- layouts/shortcodes/include.html --
+{{ with site.GetPage (.Get 0) }}
+<div>{{ .RenderShortcodes }}</div>
+{{ end }}
+-- layouts/_default/single.html --
+{{ .Content }}
+`
+
+	b := TestRunning(t, files, TestOptWarn())
+
+	b.AssertNoRenderShortcodesArtifacts()
+	b.AssertFileContent("public/p1/index.html", "<div>Hello <b>world</b>. Some **bold** text. Some Unicode: 神真美好.</div>")
+	b.EditFileReplaceAll("content/p2.md", "Hello", "Hello Edited").Build()
+	b.AssertNoRenderShortcodesArtifacts()
+	b.AssertFileContent("public/p1/index.html", "<div>Hello Edited <b>world</b>. Some **bold** text. Some Unicode: 神真美好.</div>")
+}
+
+func TestRenderShortcodesIncludeMarkdownFileWithoutTrailingNewline(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- hugo.toml --
+disableLiveReload = true
+disableKinds = ["home", "taxonomy", "term"]
+markup.goldmark.renderer.unsafe = true
+-- content/p1.md --
+---
+title: "p1"
+---
+Content p1 id-1000.{{% include "p2" %}}{{% include "p3" %}}
+
+§§§ go
+code_p1
+§§§
+-- content/p2.md --
+---
+title: "p2"
+---
+§§§ go
+code_p2
+§§§
+Foo.
+-- content/p3.md --
+---
+title: "p3"
+---
+§§§ go
+code_p3
+§§§
+-- layouts/shortcodes/include.html --
+{{ with site.GetPage (.Get 0) -}}
+{{ .RenderShortcodes -}}
+{{ end -}}
+-- layouts/_default/single.html --
+{{ .Content }}
+-- layouts/_default/_markup/render-codeblock.html --
+<code>{{ .Inner | safeHTML }}</code>
+`
+
+	b := TestRunning(t, files, TestOptWarn())
+
+	b.AssertNoRenderShortcodesArtifacts()
+	b.AssertFileContentEquals("public/p1/index.html", "<p>Content p1 id-1000.</p>\n<code>code_p2</code><p>Foo.\n</p>\n<code>code_p3</code><code>code_p1</code>")
+	b.EditFileReplaceAll("content/p1.md", "id-1000.", "id-100.").Build()
+	b.AssertNoRenderShortcodesArtifacts()
+	b.AssertFileContentEquals("public/p1/index.html", "<p>Content p1 id-100.</p>\n<code>code_p2</code><p>Foo.\n</p>\n<code>code_p3</code><code>code_p1</code>")
+	b.EditFileReplaceAll("content/p2.md", "code_p2", "codep2").Build()
+	b.AssertNoRenderShortcodesArtifacts()
+	b.AssertFileContentEquals("public/p1/index.html", "<p>Content p1 id-100.</p>\n<code>codep2</code><p>Foo.\n</p>\n<code>code_p3</code><code>code_p1</code>")
+	b.EditFileReplaceAll("content/p3.md", "code_p3", "code_p3_edited").Build()
+	b.AssertNoRenderShortcodesArtifacts()
+	b.AssertFileContentEquals("public/p1/index.html", "<p>Content p1 id-100.</p>\n<code>codep2</code><p>Foo.\n</p>\n<code>code_p3_edited</code><code>code_p1</code>")
+}

markup/goldmark/convert.go

@@ -106,7 +106,7 @@ func newMarkdown(pcfg converter.ProviderConfig) goldmark.Markdown {
 		renderer.WithNodeRenderers(util.Prioritized(emoji.NewHTMLRenderer(), 200)))
 	var (
 		extensions = []goldmark.Extender{
-			hugocontext.New(),
+			hugocontext.New(pcfg.Logger),
 			newLinks(cfg),
 			newTocExtension(tocRendererOptions),
 			blockquotes.New(),