diff --git a/tests/docs/yaml/bad-config-yaml/.gitignore b/tests/docs/yaml/bad-config-yaml/.gitignore
new file mode 100644
index 0000000000..075b2542af
--- /dev/null
+++ b/tests/docs/yaml/bad-config-yaml/.gitignore
@@ -0,0 +1 @@
+/.quarto/
diff --git a/tests/docs/yaml/bad-config-yaml/_quarto.yml b/tests/docs/yaml/bad-config-yaml/_quarto.yml
new file mode 100644
index 0000000000..385210d2b0
--- /dev/null
+++ b/tests/docs/yaml/bad-config-yaml/_quarto.yml
@@ -0,0 +1,3 @@
+project:
+  title: "bad-config-yaml"
+  type: 1
diff --git a/tests/docs/yaml/bad-config-yaml/subfolder/bad-config-yaml.qmd b/tests/docs/yaml/bad-config-yaml/subfolder/bad-config-yaml.qmd
new file mode 100644
index 0000000000..4786ee0c3c
--- /dev/null
+++ b/tests/docs/yaml/bad-config-yaml/subfolder/bad-config-yaml.qmd
@@ -0,0 +1,7 @@
+---
+title: "bad-config-yaml"
+---
+
+## Quarto
+
+This is a doc in a dummy project to check that error message is correct
diff --git a/tests/smoke/yaml/config-yaml-validation.test.ts b/tests/smoke/yaml/config-yaml-validation.test.ts
new file mode 100644
index 0000000000..6ed1bd6a13
--- /dev/null
+++ b/tests/smoke/yaml/config-yaml-validation.test.ts
@@ -0,0 +1,18 @@
+/*
+* config-yaml-validation.test.ts
+*
+* Copyright (C) 2024 Posit Software, PBC
+*
+*/
+
+import { testQuartoCmd } from "../../test.ts";
+import { docs } from "../../utils.ts";
+import { printsMessage } from "../../verify.ts";
+
+const input = docs("yaml/bad-config-yaml/subfolder/bad-config-yaml.qmd");
+
+testQuartoCmd(
+  "render",
+  [input],
+  [printsMessage("ERROR", /bad-config-yaml[\/\\]+_quarto[.]yml validation failed/)]
+);