improve yaml reading and handling of backticks. Closes #6344 and #6367.…

… (#6370)

* improve yaml reading and handling of backticks. Closes #6344 and #6367.
* smoke test
* changelog
* match backtick counts when inner code exists inside outer code cell

news/changelog-1.4.md

@@ -76,6 +76,8 @@
 ## Jupyter
 
 - Support for executing inline expressions (e.g. `` `{python} x` ``)
+- ([#6344](https://github.com/quarto-dev/quarto-cli/issues/6344)): Somewhat improve the error message in case of YAML parsing errors in metadata of Python code cells.
+- ([#6367](https://github.com/quarto-dev/quarto-cli/issues/6367)): Fix bug with nested code cells in the generation of Jupyter notebook from .qmd files.
 
 ## Dependencies
 

src/core/jupyter/jupyter.ts

@@ -258,7 +258,7 @@ const countTicks = (code: string[]) => {
   // FIXME do we need trim() here?
   const countLeadingTicks = (s: string) => {
     // count leading ticks using regexps
-    const m = s.match(/^`+/);
+    const m = s.match(/^\s*`+/);
     if (m) {
       return m[0].length;
     } else {
@@ -298,12 +298,12 @@ export async function quartoMdToJupyter(
   const yamlRegEx = /^---\s*$/;
   /^\s*```+\s*\{([a-zA-Z0-9_]+)( *[ ,].*)?\}\s*$/;
   const startCodeCellRegEx = new RegExp(
-    "^(\\s*)```+\\s*\\{" + kernelspec.language.toLowerCase() +
+    "^(\\s*)(```+)\\s*\\{" + kernelspec.language.toLowerCase() +
       "( *[ ,].*)?\\}\\s*$",
   );
   const startCodeRegEx = /^(\s*)```/;
-  const endCodeRegEx = (indent = "") => {
-    return new RegExp("^" + indent + "```\\s*$");
+  const endCodeRegEx = (indent = "", backtickCount = 0) => {
+    return new RegExp("^" + indent + "`".repeat(backtickCount) + "\\s*$");
   };
 
   // read the file into lines
@@ -368,7 +368,7 @@ export async function quartoMdToJupyter(
           kernelspec.language.toLowerCase(),
           cell.source,
         );
-        if (yaml) {
+        if (yaml && !Array.isArray(yaml) && typeof yaml === "object") {
           // use label as id if necessary
           if (includeIds && yaml[kCellLabel] && !yaml[kCellId]) {
             yaml[kCellId] = jupyterAutoIdentifier(String(yaml[kCellLabel]));
@@ -418,7 +418,8 @@ export async function quartoMdToJupyter(
   let parsedFrontMatter = false,
     inYaml = false,
     inCodeCell = false,
-    inCode = false;
+    inCode = false,
+    backtickCount = 0;
   for (const line of lines(inputContent)) {
     // yaml front matter
     if (yamlRegEx.test(line) && !inCodeCell && !inCode) {
@@ -433,13 +434,16 @@ export async function quartoMdToJupyter(
         inYaml = true;
       }
     } // begin code cell: ^```python
-    else if (startCodeCellRegEx.test(line)) {
+    else if (!inCodeCell && startCodeCellRegEx.test(line)) {
       flushLineBuffer("markdown");
       inCodeCell = true;
       codeIndent = line.match(startCodeCellRegEx)![1];
+      backtickCount = line.match(startCodeCellRegEx)![2].length;
 
       // end code block: ^``` (tolerate trailing ws)
-    } else if (endCodeRegEx(codeIndent).test(line)) {
+    } else if (
+      inCodeCell && endCodeRegEx(codeIndent, backtickCount).test(line)
+    ) {
       // in a code cell, flush it
       if (inCodeCell) {
         inCodeCell = false;
@@ -454,7 +458,7 @@ export async function quartoMdToJupyter(
       }
 
       // begin code block: ^```
-    } else if (startCodeRegEx.test(line)) {
+    } else if (!inCodeCell && startCodeRegEx.test(line)) {
       codeIndent = line.match(startCodeRegEx)![1];
       inCode = true;
       lineBuffer.push(line);
@@ -465,7 +469,6 @@ export async function quartoMdToJupyter(
 
   // if there is still a line buffer then make it a markdown cell
   flushLineBuffer("markdown");
-
   return nb;
 }
 

src/core/lib/break-quarto-md.ts

@@ -41,10 +41,10 @@ export async function breakQuartoMd(
   // regexes
   const yamlRegEx = /^---\s*$/;
   const startCodeCellRegEx = new RegExp(
-    "^\\s*```+\\s*\\{([=A-Za-z]+)( *[ ,].*)?\\}\\s*$",
+    "^\\s*(```+)\\s*\\{([=A-Za-z]+)( *[ ,].*)?\\}\\s*$",
   );
   const startCodeRegEx = /^```/;
-  const endCodeRegEx = /^\s*```+\s*$/;
+  const endCodeRegEx = /^\s*(```+)\s*$/;
 
   let language = ""; // current language block
   let directiveParams: Shortcode | undefined = undefined;
@@ -210,15 +210,17 @@ export async function breakQuartoMd(
     } // begin code cell: ^```python
     else if (startCodeCellRegEx.test(line.substring) && inPlainText()) {
       const m = line.substring.match(startCodeCellRegEx);
-      language = (m as string[])[1];
+      language = (m as string[])[2];
       await flushLineBuffer("markdown", i);
       inCodeCell = true;
+      inCode = (m as string[])[1].length;
+
       codeStartRange = line;
 
       // end code block: ^``` (tolerate trailing ws)
     } else if (
       endCodeRegEx.test(line.substring) &&
-      (inCodeCell || (inCode && tickCount(line.substring) === inCode))
+      (inCode && (line.substring.match(endCodeRegEx)!)[1].length === inCode)
     ) {
       // in a code cell, flush it
       if (inCodeCell) {

tests/docs/smoke-all/2023/07/28/6367.qmd

@@ -0,0 +1,22 @@
+---
+format: gfm
+keep-md: true
+jupyter:
+  kernelspec:
+    display_name: Python 3 (ipykernel)
+    language: python
+    name: python3
+---
+
+# example
+
+````{python}
+# using the f-string means the python string uses 2 curly braces in its syntax,
+# but the string itself is just single curly braces
+# otherwise, quarto tries to execute the code (a separate issue)
+# but quarto sees and interprets the double curly braces as a non-executable code block
+doc = f"""
+    ```{{python}}
+    ```
+"""
+````