Merge pull request #7451 from quarto-dev/feature/latex-quartomarkdowncmd

\QuartoMarkdownBase64 command in LaTeX formats

src/resources/filters/ast/customnodes.lua

@@ -183,6 +183,9 @@ function run_emulated_filter(doc, filter)
     if node.attributes.__quarto_custom_scaffold == "true" then
       return nil
     end
+    if node.identifier == _quarto.ast.vault._uuid then
+      return nil
+    end
     if filter.Div ~= nil then
       return filter.Div(node)
     end
@@ -257,6 +260,35 @@ function create_emulated_node(t, tbl, context, forwarder)
 end
 
 _quarto.ast = {
+  vault = {
+    _uuid = "3ade8a4a-fb1d-4a6c-8409-ac45482d5fc9",
+
+    _added = {},
+    _removed = {},
+    add = function(id, contents)
+      _quarto.ast.vault._added[id] = contents
+    end,
+    remove = function(id)
+      _quarto.ast.vault._removed[id] = true
+    end,
+    locate = function(doc)
+      if doc == nil then
+        doc = _quarto.ast._current_doc
+      end
+      -- attempt a fast lookup first
+      if #doc.blocks > 0 and doc.blocks[#doc.blocks].identifier == _quarto.ast.vault._uuid then
+        return doc.blocks[#doc.blocks]
+      else
+        -- otherwise search for it
+        for _, block in ipairs(doc.blocks) do
+          if block.identifier == _quarto.ast.vault._uuid then
+            return block
+          end
+        end
+      end
+      return nil
+    end,  
+  },
   custom_node_data = custom_node_data,
   create_custom_node_scaffold = create_custom_node_scaffold,
 

src/resources/filters/ast/runemulation.lua

@@ -5,6 +5,48 @@
 
 local profiler = require('profiler')
 
+-- locate or create the quarto vault,
+-- inserting the just-added nodes if needed, and mutating doc
+local ensure_vault = function(doc)
+  local vault = _quarto.ast.vault.locate(doc)
+
+  -- create if missing
+  if vault == nil then
+    vault = pandoc.Div({}, pandoc.Attr(_quarto.ast.vault._uuid, {}, {}))
+    doc.blocks:insert(vault)
+  end
+
+  for k, v in pairs(_quarto.ast.vault._added) do
+    local div = pandoc.Div(quarto.utils.as_blocks(v), pandoc.Attr(k, {}, {}))
+    vault.content:insert(div)
+  end
+  vault.content = _quarto.ast.walk(vault.content, {
+    Div = function(div)
+      if _quarto.ast.vault._removed[div.identifier] then
+        return {}
+      end
+    end
+  }) or pandoc.Blocks({}) -- to satisfy the Lua analyzer
+
+  _quarto.ast.vault._added = {}
+  _quarto.ast.vault._removed = {}
+end
+
+local function remove_vault(doc)
+  -- attempt a fast lookup first
+  if #doc.blocks > 0 and doc.blocks[#doc.blocks].identifier == _quarto.ast.vault._uuid then
+    doc.blocks:remove(#doc.blocks)
+  else
+    -- otherwise search for it
+    for i, block in ipairs(doc.blocks) do
+      if block.identifier == _quarto.ast.vault._uuid then
+        doc.blocks:remove(i)
+        break
+      end
+    end
+  end
+end
+
 local function run_emulated_filter_chain(doc, filters, afterFilterPass, profiling)
   init_trace(doc)
   for i, v in ipairs(filters) do
@@ -36,8 +78,9 @@ local function run_emulated_filter_chain(doc, filters, afterFilterPass, profilin
       if v.print_ast then
         print(pandoc.write(doc, "native"))
       else
-
+        _quarto.ast._current_doc = doc
         doc = run_emulated_filter(doc, v.filter)
+        ensure_vault(doc)
 
         add_trace(doc, v.name)
 
@@ -58,6 +101,7 @@ local function run_emulated_filter_chain(doc, filters, afterFilterPass, profilin
     end
   end
   end_trace()
+  remove_vault(doc)
   return doc
 end
 

src/resources/filters/main.lua

@@ -353,13 +353,13 @@ local quarto_post_filters = {
 }
 
 local quarto_finalize_filters = {
-    -- quarto-finalize
-    { name = "finalize-fileMetadataAndMediabag", filter =
+  -- quarto-finalize
+  { name = "finalize-combined", filter =
     combineFilters({
       file_metadata(),
-      mediabag()
-    })
-  },
+      mediabag(),
+      inject_vault_content_into_rawlatex(),
+    })},
   { name = "finalize-bookCleanup", filter = bookCleanup() },
   { name = "finalize-cites", filter = writeCites() },
   { name = "finalize-metaCleanup", filter = metaCleanup() },

src/resources/filters/normalize/astpipeline.lua

@@ -5,11 +5,12 @@ function quarto_ast_pipeline()
   return {
     { name = "normalize-table-merge-raw-html", filter = table_merge_raw_html() },
 
-    -- this can't be combined because it's top-down processing.
+    -- this filter can't be combined with others because it's top-down processing.
     -- unfortunate.
     { name = "normalize-html-table-processing", filter = parse_html_tables() },
 
     { name = "normalize-combined-1", filter = combineFilters({
+        extract_latex_quartomarkdown_commands(),
         forward_cell_subcaps(),
         parse_extended_nodes(),
         code_filename(),

src/resources/filters/normalize/extractquartodom.lua

@@ -27,4 +27,83 @@ function parse_md_in_html_rawblocks()
       end
     end
   }
-end
+end
+
+extracted_qmd_uuid = "3ab579b5-63b4-445d-bc1d-85bf6c4c04de"
+local count = 0
+
+function extract_latex_quartomarkdown_commands()
+  if not _quarto.format.isLatexOutput() then 
+    return {}
+  end
+
+  if doc == nil then
+    return {
+      RawBlock = function(el)
+        if not _quarto.format.isRawLatex(el) then
+          return nil
+        end
+        local text = el.text
+        local pattern = "(.*)(\\QuartoMarkdownBase64{)([^}]*)(})(.*)"
+        local pre, _, content, _, post = text:match(pattern)
+        if pre == nil then
+          return nil
+        end
+        while pre do
+          count = count + 1
+          local uuid = extracted_qmd_uuid .. "-" .. tostring(count)
+          _quarto.ast.vault.add(uuid, string_to_quarto_ast_blocks(quarto.base64.decode(content)))
+          text = pre .. uuid .. post
+          pre, _, content, _, post = text:match(pattern)
+        end
+        return pandoc.RawBlock(el.format, text)
+      end
+    }
+  end
+end
+
+function inject_vault_content_into_rawlatex()
+  return {
+    RawBlock = function(el)
+      if not _quarto.format.isRawLatex(el) then
+        return nil
+      end
+      local vault = _quarto.ast.vault.locate()
+      if vault == nil then
+        -- luacov: disable
+        internal_error()
+        return nil
+        -- luacov: enable
+      end
+
+      local pattern = "(.*)(3ab579b5%-63b4%-445d%-bc1d%-85bf6c4c04de%-[0-9]+)(.*)"
+      local text = el.text
+      local pre, content_id, post = text:match(pattern)
+
+
+      while pre do
+        local found = false
+        vault.content = _quarto.ast.walk(vault.content, {
+          Div = function(div)
+            if div.identifier ~= content_id then
+              return
+            end
+            _quarto.ast.vault.remove(content_id)
+            local rendered = pandoc.write(pandoc.Pandoc(div.content), "latex")
+            text = pre .. rendered .. post
+            pre, content_id, post = text:match(pattern)
+            found = true
+            return {}
+          end
+        })
+        if not found then
+          -- luacov: disable
+          internal_error()
+          return nil
+          -- luacov: enable
+        end
+      end
+      return pandoc.RawBlock(el.format, text)
+    end,
+  }
+end

tests/docs/smoke-all/2023/11/02/latex-quarto-markdown-base64.qmd

@@ -0,0 +1,15 @@
+---
+format: pdf
+a_meta_key: a_value
+---
+
+![A caption](./foo.jpg){#fig-1}
+
+
+```{=latex}
+% QGZpZy0x is "@fig-1" in base-64 encoding
+% U29tZSBfbWFya2Rvd25fIGhlcmUu is "{{< meta a_meta_key >}} Some _markdown_ here." in base-64 encoding
+\begin{tabular}{cc}
+\QuartoMarkdownBase64{QGZpZy0x} & \QuartoMarkdownBase64{e3s8IG1ldGEgYV9tZXRhX2tleSA+fX0gU29tZSBfbWFya2Rvd25fIGhlcmUu} \\
+\end{tabular}
+```