serialize and deserialize custom nodes to JSON filters

src/resources/filters/ast/customnodes.lua

@@ -536,7 +536,65 @@ _quarto.ast = {
   make_scaffold = function(ctor, node)
     return ctor(node or {}, pandoc.Attr("", {"quarto-scaffold", "hidden"}, {}))
   end,
-
+
+  -- custom_node_data_as_meta and reset_custom_node_data_from_meta
+  -- are both used to enable JSON filters to work with custom nodes
+  custom_node_data_as_meta = function()
+    local list = pandoc.List({})
+    for k, v in pairs(custom_node_data) do
+      local custom_meta = {}
+      local inner = { quarto_custom_meta = custom_meta }
+      -- FIXME we need to get the ids of all the slots
+      for k2, v2 in pairs(v) do
+        if k2 == "__quarto_custom_node" then
+          custom_meta.id = v2.attributes.__quarto_custom_id
+        else
+          inner[k2] = v2
+        end
+      end
+      list:insert(inner)
+    end
+    return pandoc.MetaList(list)
+  end,
+  reset_custom_node_data_from_meta = function(obj, custom_node_map)
+    local new_custom_node_data = {}
+    for i, v in ipairs(obj) do
+      local new_obj = {}
+      local n_objs = 0
+      local t = v.t
+      local handler = quarto._quarto.ast.resolve_handler(t)
+      assert(handler)
+
+      local qcm_obj = nil
+
+      for k2, v2 in pairs(v) do
+        if k2 == "quarto_custom_meta" then
+          qcm_obj = v2
+        else
+          new_obj[k2] = v2
+        end
+      end
+
+      assert(qcm_obj)
+      local id = qcm_obj.id
+      local forwarder = { }
+      if tisarray(handler.slots) then
+        for i, slot in ipairs(handler.slots) do
+          forwarder[slot] = i
+        end
+      else
+        forwarder = handler.slots
+      end
+
+      new_obj.__quarto_custom_node = custom_node_map[id]
+      n_objs = math.max(n_objs, tonumber(id))
+      new_custom_node_data[id] = _quarto.ast.create_proxy_accessor(
+        custom_node_map[id], new_obj, forwarder)
+    end
+    _quarto.ast.custom_node_data = new_custom_node_data
+    custom_node_data = new_custom_node_data
+  end,
+
   scoped_walk = scoped_walk,
 
   walk = run_emulated_filter,

src/resources/filters/common/wrapped-filter.lua

@@ -97,22 +97,16 @@ function makeWrappedJsonFilter(scriptFile, filterHandler)
         path = quarto.utils.resolve_path_relative_to_document(scriptFile)
         local custom_node_map = {}
         local has_custom_nodes = false
-        doc = doc:walk({
-          -- FIXME: This is broken with new AST. Needs to go through Custom node instead.
-          RawInline = function(raw)
-            local custom_node, t, kind = _quarto.ast.resolve_custom_data(raw)
-            if custom_node ~= nil then
-              has_custom_nodes = true
-              custom_node = safeguard_for_meta(custom_node)
-              table.insert(custom_node_map, { id = raw.text, tbl = custom_node, t = t, kind = kind })
-            end
+        doc = _quarto.ast.walk(doc, {
+          Custom = function(_)
+            has_custom_nodes = true
           end,
           Meta = function(meta)
             if has_custom_nodes then
-              meta["quarto-custom-nodes"] = pandoc.MetaList(custom_node_map)
+              meta["quarto-custom-node-data"] = _quarto.ast.custom_node_data_as_meta()
             end
             return meta
-          end
+          end          
         })
         local success, result = pcall(pandoc.utils.run_json_filter, doc, path)
         if not success then
@@ -129,14 +123,29 @@ function makeWrappedJsonFilter(scriptFile, filterHandler)
           fail(table.concat(message, "\n"))
           return nil
         end
-        if has_custom_nodes then
-          doc:walk({
-            Meta = function(meta)
-              _quarto.ast.reset_custom_tbl(meta["quarto-custom-nodes"])
+        assert(result)
+        local custom_node_map = {}
+        -- can't call _quarto.ast.walk here
+        -- because the custom_node_map data is not restored yet
+        -- so we use a plain :walk{} call and check for the
+        -- custom attributes
+        result:walk({
+          Span = function(span)
+            if span.attributes.__quarto_custom == "true" then
+              custom_node_map[span.attributes.__quarto_custom_id] = span
             end
-          })
-        end
-
+          end,
+          Div = function(div)
+            if div.attributes.__quarto_custom == "true" then
+              custom_node_map[div.attributes.__quarto_custom_id] = div
+            end
+          end,
+          Meta = function(meta)
+            if meta["quarto-custom-node-data"] ~= nil then
+              _quarto.ast.reset_custom_node_data_from_meta(meta["quarto-custom-node-data"], custom_node_map)
+            end
+          end
+        })
         return result
       end
     }

tests/docs/smoke-all/2024/10/29/test-shortcode-json-filter-insertion/insert_shortcode.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+import json
+import sys
+
+inp = sys.stdin.read()
+doc = json.loads(inp)
+
+#######
+# p_* are a bad version of a micro-panflute
+
+def p_string(string):
+  return {
+    "t": "String",
+    "c": string
+  }
+
+def p_meta_string(string):
+  return {
+    "t": "MetaString",
+    "c": string
+  }
+
+def p_meta_val(val):
+  if type(val) == str:
+    return p_meta_string(val)
+  if type(val) == dict:
+    return p_meta_map(val)
+  if type(val) == list:
+    return p_meta_list(val)
+  raise Exception("Unknown type: " + str(type(val)))
+
+def p_meta_list(list):
+  return {
+    "t": "MetaList",
+    "c": [p_meta_val(v) for v in list]
+  }
+
+def p_meta_map(dict):
+  result = {}
+  for k, v in dict.items():
+    result[k] = p_meta_val(v)
+  return {
+    "t": "MetaMap",
+    "c": result
+  }
+
+def p_attr(id, classes, attrs):
+  return [id, classes, list([k, v] for k, v in attrs.items())]
+
+def p_para(content):
+  return {
+    "t": "Para",
+    "c": content
+  }
+
+#######
+
+max_id = 0
+for node in doc["meta"]["quarto-custom-node-data"]["c"]:
+  if node["t"] == "MetaMap":
+    max_id = max(max_id, int(node["c"]["quarto_custom_meta"]["c"]["id"]["c"]))
+
+def shortcode_data():
+  return {
+    "name": "meta",
+    "params": [ { "type": "param", "value": "baz" } ],
+    "t": "Shortcode",
+    "unparsed_content": r"{{< meta baz >}}",
+    "quarto_custom_meta": {
+      "id": str(max_id + 1) # this needs to be unique!!
+    }
+  }
+
+def shortcode_span():
+  return {
+    "t": "Span",
+    "c": [ 
+      p_attr("", [], {"__quarto_custom": "true", "__quarto_custom_type": "Shortcode", "__quarto_custom_context": "Inline", "__quarto_custom_id": str(max_id + 1)}),
+      [] # no content
+    ]
+  }
+
+doc["meta"]["quarto-custom-node-data"]["c"].append(p_meta_val(shortcode_data()))
+doc["blocks"].append(p_para([shortcode_span()]))
+
+print(json.dumps(doc))

tests/docs/smoke-all/2024/10/29/test-shortcode-json-filter-insertion/json-filter-shortcode-insertion.qmd

@@ -0,0 +1,25 @@
+---
+filters:
+  - at: pre-quarto
+    path: insert_shortcode.py
+    type: json
+title: Hello
+foo: bar
+baz: "d79920da-f2a6-4f3e-9c5c-84c9854d5b11"
+_quarto:
+  tests:
+    html:
+      ensureFileRegexMatches:
+        - ["d79920da-f2a6-4f3e-9c5c-84c9854d5b11"]
+        - []
+---
+
+{{< meta foo >}}
+
+::: {#fig-1}
+
+This is the content.
+
+This is a caption.
+
+:::