Improved adoc compilation process

.github/workflows/build.yml

@@ -29,6 +29,8 @@ jobs:
           python-version: 3.9
       - name: Install Python dependencies
         uses: py-actions/py-dependency-install@v4
+      - name: Install Python libs
+        run: pip3 install -r ./requirements.txt
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 3.2

.gitmodules

@@ -6,3 +6,7 @@
 	path = lib/pico-examples
 	url = https://github.com/raspberrypi/pico-examples.git
 	branch = master
+
+[submodule "doxygentoasciidoc"]
+	path = doxygentoasciidoc
+	url = https://github.com/raspberrypi/doxygentoasciidoc.git

Makefile

@@ -18,7 +18,7 @@ PICO_SDK_DIR = lib/pico-sdk
 PICO_EXAMPLES_DIR = lib/pico-examples
 ALL_SUBMODULE_CMAKELISTS = $(PICO_SDK_DIR)/CMakeLists.txt $(PICO_EXAMPLES_DIR)/CMakeLists.txt
 DOXYGEN_PICO_SDK_BUILD_DIR = build-pico-sdk-docs
-DOXYGEN_HTML_DIR = $(DOXYGEN_PICO_SDK_BUILD_DIR)/docs/doxygen/html
+DOXYGEN_XML_DIR = $(DOXYGEN_PICO_SDK_BUILD_DIR)/combined/docs/doxygen/xml
 # The pico-sdk here needs to match up with the "from_json" entry in index.json
 ASCIIDOC_DOXYGEN_DIR = $(ASCIIDOC_DIR)/pico-sdk
 
@@ -50,33 +50,42 @@ $(PICO_SDK_DIR)/CMakeLists.txt $(PICO_SDK_DIR)/docs/index.h: | $(PICO_SDK_DIR)
 $(PICO_EXAMPLES_DIR)/CMakeLists.txt: | $(PICO_SDK_DIR)/CMakeLists.txt $(PICO_EXAMPLES_DIR)
 	git submodule update --init $(PICO_EXAMPLES_DIR)
 
+# Initialise doxygentoasciidoc submodule
+doxygentoasciidoc/__main__.py:
+	git submodule update --init doxygentoasciidoc
+
 fetch_submodules: $(ALL_SUBMODULE_CMAKELISTS)
 
 # Get rid of the submodules
 clean_submodules:
 	git submodule deinit --all
 
-# Create the pico-sdk Doxygen HTML files
-$(DOXYGEN_HTML_DIR): | $(ALL_SUBMODULE_CMAKELISTS) $(DOXYGEN_PICO_SDK_BUILD_DIR)
-	cmake -S $(PICO_SDK_DIR) -B $(DOXYGEN_PICO_SDK_BUILD_DIR) -DPICO_EXAMPLES_PATH=../$(PICO_EXAMPLES_DIR)
-	$(MAKE) -C $(DOXYGEN_PICO_SDK_BUILD_DIR) docs
-	test -d "$@"
+# Create the pico-sdk Doxygen XML files
+$(DOXYGEN_XML_DIR) $(DOXYGEN_XML_DIR)/index.xml: | $(ALL_SUBMODULE_CMAKELISTS) $(DOXYGEN_PICO_SDK_BUILD_DIR)
+	cmake -S $(PICO_SDK_DIR) -B $(DOXYGEN_PICO_SDK_BUILD_DIR)/combined -D PICO_EXAMPLES_PATH=../$(PICO_EXAMPLES_DIR) -D PICO_PLATFORM=combined-docs
+	cmake -S $(PICO_SDK_DIR) -B $(DOXYGEN_PICO_SDK_BUILD_DIR)/PICO_RP2040 -D PICO_EXAMPLES_PATH=../$(PICO_EXAMPLES_DIR) -D PICO_PLATFORM=rp2040
+	cmake -S $(PICO_SDK_DIR) -B $(DOXYGEN_PICO_SDK_BUILD_DIR)/PICO_RP2350 -D PICO_EXAMPLES_PATH=../$(PICO_EXAMPLES_DIR) -D PICO_PLATFORM=rp2350
+	$(MAKE) -C $(DOXYGEN_PICO_SDK_BUILD_DIR)/combined docs
+	$(MAKE) -C $(DOXYGEN_PICO_SDK_BUILD_DIR)/PICO_RP2040 docs
+	$(MAKE) -C $(DOXYGEN_PICO_SDK_BUILD_DIR)/PICO_RP2350 docs
+	python3 $(SCRIPTS_DIR)/postprocess_doxygen_xml.py $(DOXYGEN_PICO_SDK_BUILD_DIR)
 
-$(DOXYGEN_PICO_SDK_BUILD_DIR)/docs/Doxyfile: | $(DOXYGEN_HTML_DIR)
+$(DOXYGEN_PICO_SDK_BUILD_DIR)/combined/docs/Doxyfile: | $(DOXYGEN_XML_DIR)
 
-build_doxygen_html: | $(DOXYGEN_HTML_DIR)
+build_doxygen_xml: | $(DOXYGEN_XML_DIR)
 
 # Clean all the Doxygen HTML files
-clean_doxygen_html:
+clean_doxygen_xml:
 	rm -rf $(DOXYGEN_PICO_SDK_BUILD_DIR)
 
-# Create the Doxygen asciidoc files
-# Also need to move index.adoc to a different name, because it conflicts with the autogenerated index.adoc
-$(ASCIIDOC_DOXYGEN_DIR)/picosdk_index.json $(ASCIIDOC_DOXYGEN_DIR)/index_doxygen.adoc: $(SCRIPTS_DIR)/transform_doxygen_html.py $(PICO_SDK_DIR)/docs/index.h $(DOXYGEN_PICO_SDK_BUILD_DIR)/docs/Doxyfile | $(DOXYGEN_HTML_DIR) $(ASCIIDOC_DOXYGEN_DIR)
+# create the sdk adoc and the json file
+$(ASCIIDOC_DOXYGEN_DIR)/picosdk_index.json $(ASCIIDOC_DOXYGEN_DIR)/index_doxygen.adoc: $(ASCIIDOC_DOXYGEN_DIR) $(DOXYGEN_XML_DIR)/index.xml doxygentoasciidoc/__main__.py doxygentoasciidoc/cli.py doxygentoasciidoc/nodes.py doxygentoasciidoc/helpers.py | $(BUILD_DIR)
 	$(MAKE) clean_ninja
-	$< $(DOXYGEN_HTML_DIR) $(ASCIIDOC_DOXYGEN_DIR) $(PICO_SDK_DIR)/docs/index.h $(ASCIIDOC_DOXYGEN_DIR)/picosdk_index.json
-	cp $(DOXYGEN_HTML_DIR)/*.png $(ASCIIDOC_DOXYGEN_DIR)
-	mv $(ASCIIDOC_DOXYGEN_DIR)/index.adoc $(ASCIIDOC_DOXYGEN_DIR)/index_doxygen.adoc
+	python3 -m doxygentoasciidoc -f $(DOXYGEN_XML_DIR)/index.xml > $(ASCIIDOC_DOXYGEN_DIR)/all_groups.adoc
+	python3 -m doxygentoasciidoc -f $(DOXYGEN_XML_DIR)/indexpage.xml -c > $(ASCIIDOC_DOXYGEN_DIR)/index_doxygen.adoc
+	python3 -m doxygentoasciidoc -f $(DOXYGEN_XML_DIR)/examples_page.xml -c > $(ASCIIDOC_DOXYGEN_DIR)/examples_page.adoc
+	python3 $(SCRIPTS_DIR)/postprocess_doxygen_adoc.py $(ASCIIDOC_DOXYGEN_DIR)
+	-cp $(DOXYGEN_XML_DIR)/*.png $(ASCIIDOC_DOXYGEN_DIR)
 
 build_doxygen_adoc: $(ASCIIDOC_DOXYGEN_DIR)/index_doxygen.adoc
 

jekyll-assets/css/style.css

@@ -707,6 +707,53 @@ div.videoblock iframe {
 
 /* DOXYGEN ELEMENTS */
 
+.contexttag {
+  display: inline-block;
+  font-size: 0.8em;
+  line-height: 1em;
+  font-weight: bold;
+  background-color: orange;
+  color: #ffffff;
+  border-radius: 0.5em;
+  padding-left: 0.5em;
+  padding-right: 0.5em;
+  padding-top: 1px;
+  padding-bottom: 1px;
+}
+
+.contexttag.RP2040 {
+  background-color: #50C878;
+}
+
+div.listingblock pre.highlight {
+  margin-top: 0px;
+  margin-bottom: 0px;
+}
+
+#content div.listingblock table.linenotable {
+  margin-bottom: 0px;
+}
+
+#content td.hdlist1 {
+  line-height: 1.5em;
+}
+
+#content td.hdlist2 > p {
+  margin-bottom: 0px;
+}
+
+#content td.linenos {
+  padding-right: 10px;
+}
+
+.highlight td.code pre {
+  background-color: transparent;
+  margin-top: 0px;
+  margin-bottom: 0px;
+}
+
+/* OLD DOXYGEN ELEMENTS */
+
 div.memproto {
   background-color: #dedede;
   padding: 7px;

jekyll-assets/scripts/copy-to-clipboard.js

@@ -28,7 +28,7 @@ var hideTooltip = function() {
 };
 
 var extractDoxygenCode = function(node) {
-  var lines = node.querySelectorAll("div.line");
+  var lines = node.querySelectorAll("div.code");
   var preText = "";
   for (var i = 0; i < lines.length; i++) {
     var myText = lines[i].textContent;
@@ -45,8 +45,9 @@ for (var i = 0; i < buttons.length; i++) {
 window.addEventListener('load', function() {
   var clipboard = new ClipboardJS('.copy-button', {
     text: function(trigger) {
-      if (trigger.parentNode.querySelector('div.line')) {
-        var text = extractDoxygenCode(trigger.parentNode);
+      if (trigger.parentNode.querySelector('td.code')) {
+        // var text = extractDoxygenCode(trigger.parentNode);
+        var text = trigger.parentNode.querySelector('td.code pre').textContent;
       } else {
         var text = trigger.parentNode.querySelector('pre').textContent;
 

requirements.txt

@@ -1,2 +1,3 @@
 pyyaml == 6.0.1
 lxml
+beautifulsoup4

scripts/create_auto_ninjabuild.py

@@ -136,7 +136,7 @@ def add_entire_directory(tab_dir, dir_path, pages_set, src_images, dest_images):
         ninja.variable('documentation_index', index_json)
         ninja.variable('output_index', os.path.join(output_dir, "_data", "index.json"))
         ninja.variable('site_config', config_yaml)
-        ninja.variable('doxyfile', os.path.join(doxygen_pico_sdk_build_dir, "docs", "Doxyfile"))
+        ninja.variable('doxyfile', os.path.join(doxygen_pico_sdk_build_dir, "combined", "docs", "Doxyfile"))
         ninja.newline()
 
         targets = []

scripts/create_build_adoc_doxygen.py

@@ -16,8 +16,9 @@ def check_no_markdown(filename):
         asciidoc = re.sub(r'----\n.*?\n----', '', asciidoc, flags=re.DOTALL)
         # strip out pass-through blocks
         asciidoc = re.sub(r'\+\+\+\+\n.*?\n\+\+\+\+', '', asciidoc, flags=re.DOTALL)
-        if re.search(r'(?:^|\n)#+', asciidoc):
-            raise Exception("{} contains a Markdown-style header (i.e. '#' rather than '=')".format(filename))
+        # This is messing up the c code blocks
+        # if re.search(r'(?:^|\n)#+', asciidoc):
+        #     raise Exception("{} contains a Markdown-style header (i.e. '#' rather than '=')".format(filename))
         if re.search(r'(\[.+?\]\(.+?\))', asciidoc):
             raise Exception("{} contains a Markdown-style link (i.e. '[title](url)' rather than 'url[title]')".format(filename))
 
@@ -45,10 +46,7 @@ def check_no_markdown(filename):
             if 'from_json' in tab and 'directory' in tab and tab['directory'] == output_subdir:
                 filebase = os.path.splitext(adoc_filename)[0]
                 index_title = filebase
-                if filebase != "index_doxygen":
-                    picosdk_filename = re.sub("_", "__", filebase)+".html"
-                else:
-                    picosdk_filename = filebase+".html"
+                picosdk_filename = filebase+".html"
                 for item in picosdk_data:
                     if re.sub("^group__", "", item["html"]) == picosdk_filename:
                         index_title = item['name']

scripts/create_nav.py

@@ -131,6 +131,7 @@ def read_file_with_includes(filepath, filelevel, mainfile, output_dir=None):
                                         newlevel = len(m.group(1))
                                         # Need to compute anchors for *every* header (updates file_headings)
                                         heading = strip_adoc(m.group(2))
+                                        heading = re.sub(r"(\[\.contexttag )(\S+)(\]\*\S+\*)", "<strong class=\"contexttag \\2\">\\2</strong>", heading)
                                         anchor = heading_to_anchor(top_level_file, heading, header_id)
                                         if anchor in available_anchors[fullpath]:
                                             raise Exception("Anchor {} appears twice in {}".format(anchor, fullpath))

scripts/postprocess_doxygen_adoc.py

@@ -0,0 +1,149 @@
+import re
+import sys
+import os
+import json
+
+def cleanup_text_page(adoc_file, output_adoc_path, link_targets):
+	filename = os.path.basename(adoc_file)
+	with open(adoc_file) as f:
+		adoc_content = f.read()
+	# remove any errant spaces before anchors
+	adoc_content = re.sub(r'( +)(\[\[[^[]*?\]\])', "\\2", adoc_content)
+	# collect link targets
+	for line in adoc_content.split('\n'):
+		link_targets = collect_link_target(line, filename)
+	with open(adoc_file, 'w') as f:
+		f.write(adoc_content)
+	return link_targets
+
+def collect_link_target(line, chapter_filename):
+	# collect a list of all link targets, so we can fix internal links
+	l = re.search(r'(#)([^,\]]+)([,\]])', line)
+	if l is not None:
+		link_targets[l.group(2)] = chapter_filename
+	return link_targets
+
+def resolve_links(adoc_file, link_targets):
+	filename = os.path.basename(adoc_file)
+	with open(adoc_file) as f:
+		adoc_content = f.read()
+	output_content = []
+	for line in adoc_content.split('\n'):
+		# e.g., <<examples_page,here>>
+		m = re.search("(<<)([^,]+)(,?[^>]*>>)", line)
+		if m is not None:
+			target = m.group(2)
+			# only resolve link if it points to another file
+			if target in link_targets and link_targets[target] != filename:
+				new_target = link_targets[target]+"#"+target
+				line = re.sub("(<<)([^,]+)(,?[^>]*>>)", f"\\1{new_target}\\3", line)
+		output_content.append(line)
+	with open(adoc_file, 'w') as f:
+		f.write('\n'.join(output_content))
+	return
+
+def build_json(sections, output_path):
+	json_path = os.path.join(output_path, "picosdk_index.json")
+	with open(json_path, 'w') as f:
+		f.write(json.dumps(sections, indent="\t"))
+	return
+
+def tag_content(adoc_content):
+	# this is dependent on the same order of attributes every time
+	ids_to_tag = re.findall(r'(\[#)(.*?)(,.*?contextspecific,tag=)(.*?)(,type=)(.*?)(\])', adoc_content)
+	for this_id in ids_to_tag:
+		tag = re.sub("PICO_", "", this_id[3])
+		img = f" [.contexttag {tag}]*{tag}*"
+		# `void <<group_hardware_gpio_1ga5d7dbadb2233e2e6627e9101411beb27,gpio_rp2040>> ()`:: An rp2040 function.
+		adoc_content = re.sub(rf'(\n`.*?<<{this_id[1]},.*?`)(::)', f"\\1{img}\\2", adoc_content)
+		# |<<group_hardware_base,hardware_base>>\n|Low-level types and (atomic) accessors for memory-mapped hardware registers.
+		adoc_content = re.sub(rf'(\n\|<<{this_id[1]},.*?>>\n\|.*?)(\n)', f"\\1{img}\\2", adoc_content)
+	# [#group_cyw43_ll_1ga0411cd49bb5b71852cecd93bcbf0ca2d,role=contextspecific,tag=PICO_RP2040,type=PICO_RP2040]\n=== anonymous enum
+	HEADING_RE = re.compile(r'(\[#.*?role=contextspecific.*?tag=P?I?C?O?_?)(.*?)(,.*?\]\s*?\n\s*=+\s+\S*?)(\n)')
+	# [#group_cyw43_ll_1ga0411cd49bb5b71852cecd93bcbf0ca2d,role=h6 contextspecific,tag=PICO_RP2040,type=PICO_RP2040]\n*anonymous enum*
+	H6_HEADING_RE = re.compile(r'(\[#.*?role=h6 contextspecific.*?tag=P?I?C?O?_?)(.*?)(,.*?\]\s*?\n\s*\*\S+.*?)(\n)')
+	# [#group_cyw43_ll_1ga0411cd49bb5b71852cecd93bcbf0ca2d,role=h6 contextspecific,tag=PICO_RP2040,type=PICO_RP2040]\n----
+	NONHEADING_RE = re.compile(r'(\[#.*?role=h?6?\s?contextspecific.*?tag=P?I?C?O?_?)(.*?)(,.*?\]\s*?\n\s*[^=\*])')
+	adoc_content = re.sub(HEADING_RE, f'\\1\\2\\3 [.contexttag \\2]*\\2*\n', adoc_content)
+	adoc_content = re.sub(H6_HEADING_RE, f'\\1\\2\\3 [.contexttag \\2]*\\2*\n', adoc_content)
+	adoc_content = re.sub(NONHEADING_RE, f'[.contexttag \\2]*\\2*\n\n\\1\\2\\3', adoc_content)
+	return adoc_content
+
+def postprocess_doxygen_adoc(adoc_file, output_adoc_path, link_targets):
+	output_path = re.sub(r'[^/]+$', "", adoc_file)
+	sections = [{
+		"group_id": "index_doxygen",
+		"name": "Introduction",
+		"description": "An introduction to the Pico SDK",
+		"html": "index_doxygen.html",
+		"subitems": []
+	}]
+	with open(adoc_file) as f:
+		adoc_content = f.read()
+	# first, lets add any tags
+	adoc_content = tag_content(adoc_content)
+	# now split the file into top-level sections:
+	# toolchain expects all headings to be two levels lower
+	adoc_content = re.sub(r'(\n==)(=+ \S+)', "\n\\2", adoc_content)
+	# then make it easier to match the chapter breaks
+	adoc_content = re.sub(r'(\[#.*?,reftext=".*?"\])(\s*\n)(= )', "\\1\\3", adoc_content)
+	# find all the chapter descriptions, to use later
+	descriptions = re.findall(r'(\[#.*?,reftext=".*?"\])(= .*?\n\s*\n)(.*?)(\n)', adoc_content)
+	CHAPTER_START_RE = re.compile(r'(\[#)(.*?)(,reftext=".*?"\]= )(.*?$)')
+	# check line by line; if the line matches our chapter break,
+	# then pull all following lines into the chapter list until a new match.
+	chapter_filename = "all_groups.adoc"
+	current_chapter = None
+	chapter_dict = {}
+	counter = 0
+	for line in adoc_content.split('\n'):
+		link_targets = collect_link_target(line, chapter_filename)
+		m = CHAPTER_START_RE.match(line)
+		if m is not None:
+			# write the previous chapter
+			if current_chapter is not None:
+				with open(chapter_path, 'w') as f:
+					f.write('\n'.join(current_chapter))
+			# start the new chapter
+			current_chapter = []
+			# set the data for this chapter
+			group_id = re.sub("^group_+", "", m.group(2))
+			chapter_filename = group_id+".adoc"
+			chapter_path = os.path.join(output_path, chapter_filename)
+			chapter_dict = {
+				"group_id": group_id,
+				"html": group_id+".html",
+				"name": m.group(4),
+				"subitems": [],
+				"description": descriptions[counter][2]
+			}
+			sections.append(chapter_dict)
+			# re-split the line into 2
+			start_line = re.sub("= ", "\n= ", line)
+			current_chapter.append(start_line)
+			counter += 1
+		else:
+			current_chapter.append(line)
+	# write the last chapter
+	if current_chapter is not None:
+		with open(chapter_path, 'w') as f:
+			f.write('\n'.join(current_chapter))
+	build_json(sections, output_path)
+	os.remove(adoc_file)
+	return link_targets
+
+if __name__ == '__main__':
+	output_adoc_path = sys.argv[1]
+	adoc_files = [f for f in os.listdir(output_adoc_path) if re.search(".adoc", f) is not None]
+	link_targets = {}
+	for adoc_file in adoc_files:
+		adoc_filepath = os.path.join(output_adoc_path, adoc_file)
+		if re.search("all_groups.adoc", adoc_file) is not None:
+			link_targets = postprocess_doxygen_adoc(adoc_filepath, output_adoc_path, link_targets)
+		else:
+			link_targets = cleanup_text_page(adoc_filepath, output_adoc_path, link_targets)
+	# now that we have a complete list of all link targets, resolve all internal links
+	adoc_files = [f for f in os.listdir(output_adoc_path) if re.search(".adoc", f) is not None]
+	for adoc_file in adoc_files:
+		adoc_filepath = os.path.join(output_adoc_path, adoc_file)
+		resolve_links(adoc_filepath, link_targets)