✨ Add needs_json_exclude_fields configuration (#1246)

This PR replaces the hardcoded `NeedsList. JSON_KEY_EXCLUSIONS_NEEDS` with the `needs_json_exclude_fields` configuration, such that users can override what fields are excluded.

Additionally, back links have been re-instated as being output as default
(the size increase of the `needs.json` can now be mitigated by `needs_json_remove_defaults` and/or `needs_json_exclude_fields`)

Finally, `"needs_defaults_removed": true` is now output in the `needs.json`, if defaults have been removed. This will allow tools reading the file to identify this.

docs/builders.rst

@@ -66,9 +66,11 @@ Format
 As well as the ``filters`` and ``needs`` data, the **needs.json** file also contains the ``needs_schema``.
 This is a JSON schema of for the data structure of a single need,
 and also includes a ``field_type`` for each field, to denote the source of the field,
-that can be one of: ``core``, ``links``, ``extra``, ``global``.
+that can be one of: ``core``, ``links``, ``backlinks``, ``extra``, ``global``.
 
-See also :ref:`needs_build_json_per_id` and :ref:`needs_json_remove_defaults` for more options on modifying the content of the ``needs.json`` file.
+See also :ref:`needs_json_exclude_fields`, :ref:`needs_json_remove_defaults`, and :ref:`needs_reproducible_json` for more options on modifying the content of the ``needs.json`` file.
+
+.. note:: ``needs_defaults_removed`` is a flag that is set to ``true`` if the defaults are removed from the needs. If it is missing or set to ``false``, the defaults are not removed.
 
 .. code-block:: python
 
@@ -123,12 +125,12 @@ See also :ref:`needs_build_json_per_id` and :ref:`needs_json_remove_defaults` fo
                     ...
                 }
             },
+            "needs_defaults_removed": true,
             "needs": {
                 "IMPL_01": {
                     "id": "IMPL_01",
                     "type": "impl",
                     "links": ["OWN_ID_123"],
-                    "status": null,
                     ...
                 },
                 ...
@@ -177,6 +179,7 @@ See also :ref:`needs_build_json_per_id` and :ref:`needs_json_remove_defaults` fo
                 },
                 ...
             },
+            "needs_defaults_removed": true,
             "needs": {
                 "IMPL_01": {
                     "id": "IMPL_01",

docs/conf.py

@@ -643,6 +643,7 @@ def custom_defined_func():
 
 # build needs.json to make permalinks work
 needs_build_json = True
+needs_json_remove_defaults = True
 
 # build needs_json for every needs-id to make detail panel
 needs_build_json_per_id = False
@@ -702,8 +703,10 @@ def custom_defined_func():
 from docutils import nodes  # noqa: E402
 from sphinx.application import Sphinx  # noqa: E402
 from sphinx.directives import SphinxDirective  # noqa: E402
+from sphinx.roles import SphinxRole  # noqa: E402
 
 from sphinx_needs.api.need import add_external_need  # noqa: E402
+from sphinx_needs.config import NeedsSphinxConfig  # noqa: E402
 from sphinx_needs.data import SphinxNeedsData  # noqa: E402
 from sphinx_needs.needsfile import NeedsList  # noqa: E402
 
@@ -741,6 +744,14 @@ def run(self):
         return [root]
 
 
+class NeedConfigDefaultRole(SphinxRole):
+    """Role to add a default configuration value to the documentation."""
+
+    def run(self):
+        default = NeedsSphinxConfig.get_default(self.text)
+        return [[nodes.literal("", repr(default), language="python")], []]
+
+
 def create_tutorial_needs(app: Sphinx, _env, _docnames):
     """Create a JSON to import in the tutorial.
 
@@ -771,4 +782,5 @@ def create_tutorial_needs(app: Sphinx, _env, _docnames):
 
 def setup(app: Sphinx):
     app.add_directive("need-example", NeedExampleDirective)
+    app.add_role("need_config_default", NeedConfigDefaultRole())
     app.connect("env-before-read-docs", create_tutorial_needs, priority=600)

docs/configuration.rst

@@ -1795,6 +1795,17 @@ needs_reproducible_json
 Setting ``needs_reproducible_json = True`` will ensure the ``needs.json`` output is reproducible,
 e.g. by removing timestamps from the output.
 
+.. _needs_json_exclude_fields:
+
+needs_json_exclude_fields
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.2.0
+
+Setting ``needs_json_exclude_fields = ["key1", "key2"]`` will exclude the given fields from all needs in the ``needs.json`` output.
+
+Default: :need_config_default:`json_exclude_fields`
+
 .. _needs_json_remove_defaults:
 
 needs_json_remove_defaults 
@@ -1803,8 +1814,11 @@ needs_json_remove_defaults
 .. versionadded:: 2.1.0
 
 Setting ``needs_json_remove_defaults = True`` will remove all need fields with default from ``needs.json``, greatly reducing its size.
+
 The defaults can be retrieved from the ``needs_schema`` now also output in the JSON file (see :ref:`this section <needs_builder_format>` for the format).
 
+Default: :need_config_default:`json_remove_defaults`
+
 .. _needs_build_json_per_id:
 
 needs_build_json_per_id 

sphinx_needs/config.py

@@ -7,7 +7,7 @@
 from sphinx.application import Sphinx
 from sphinx.config import Config as _SphinxConfig
 
-from sphinx_needs.data import GraphvizStyleType
+from sphinx_needs.data import GraphvizStyleType, NeedsCoreFields
 from sphinx_needs.defaults import DEFAULT_DIAGRAM_TEMPLATE
 
 if TYPE_CHECKING:
@@ -442,6 +442,15 @@ def __setattr__(self, name: str, value: Any) -> None:
         default=False, metadata={"rebuild": "html", "types": (bool,)}
     )
     """If True, the JSON needs file should be idempotent for multiple builds fo the same documentation."""
+    json_exclude_fields: list[str] = field(
+        default_factory=lambda: [
+            name
+            for name, params in NeedsCoreFields.items()
+            if params.get("exclude_json")
+        ],
+        metadata={"rebuild": "html", "types": (list,)},
+    )
+    """List of keys to exclude from the JSON needs file."""
     json_remove_defaults: bool = field(
         default=False, metadata={"rebuild": "html", "types": (bool,)}
     )
@@ -525,3 +534,11 @@ def add_config_values(cls, app: Sphinx) -> None:
                 item.metadata["rebuild"],
                 types=item.metadata["types"],
             )
+
+    @classmethod
+    def get_default(cls, name: str) -> Any:
+        """Get the default value for a config item."""
+        _field = next(field for field in fields(cls) if field.name == name)
+        if _field.default_factory is not MISSING:
+            return _field.default_factory()
+        return _field.default

sphinx_needs/data.py

@@ -63,7 +63,7 @@ class CoreFieldParameters(TypedDict):
     show_in_layout: NotRequired[bool]
     """Whether to show the field in the rendered layout of the need by default (False if not present)."""
     exclude_json: NotRequired[bool]
-    """Whether to exclude the field from the JSON representation (False if not present)."""
+    """Whether field should be part of the default exclusions from the JSON representation (False if not present)."""
 
 
 NeedsCoreFields: Final[Mapping[str, CoreFieldParameters]] = {

sphinx_needs/needsfile.py

@@ -9,8 +9,9 @@
 import json
 import os
 import sys
+from copy import deepcopy
 from datetime import datetime
-from typing import Any
+from typing import Any, Iterable
 
 from jsonschema import Draft7Validator
 from sphinx.config import Config
@@ -22,7 +23,9 @@
 log = get_logger(__name__)
 
 
-def generate_needs_schema(config: Config) -> dict[str, Any]:
+def generate_needs_schema(
+    config: Config, exclude_properties: Iterable[str] = ()
+) -> dict[str, Any]:
     """Generate a JSON schema for all fields in each need item.
 
     It is based on:
@@ -46,9 +49,7 @@ def generate_needs_schema(config: Config) -> dict[str, Any]:
     # (this is the case for `type` added by the github service)
     # hence this is why we add the core options after the extra options
     for name, core_params in NeedsCoreFields.items():
-        if core_params.get("exclude_json"):
-            continue
-        properties[name] = core_params["schema"]
+        properties[name] = deepcopy(core_params["schema"])
         properties[name]["description"] = f"{core_params['description']}"
         properties[name]["field_type"] = "core"
 
@@ -62,6 +63,13 @@ def generate_needs_schema(config: Config) -> dict[str, Any]:
             "field_type": "links",
             "default": [],
         }
+        properties[link["option"] + "_back"] = {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": "Backlink field",
+            "field_type": "backlinks",
+            "default": [],
+        }
 
     for name in needs_config.global_options:
         if name not in properties:
@@ -72,6 +80,10 @@ def generate_needs_schema(config: Config) -> dict[str, Any]:
                 "default": "",
             }
 
+    for name in exclude_properties:
+        if name in properties:
+            del properties[name]
+
     return {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "type": "object",
@@ -80,27 +92,13 @@ def generate_needs_schema(config: Config) -> dict[str, Any]:
 
 
 class NeedsList:
-    JSON_KEY_EXCLUSIONS_NEEDS = {
-        name for name, params in NeedsCoreFields.items() if params.get("exclude_json")
-    }
-
     def __init__(
         self, config: Config, outdir: str, confdir: str, add_schema: bool = True
     ) -> None:
         self.config = config
         self.needs_config = NeedsSphinxConfig(config)
         self.outdir = outdir
         self.confdir = confdir
-        self._schema = generate_needs_schema(config) if add_schema else None
-        self._need_defaults = (
-            {
-                name: value["default"]
-                for name, value in self._schema["properties"].items()
-                if "default" in value
-            }
-            if self._schema
-            else {}
-        )
         self.current_version = config.version
         self.project = config.project
         self.needs_list = {
@@ -111,9 +109,23 @@ def __init__(
         if not self.needs_config.reproducible_json:
             self.needs_list["created"] = ""
         self.log = log
-        # also exclude back links for link types dynamically set by the user
-        back_link_keys = {x["option"] + "_back" for x in self.needs_config.extra_links}
-        self._exclude_need_keys = self.JSON_KEY_EXCLUSIONS_NEEDS | back_link_keys
+
+        self._exclude_need_keys = set(self.needs_config.json_exclude_fields)
+
+        self._schema = (
+            generate_needs_schema(config, exclude_properties=self._exclude_need_keys)
+            if add_schema
+            else None
+        )
+        self._need_defaults = (
+            {
+                name: value["default"]
+                for name, value in self._schema["properties"].items()
+                if "default" in value
+            }
+            if self._schema
+            else {}
+        )
 
     def update_or_add_version(self, version: str) -> None:
         if version not in self.needs_list["versions"].keys():
@@ -125,6 +137,8 @@ def update_or_add_version(self, version: str) -> None:
             }
             if self._schema:
                 self.needs_list["versions"][version]["needs_schema"] = self._schema
+            if self.needs_config.json_remove_defaults:
+                self.needs_list["versions"][version]["needs_defaults_removed"] = True
             if not self.needs_config.reproducible_json:
                 self.needs_list["versions"][version]["created"] = ""
 

tests/__snapshots__/test_basic_doc.ambr

@@ -39,13 +39,17 @@
             'layout': '',
             'links': list([
             ]),
+            'links_back': list([
+            ]),
             'max_amount': '',
             'max_content_lines': '',
             'modifications': 0,
             'params': '',
             'parent_need': '',
             'parent_needs': list([
             ]),
+            'parent_needs_back': list([
+            ]),
             'parts': dict({
             }),
             'post_template': None,
@@ -105,13 +109,17 @@
             'layout': '',
             'links': list([
             ]),
+            'links_back': list([
+            ]),
             'max_amount': '',
             'max_content_lines': '',
             'modifications': 0,
             'params': '',
             'parent_need': '',
             'parent_needs': list([
             ]),
+            'parent_needs_back': list([
+            ]),
             'parts': dict({
             }),
             'post_template': None,
@@ -339,6 +347,16 @@
               }),
               'type': 'array',
             }),
+            'links_back': dict({
+              'default': list([
+              ]),
+              'description': 'Backlink field',
+              'field_type': 'backlinks',
+              'items': dict({
+                'type': 'string',
+              }),
+              'type': 'array',
+            }),
             'max_amount': dict({
               'default': '',
               'description': 'Added by service github-issues',
@@ -379,6 +397,16 @@
               }),
               'type': 'array',
             }),
+            'parent_needs_back': dict({
+              'default': list([
+              ]),
+              'description': 'Backlink field',
+              'field_type': 'backlinks',
+              'items': dict({
+                'type': 'string',
+              }),
+              'type': 'array',
+            }),
             'parts': dict({
               'additionalProperties': dict({
                 'type': 'object',