Merge branch 'master' into master

.github/workflows/benchmark.yaml

@@ -8,7 +8,7 @@ jobs:
     name: "Benchmark time & memory"
     runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
       - name: Set Up Python
         uses: actions/setup-python@v4
         with:

.github/workflows/ci.yaml

@@ -23,7 +23,7 @@ jobs:
             sphinx-version: "4.5.0"
             docutils-version: "0.15"
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
       - name: Set Up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -50,7 +50,7 @@ jobs:
     name: Lint
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.github/workflows/docker.yaml

@@ -31,14 +31,14 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE
       - name: Checkout 🛎️
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
       -
         name: Set up Docker Build 🐋
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: |
             ${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.image }}
@@ -56,14 +56,14 @@ jobs:
       -
         # Login to Docker Hub Docker Registry for deployment
         name: Login to Docker Hub Docker Registry 🐸
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push html image to Doker Hub Docker Registry 🐳
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           push: ${{ github.event_name != 'pull_request' && steps.deploycheck.outputs.value == 'y' }}
           file: docker/Dockerfile

.github/workflows/docs.yaml

@@ -7,7 +7,7 @@ jobs:
     name: Docs-Linkcheck
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
       - name: Set Up Python
         uses: actions/setup-python@v4
         with:

.github/workflows/js_test.yml

@@ -12,7 +12,7 @@ jobs:
       uses: actions/setup-node@v3
       with:
         node-version: 18
-    - uses: actions/checkout@v3.5.3
+    - uses: actions/checkout@v4
     - name: Update pip
       run: |
         pip install -U wheel

.github/workflows/release.yaml

@@ -8,7 +8,7 @@ jobs:
     name: Build packages
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
@@ -55,7 +55,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: publish_test
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:

.readthedocs.yml

@@ -2,6 +2,8 @@ version: 2
 
 sphinx:
   configuration: docs/conf.py
+  builder: html
+  fail_on_warning: true
 
 python:
   install:

docs/builders.rst

@@ -163,3 +163,21 @@ or
 .. hint::
 
     As an alternative, you can set the config option :ref:`needs_build_needumls` to export the needumls files during each build.
+
+
+.. _needs_id_builder:
+
+needs_id
+--------
+.. versionadded:: 1.4.0
+
+The **needs_id** builder exports all found needs and selected filter results to a set json files of each need with the name is ``id`` of need.
+
+The build creates a folder called :ref:``needs_build_json_per_id_path`` and all file json of each need inside the given build-folder.
+
+Usage
++++++
+
+.. code-block:: bash
+
+    sphinx-build -b needs_id source_dir build_dir

docs/changelog.rst

@@ -7,7 +7,7 @@ License
 -------
 
 .. include:: ../LICENSE
-
+  
 2.0.0
 -----
 Released: under development
@@ -16,11 +16,13 @@ Released: under development
 -----
 Released: under development
 
+* Improvement: Added Builder :ref:`needs_id_builder` added and config option :ref:`needs_build_json_per_id` in `conf.py`. 
 * Improvement: Reduce document build time, by memoizing the inline parse in ``build_need`` (`#968 <https://github.com/useblocks/sphinx-needs/pull/968>`_)
-
 * Change `NeedsBuilder` format to `needs` (`#978 <https://github.com/useblocks/sphinx-needs/pull/978>`_)
-
 * Improvement: Suffix all warnings with ``[needs]``, and allow them to be suppressed (`#975 <https://github.com/useblocks/sphinx-needs/pull/975>`_)
+* Improvement: :ref:`needextend` for single needs is much faster.
+* Improvement: General performance improvement (up to 50%) and less memory consumption (~40%).
+* Improvement: external_needs is using cached templates to save generation time.
 
 * Bugfix: Check filter strings for correctness.
   (`#964 <https://github.com/useblocks/sphinx-needs/pull/964>`_)
@@ -916,3 +918,5 @@ custom css definitions you need to update them.
 * Integrated interaction with the activated plantuml sphinx extension
 
 * Added role **need** to create a reference to a need by giving the id
+
+

docs/conf.py

@@ -62,6 +62,13 @@
     "sphinx": ("https://www.sphinx-doc.org/en/master", None),
 }
 
+nitpicky = True
+nitpick_ignore = [
+    ("py:class", "T"),
+    ("py:class", "sphinx_needs.debug.T"),
+    ("py:class", "sphinx_needs.data.NeedsInfoType"),
+]
+
 # smartquotes = False
 
 needs_debug_measurement = True
@@ -361,6 +368,9 @@ def custom_defined_func():
 # build needs.json to make permalinks work
 needs_build_json = True
 
+# build needs_json for every needs-id to make detail panel
+needs_build_json_per_id = False
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

docs/configuration.rst

@@ -1792,6 +1792,50 @@ Example:
    The created ``needs.json`` file gets stored in the ``outdir`` of the current builder.
    So if ``html`` is used as builder, the final location is e.g. ``_build/html/needs.json``.
 
+
+.. _needs_build_json_per_id:
+
+needs_build_json_per_id 
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.4.0
+    
+Builds list json files for each need. The name of each file is the ``id`` of need.
+This option works like :ref:`needs_build_json`. 
+
+Default: False
+
+Example:
+
+.. code-block:: python
+
+    needs_build_json_per_id = False
+
+.. hint::
+
+    The created single json file per need, located in :ref:`needs_build_json_per_id_path` folder, e.g ``_build/needs_id/abc_432.json``
+ 
+.. _needs_build_json_per_id_path:
+
+needs_build_json_per_id_path 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.4.0
+    
+This option sets the location of the set of ``needs.json`` for every needs-id.
+
+Default value: ``needs_id``
+
+Example:
+
+.. code-block:: python
+
+    needs_build_json_per_id_path = "needs_id"
+
+.. hint::
+
+    The created ``needs_id`` folder gets stored in the ``outdir`` of the current builder. The final location is e.g. ``_build/needs_id``
+
 .. _needs_build_needumls:
 
 needs_build_needumls
@@ -1895,6 +1939,22 @@ constraints_passed is a bool showing if ALL constraints of a corresponding need
 
 constraints_results is a dictionary similar in structure to needs_constraints above. Instead of executable python statements, inner values contain a bool describing if check_0, check_1 ... passed successfully.
 
+.. versionadded:: 1.4.0
+
+    The ``"error_message"`` key can contain a string, with Jinja templating, which will be displayed if the constraint fails, and saved on the need as ``constraints_error``:
+
+    .. code-block:: python
+
+        needs_constraints = {
+
+            "critical": {
+                "check_0": "'critical' in tags",
+                "severity": "CRITICAL",
+                "error_message": "need {% raw %}{{id}}{% endraw %} does not fulfill CRITICAL constraint, because tags are {% raw %}{{tags}}{% endraw %}"
+            }
+        
+        }
+
 
 .. code-block:: rst
 

docs/contributing.rst

@@ -56,6 +56,7 @@ installed in an isolated environment by Poetry.
 
 List make targets
 -----------------
+
 **Sphinx-Needs** uses ``make`` to invoke most development related actions.
 
 Use ``make list`` to get a list of available targets.
@@ -92,20 +93,45 @@ To check if all used links in the documentation are still valid, run:
 
 Running Tests
 -------------
-.. hint::
 
-   Please be sure to have the dependencies of the official documentation installed:
+You can either run the tests directly using ``pytest``, in an existing environment:
+
+.. code-block:: bash
+
+   pytest tests/
+
+Or you can use the provided Makefile:
 
 .. code-block:: bash
 
-   pip install -r docs/requirements.txt
    make test
 
+Note some tests use `syrupy <https://github.com/tophat/syrupy>`__ to perform snapshot testing.
+These snapshots can be updated by running:
+
+.. code-block:: bash
+
+   pytest tests/ --snapshot-update
+
+.. hint::
+
+   Please be sure to have the dependencies of the official documentation also installed:
+
+   .. code-block:: bash
+
+      pip install -r docs/requirements.txt
+
 Linting & Formatting
 --------------------
 
-**Sphinx-Needs** uses `black <https://github.com/psf/black>`_ and
-`isort <https://pycqa.github.io/isort/>`_ to format its source code.
+**Sphinx-Needs** uses `pre-commit <https://pre-commit.com/>`__ to run formatting and checking of source code.
+This can be run directly using:
+
+.. code-block:: bash
+
+    pre-commit run --all-files
+
+or via the provided Makefile:
 
 .. code-block:: bash
 
@@ -182,5 +208,66 @@ The release jobs will build the source and wheel distribution and try to upload
 to ``test.pypi.org`` and ``pypy.org``.
 
 
+Structure of the extension's logic
+----------------------------------
+
+The following is an outline of the build events which this extension adds to the :ref:`Sphinx build process <events>`:
+
+#. After configuration has been initialised (``config-inited`` event):
+
+   - Register additional directives, directive options and warnings (``load_config``)
+   - Check configuration consistency (``check_configuration``)
+
+#. Before reading changed documents (``env-before-read-docs`` event):
+
+   - Initialise ``BuildEnvironment`` variables (``prepare_env``)
+   - Register services (``prepare_env``)
+   - Register functions  (``prepare_env``)
+   - Initialise default extra options (``prepare_env``)
+   - Initialise extra link types (``prepare_env``)
+   - Ensure default configurations are set (``prepare_env``)
+   - Start process timing, if enabled (``prepare_env``)
+   - Load external needs (``load_external_needs``)
+
+#. For all removed and changed documents (``env-purge-doc`` event):
+
+   - Remove all cached need items that originate from the document (``purge_needs``)
+
+#. For changed documents (``doctree-read`` event, priority 880 of transforms)
+
+   - Determine and add data on parent sections and needs(``analyse_need_locations``)
+   - Remove ``Need`` nodes marked as ``hidden`` (``analyse_need_locations``)
+
+#. When building in parallel mode (``env-merge-info`` event), merge ``BuildEnvironment`` data (``merge_data``)
+
+#. After all documents have been read and transformed (``env-updated`` event) (NOTE these are skipped for ``needs`` builder)
+
+   - Copy vendored JS libraries (with CSS) to build folder (``install_lib_static_files``)
+   - Generate permalink file (``install_permalink_file``)
+   - Copy vendored CSS files to build folder (``install_styles_static_files``)
+
+#. Note, the ``BuildEnvironment`` is cached at this point, only if any documents were updated.
+
+#. For all changed documents, or their dependants (``doctree-resolved``)
+
+   - Replace all ``Needextract`` nodes with a list of the collected ``Need`` (``process_creator``)
+   - Remove all ``Need`` nodes, if ``needs_include_needs`` is ``True`` (``process_need_nodes``)
+   - Call dynamic functions, set as values on the need data items and replace them with their return values (``process_need_nodes -> resolve_dynamic_values``)
+   - Replace needs data variant values (``process_need_nodes -> resolve_variants_options``)
+   - Check for dead links (``process_need_nodes -> check_links``)
+   - Generate back links (``process_need_nodes -> create_back_links``)
+   - Process constraints, for each ``Need`` node (``process_need_nodes -> process_constraints``)
+   - Perform all modifications on need data items, due to ``Needextend`` nodes (``process_need_nodes -> extend_needs_data``)
+   - Format each ``Need`` node to give the desired visual output (``process_need_nodes -> print_need_nodes``)
+   - Process all other need specific nodes, replacing them with the desired visual output (``process_creator``)
+
+#. At the end of the build (``build-finished`` event)
+
+   - Call all user defined need data checks, a.k.a `needs_warnings` (``process_warnings``)
+   - Write the ``needs.json`` to the output folder, if `needs_build_json = True` (``build_needs_json``)
+   - Write the ``needs.json`` per ID to the output folder, if `needs_build_json_per_id = True` (``build_needs_id_json``)
+   - Write all UML files to the output folder, if `needs_build_needumls = True` (``build_needumls_pumls``)
+   - Print process timing, if `needs_debug_measurement = True`  (``process_timing``)
+
 .. Include our contributors and maintainers.
 .. include:: ../AUTHORS