Generic Sphinx-Needs JS test framework

.github/workflows/ci.yaml

@@ -28,6 +28,12 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Install Cypress Test Framework
+        run: npm install cypress
       - name: Install Nox Dependencies
         run: |
           python -m pip install poetry nox nox-poetry pyparsing==3.0.4

Makefile

@@ -10,11 +10,11 @@ lint:
 
 .PHONY: test
 test:
-	poetry run pytest -n auto --tb=long --ignore=tests/benchmarks tests/
+	poetry run pytest -n auto -m "not jstest" --tb=long --ignore=tests/benchmarks tests/
 
 .PHONY: test
 test-short:
-	poetry run pytest -n auto --tb=long --ignore-glob="*official*" --ignore=tests/benchmarks tests/
+	poetry run pytest -n auto -m "not jstest" --tb=long --ignore-glob="*official*" --ignore=tests/benchmarks tests/
 
 .PHONY: benchmark-time
 benchmark-time:

docs/contributing.rst

@@ -101,6 +101,91 @@ Running Tests
    pip install -r docs/requirements.txt
    make test
 
+Running JavaScript Tests in Python Test Files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**Setup Cypress Locally**
+
+* Install Node JS on your computer and ensure it can be accessed through the CMD.
+* Install Cypress using the npm package manager by running npm install cypress. Read more from this link https://docs.cypress.io/guides/getting-started/installing-cypress#npm-install.
+* Verify if Cypress is installed correctly and is executable by running: npx cypress verify. Read more from this link https://docs.cypress.io/guides/guides/command-line.
+* If everything is successful then we can use Cypress.
+
+**Enable Cypress Test in Python Test Files**
+
+* First, create a test folder to store your Cypress JS test files (files should end with: ``*.cy.js``). For each Cypress JS test file, you will need to write the Cypress JS test cases in the file. You can read more from the `Cypress Docs <https://docs.cypress.io/>`_. You can also check the ``tests/js_test/js-test-sn-collapse-button.cy.js`` file as reference.
+* In your Python test files, you must mark every JS related test case with the marker - ``jstest`` and you also need to pass the ``spec_pattern`` key-value pair as part of the ``test_app`` fixture parameter. For example, your test case could look like the below:
+    .. code-block:: python
+
+        # test_js_code
+
+        import pytest
+
+
+        @pytest.mark.jstest
+        @pytest.mark.parametrize(
+            "test_app",
+            [
+                {
+                    "buildername": "html",
+                    "srcdir": "doc_test/variant_doc",
+                    "tags": ["tag_a"],
+                    "spec_pattern": "js_test/js-test-sn-collapse-button.cy.js"
+                }
+            ],
+            indirect=True,
+        )
+        def test_collapse_button_in_docs(test_app):
+            ...
+
+    .. note::
+
+        The ``spec_pattern`` key is required to ensure Cypress locates your test files or folder. Visit this link for more info on how to set the `spec_pattern <https://docs.cypress.io/guides/guides/command-line#cypress-run-spec-lt-spec-gt>`_.
+
+* After you have set the ``spec_pattern`` key-value pair as part of the ``test_app`` fixture parameter, you can call the ``app.test_js()`` in your Python test case to run the JS test for the ``spec_pattern`` you provided. For example, you can use it like below:
+    .. code-block:: python
+
+        # test_js_code
+
+        import pytest
+
+
+        @pytest.mark.jstest
+        @pytest.mark.parametrize(
+            "test_app",
+            [
+                {
+                    "buildername": "html",
+                    "srcdir": "doc_test/variant_doc",
+                    "tags": ["tag_a"],
+                    "spec_pattern": "js_test/js-test-sn-collapse-button.cy.js"
+                }
+            ],
+            indirect=True,
+        )
+        def test_collapse_button_in_docs(test_app):
+            """Check if the Sphinx-Needs collapse button works in the provided documentation source."""
+            app = test_app
+            app.build()
+
+            # Call `app.test_js()` to run the JS test for a particular specPattern
+            js_test_result = app.test_js()
+
+            # Check the return code and stdout
+            assert js_test_result["returncode"] == 0
+            assert "All specs passed!" in js_test_result["stdout"].decode("utf-8")
+
+    .. note::
+
+        ``app.test_js()`` will return a dictionary object containing the ``returncode``, ``stdout``, and ``stderr``. Example:
+
+        .. code-block:: python
+
+            return {
+                "returncode": 0,
+                "stdout": "Test passed string",
+                "stderr": "Errors encountered,
+            }
+
 Linting & Formatting
 --------------------
 

pyproject.toml

@@ -73,6 +73,9 @@ packaging = "*"
 
 [tool.pytest.ini_options]
 asyncio_mode= "auto"
+markers = [
+    "jstest: marks tests as JavaScript test (deselect with '-m \"not jstest\"')",
+]
 
 [tool.black]
 line-length = 120

tests/conftest.py

@@ -1,8 +1,14 @@
 """Pytest conftest module containing common test configuration and fixtures."""
+import json
+import os.path
 import shutil
+import subprocess
+from pathlib import Path
 from tempfile import mkdtemp
+from typing import Any, Dict
 
 import pytest
+from sphinx.application import Sphinx
 from sphinx.testing.path import path
 
 pytest_plugins = "sphinx.testing.fixtures"
@@ -15,6 +21,82 @@ def copy_srcdir_to_tmpdir(srcdir, tmp):
     return tmproot
 
 
+def get_abspath(relpath):
+    if relpath and isinstance(relpath, str):
+        abspath = Path(__file__).parent.joinpath(relpath).resolve()
+        return str(abspath)
+    return relpath
+
+
+def test_js(self) -> Dict[str, Any]:
+    cypress_testpath = get_abspath(self.spec_pattern)
+
+    if not cypress_testpath and not (os.path.isabs(cypress_testpath) and os.path.exists(cypress_testpath)):
+        return {
+            "returncode": 1,
+            "stdout": None,
+            "stderr": f"The spec_pattern you provided cannot be found. (spec_pattern: {self.spec_pattern})",
+        }
+
+    js_test_config = {
+        "specPattern": cypress_testpath,
+        "supportFile": get_abspath("js_test/cypress/support/e2e.js"),
+        "fixturesFolder": False,
+        "baseUrl": "http://localhost:65323",
+    }
+
+    cypress_config = f"{json.dumps(js_test_config)}"
+    cypress_config_file = get_abspath("js_test/cypress.config.js")
+
+    # Start the HTTP server using subprocess
+    server_process = subprocess.Popen(["python", "-m", "http.server", "-d", f"{self.outdir}", "65323"])
+
+    try:
+        # Run the Cypress test command
+        completed_process = subprocess.run(
+            [
+                "npx",
+                "cypress",
+                "run",
+                "--browser",
+                "chrome",
+                "--config-file",
+                rf"{cypress_config_file}",
+                "--config",
+                rf"{cypress_config}",
+            ],
+            capture_output=True,
+        )
+
+        # To stop the server, we can terminate the process
+        server_process.terminate()
+        server_process.wait(timeout=5)  # Wait for up to 5 seconds for the process to exit
+        # print("Server stopped successfully.")
+
+        # Send back return code, stdout, and stderr
+        return {
+            "returncode": completed_process.returncode,
+            "stdout": completed_process.stdout,
+            "stderr": completed_process.stderr,
+        }
+    except subprocess.TimeoutExpired:
+        server_process.kill()
+        return {
+            "returncode": 1,
+            "stdout": None,
+            "stderr": "Server forcibly terminated due to timeout.",
+        }
+    except (Exception, subprocess.CalledProcessError) as e:
+        # Stop server when an exception occurs
+        server_process.terminate()
+        server_process.wait(timeout=5)  # Wait for up to 5 seconds for the process to exit
+        return {
+            "returncode": 1,
+            "stdout": "Server stopped due to error.",
+            "stderr": e,
+        }
+
+
 @pytest.fixture(scope="function")
 def test_app(make_app, request):
     # We create a temp-folder on our own, as the util-functions from sphinx and pytest make troubles.
@@ -31,8 +113,8 @@ def test_app(make_app, request):
     srcdir = builder_params.get("srcdir")
     src_dir = copy_srcdir_to_tmpdir(srcdir, sphinx_test_tempdir)
 
-    # return sphinx.testing fixture make_app and new srcdir which in sphinx_test_tempdir
-    app = make_app(
+    # return sphinx.testing fixture make_app and new srcdir which is in sphinx_test_tempdir
+    app: Sphinx = make_app(
         buildername=builder_params.get("buildername", "html"),
         srcdir=src_dir,
         freshenv=builder_params.get("freshenv"),
@@ -43,6 +125,10 @@ def test_app(make_app, request):
         docutilsconf=builder_params.get("docutilsconf"),
         parallel=builder_params.get("parallel", 0),
     )
+    # Add the spec_pattern as an attribute to the Sphinx app object
+    app.spec_pattern = builder_params.get("spec_pattern", "")
+    # Add the test_js() function as an attribute to the Sphinx app object
+    app.test_js = test_js.__get__(app, Sphinx)
 
     yield app
 

tests/doc_test/variant_doc/index.rst

@@ -35,10 +35,3 @@ Variant Handling Test
    :maxdepth: 2
    :caption: Contents:
 
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`

tests/js_test/js-test-sn-collapse-button.cy.js

@@ -0,0 +1,86 @@
+describe('Test Sphinx Needs Collapse', () => {
+  it('Visit Sphinx Needs Homepage', () => {
+    // 1. Given a user visits http://localhost:65323/
+    cy.visit('/')
+
+    cy.get('table.need span.needs.needs_collapse').each(($el, index, $list) => {
+    // 2. When page loads, select all elements that matches the selector `table.need span.needs.needs_collapse`
+
+        var id = $el.attr("id");
+        var parts = id.split("__");
+        var rows = parts.slice(2);
+
+        var table = $el.closest('table');
+        var need_table_id = table.closest("div[id^=SNCB-]").attr("id");
+
+        // 3. Check if the id of the element contains show or hide
+        if (parts[1] == "show") {
+            cy.get($el).within(() => {
+              // 4. Then check if `span.needs.visible` has the class `collapse_is_hidden`
+              cy.get('span.needs.visible').should('have.class', 'collapse_is_hidden')
+            })
+        } else {
+            cy.get($el).within(() => {
+              // 4. Then check if `span.needs.collapse` has the class `collapse_is_hidden`
+              cy.get('span.needs.collapsed').should('have.class', 'collapse_is_hidden')
+            })
+
+            for (var row in rows) {
+                // 5. And check if `#${need_table_id} table tr.${rows[row]}` has the class `collapse_is_hidden`
+                cy.get(`#${need_table_id} table tr.${rows[row]}`).should('have.class', 'collapse_is_hidden')
+            }
+        }
+    })
+  })
+})
+
+describe('Test Sphinx Needs Collapse Click', () => {
+  it('Visit Sphinx Needs Directive page', () => {
+    // 1. Given a user visits http://localhost:65323/
+    cy.visit('/')
+
+    cy.get('table.need span.needs.needs_collapse').each(($el, index, $list) => {
+    // 2. When page loads, select all elements that matches the selector `table.need span.needs.needs_collapse`
+
+        var id = $el.attr("id");
+        var parts = id.split("__");
+        var rows = parts.slice(2);
+
+        var table = $el.closest('table');
+        var need_table_id = table.closest("div[id^=SNCB-]").attr("id");
+
+        if (parts[1] == "show") {
+            // 3. Click collapse/expand button
+            cy.get($el).click()
+
+            for (var row in rows) {
+                // 4. And check if `#${need_table_id} table tr.${rows[row]}` has the class `collapse_is_hidden`
+                cy.get(`#${need_table_id} table tr.${rows[row]}`).should('have.class', 'collapse_is_hidden')
+            }
+
+            cy.get($el).within(() => {
+                  // 5. Then check if `span.needs.collapse` has the class `collapse_is_hidden`
+                  cy.get('span.needs.collapsed').should('have.class', 'collapse_is_hidden')
+                  // 6. And check if `span.needs.visible` has the class `collapse_is_hidden`
+                  cy.get('span.needs.visible').should('not.have.class', 'collapse_is_hidden')
+                })
+        } else{
+            // 3. Click collapse/expand button
+            cy.get($el).click()
+
+            for (var row in rows) {
+                // 4. And check if `#${need_table_id} table tr.${rows[row]}` has the class `collapse_is_hidden`
+                cy.get(`#${need_table_id} table tr.${rows[row]}`).should('not.have.class', 'collapse_is_hidden')
+            }
+
+            cy.get($el).within(() => {
+                  // 5. Then check if `span.needs.collapse` has the class `collapse_is_hidden`
+                  cy.get('span.needs.collapsed').should('not.have.class', 'collapse_is_hidden')
+                  // 6. Check if `span.needs.visible` has the class `collapse_is_hidden`
+                  cy.get('span.needs.visible').should('have.class', 'collapse_is_hidden')
+                })
+        }
+
+    })
+  })
+})