Introduce automated tests (#1105)

This introduces snapshot testing for 4 specific test plans which have been pulled into a dedicated `__mocks__` folder:
* alert (using v2 test format version)
* banner (using v1 test format version)
* command-button (using v2 test format version, with more complexity than alert)
* horizontal-slider (using v2 test format version, which includes assertionExceptions)

This PR does significant restructuring of the `create-all-tests` and the `review-tests` scripts to make the results of the main generator scripts testable (ie. content that gets added into the `/build` folder).

.eslintignore

@@ -2,3 +2,6 @@ tests/**/reference/**/*
 build/tests/**/*
 # keys are manually parsed in process-test-directory.js and depend on format
 tests/resources/keys.mjs
+
+# Automated tests related resources
+__test__/__mocks__/tests/**/reference/**/*

.github/workflows/generate-and-commit-files.yml

@@ -5,7 +5,12 @@ on:
 
 jobs:
   generate-and-commit-files:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
 
     steps:
       - uses: actions/checkout@v3
@@ -26,27 +31,6 @@ jobs:
         run: npm run build
 
       - uses: stefanzweifel/git-auto-commit-action@v4
-        if: ${{ github.actor != 'dependabot[bot]' }}
+        if: ${{ matrix.os == 'ubuntu-latest' && github.actor != 'dependabot[bot]' }}
         with:
           commit_message: Generate .html source files with scripts automatically
-
-  generate-files-windows:
-    runs-on: windows-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: refs/pull/${{ github.event.pull_request.number }}/head
-          fetch-depth: 0
-
-      - name: Use Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: '16.x'
-          cache: 'npm'
-
-      - name: Install dependencies
-        run: npm ci
-
-      - name: Generate test and review files
-        run: npm run build

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Run tests
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: refs/pull/${{ github.event.pull_request.number }}/head
+          fetch-depth: 0
+
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16.x'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run npm test
+        run: npm test

__test__/__mocks__/tests/alert/data/assertions.csv

@@ -0,0 +1,3 @@
+assertionId,priority,assertionStatement,assertionPhrase,refIds
+roleAlert,3,Role 'alert' is conveyed,convey role 'alert',
+textHello,1,Text 'Hello' is conveyed,convey text 'Hello',

__test__/__mocks__/tests/alert/data/jaws-commands.csv

@@ -0,0 +1,5 @@
+testId,command,settings,assertionExceptions,presentationNumber
+triggerAlert,space,virtualCursor,,1.0
+triggerAlert,enter,virtualCursor,,1.1
+triggerAlert,space,pcCursor,,2.0
+triggerAlert,enter,pcCursor,,2.1

__test__/__mocks__/tests/alert/data/js/setFocusOnButton.js

@@ -0,0 +1,2 @@
+// sets focus on the 'Trigger Alert' button
+testPageDocument.querySelector('#alert-trigger').focus();

__test__/__mocks__/tests/alert/data/nvda-commands.csv

@@ -0,0 +1,5 @@
+testId,command,settings,assertionExceptions,presentationNumber
+triggerAlert,space,browseMode,,1.0
+triggerAlert,enter,browseMode,,1.1
+triggerAlert,space,focusMode,,2.0
+triggerAlert,enter,focusMode,,2.1

__test__/__mocks__/tests/alert/data/references.csv

@@ -0,0 +1,10 @@
+refId,type,value,linkText
+author,metadata,Isabel Del Castillo,
+authorEmail,metadata,[email protected],
+title,metadata,Alert Example,
+reference,metadata,reference/2022-4-8_144013/alert.html,Test Case Page for Alert Example
+designPattern,metadata,https://www.w3.org/WAI/ARIA/apg/patterns/alert/,APG Pattern: Alert
+example,metadata,https://www.w3.org/WAI/ARIA/apg/patterns/alert/examples/alert/,APG Example: Alert
+alert,aria,alert,alert
+aria-live,aria,aria-live,aria-live
+aria-atomic,aria,aria-atomic,aria-atomic

__test__/__mocks__/tests/alert/data/scripts.csv

@@ -0,0 +1,2 @@
+setupScript,setupScriptDescription
+setFocusOnButton,sets focus on the 'Trigger Alert' button

__test__/__mocks__/tests/alert/data/tests.csv

@@ -0,0 +1,2 @@
+testId,title,presentationNumber,setupScript,instructions,assertions
+triggerAlert,Trigger an alert,3,setFocusOnButton,"Starting at the 'Trigger Alert' button, activate the button to trigger the alert.",roleAlert textHello

__test__/__mocks__/tests/alert/data/voiceover_macos-commands.csv

@@ -0,0 +1,4 @@
+testId,command,settings,assertionExceptions,presentationNumber
+triggerAlert,ctrl+opt+space,,,3.0
+triggerAlert,space,,,3.1
+triggerAlert,enter,,,3.2

__test__/__mocks__/tests/alert/reference/2022-4-8_144013/alert.html

@@ -0,0 +1,37 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Alert Example</title>
+
+    <!--  js and css for this example. -->
+    <link href="css/alert.css" rel="stylesheet">
+    <script src="js/alert.js" type="text/javascript"></script>
+  </head>
+  <body>
+    <main>
+    <h1>Alert Example</h1>
+    <p>
+      The below example demonstrates the <a href="https://w3c.github.io/#alert">design pattern for alert</a>.
+      Activating the <q>Trigger Alert</q> button causes a message to be inserted into the example alert element.
+    </p>
+    <section>
+      <div class="example-header">
+        <h2 id="ex_label">Example</h2>
+      </div>
+      <div role="separator" id="ex_start_sep" aria-labelledby="ex_start_sep ex_label" aria-label="Start of"></div>
+      <div id="ex1">
+        <button type="button" id="alert-trigger">Trigger Alert</button>
+
+        <div id="example" role="alert"></div>
+
+        <!--  The following script element contains the content that will be inserted into the alert element. -->
+        <script type="text/template" id="alert-template">
+          <p>Hello</p>
+        </script>
+      </div>
+      <div role="separator" id="ex_end_sep" aria-labelledby="ex_end_sep ex_label" aria-label="End of"></div>
+    </section>
+    </main>
+  </body>
+</html>

__test__/__mocks__/tests/alert/reference/2022-4-8_144013/alert.setFocusOnButton.html

@@ -0,0 +1,60 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Alert Example</title>
+
+    <!--  js and css for this example. -->
+    <link href="css/alert.css" rel="stylesheet">
+    <script src="js/alert.js" type="text/javascript"></script>
+
+  <!-- Generated by process-test-directory.js -->
+  <script>
+    (function() {
+      function setupScript(testPageDocument) {
+        // setFocusOnButton
+        // sets focus on the 'Trigger Alert' button
+        testPageDocument.querySelector('#alert-trigger').focus();
+
+      };
+      document.addEventListener('click', function(event) {
+        if (event.target.classList.contains('button-run-test-setup')) {
+          event.target.disabled = true;
+          setupScript(document);
+        }
+      });
+    })();
+  </script>
+  <!-- End of generated output --></head>
+  <body>
+    <main>
+    <h1>Alert Example</h1>
+    <p>
+      The below example demonstrates the <a href="https://w3c.github.io/#alert">design pattern for alert</a>.
+      Activating the <q>Trigger Alert</q> button causes a message to be inserted into the example alert element.
+    </p>
+    <section>
+      <div class="example-header">
+        <h2 id="ex_label">Example</h2>
+      </div>
+      <div role="separator" id="ex_start_sep" aria-labelledby="ex_start_sep ex_label" aria-label="Start of"></div>
+      <div id="ex1">
+        <button type="button" id="alert-trigger">Trigger Alert</button>
+
+        <div id="example" role="alert"></div>
+
+        <!--  The following script element contains the content that will be inserted into the alert element. -->
+        <script type="text/template" id="alert-template">
+          <p>Hello</p>
+        </script>
+      </div>
+      <div role="separator" id="ex_end_sep" aria-labelledby="ex_end_sep ex_label" aria-label="End of"></div>
+    </section>
+
+    <!-- Generated by process-test-directory.js -->
+    <div style="position: relative; left: 0; right: 0; height: 2rem;">
+      <button class="button-run-test-setup" autofocus style="height: 100%; width: 100%;">Run Test Setup</button>
+    </div>
+    <!-- End of generated output --></main>
+  </body>
+</html>

__test__/__mocks__/tests/alert/reference/2022-4-8_144013/css/alert.css

@@ -0,0 +1,10 @@
+[role="alert"] {
+  padding: 10px;
+  border: 2px solid hsl(206deg 74% 54%);
+  border-radius: 4px;
+  background: hsl(206deg 74% 90%);
+}
+
+[role="alert"]:empty {
+  display: none;
+}

__test__/__mocks__/tests/alert/reference/2022-4-8_144013/js/alert.js

@@ -0,0 +1,29 @@
+/*
+ *   This content is licensed according to the W3C Software License at
+ *   https://www.w3.org/Consortium/Legal/2015/copyright-software-and-document
+ *
+ */
+
+'use strict';
+
+window.addEventListener('load', function () {
+  var button = document.getElementById('alert-trigger');
+
+  button.addEventListener('click', addAlert);
+});
+
+/*
+ * @function addAlert
+ *
+ * @desc Adds an alert to the page
+ *
+ * @param   {object}  event  -  Standard W3C event object
+ *
+ */
+
+function addAlert() {
+  var example = document.getElementById('example');
+  var template = document.getElementById('alert-template').innerHTML;
+
+  example.innerHTML = template;
+}

__test__/__mocks__/tests/banner/data/commands.csv

@@ -0,0 +1,43 @@
+testId,task,mode,at,commandA,commandB,commandC,commandD,commandE
+1,Navigate forwards into a banner landmark,READING,JAWS,R,DOWN,TAB,U,
+1,Navigate forwards into a banner landmark,READING,NVDA,D,DOWN,TAB,U,K
+2,Navigate forwards into a banner landmark,INTERACTION,JAWS,TAB,,,,
+2,Navigate forwards into a banner landmark,INTERACTION,NVDA,TAB,,,,
+3,Navigate forwards into a banner landmark,INTERACTION,VOICEOVER_MACOS,"CTRL_OPT_RIGHT,CTRL_OPT_RIGHT",CTRL_OPT_CMD_L,,,
+4,Navigate backwards into a banner landmark,READING,JAWS,UP,SHIFT_TAB,SHIFT_U,,
+4,Navigate backwards into a banner landmark,READING,NVDA,UP,SHIFT_TAB,SHIFT_U,SHIFT_K,
+5,Navigate backwards into a banner landmark,INTERACTION,JAWS,SHIFT_TAB,,,,
+5,Navigate backwards into a banner landmark,INTERACTION,NVDA,SHIFT_TAB,,,,
+6,Navigate backwards into a banner landmark,INTERACTION,VOICEOVER_MACOS,"CTRL_OPT_LEFT,CTRL_OPT_LEFT",SHIFT_CTRL_OPT_CMD_L,,,
+7,Navigate forwards out of a banner landmark,READING,JAWS,DOWN,U,,,
+7,Navigate forwards out of a banner landmark,READING,NVDA,DOWN,U,K,,
+8,Navigate forwards out of a banner landmark,INTERACTION,JAWS,TAB,,,,
+8,Navigate forwards out of a banner landmark,INTERACTION,NVDA,TAB,,,,
+9,Navigate forwards out of a banner landmark,INTERACTION,VOICEOVER_MACOS,"CTRL_OPT_RIGHT,CTRL_OPT_RIGHT",CTRL_OPT_CMD_L,,,
+10,Navigate backwards out of a banner landmark,READING,JAWS,UP,SHIFT_U,,,
+10,Navigate backwards out of a banner landmark,READING,NVDA,UP,SHIFT_K,SHIFT_U,,
+11,Navigate backwards out of a banner landmark,INTERACTION,JAWS,SHIFT_TAB,,,,
+11,Navigate backwards out of a banner landmark,INTERACTION,NVDA,SHIFT_TAB,,,,
+12,Navigate backwards out of a banner landmark,INTERACTION,VOICEOVER_MACOS,"CTRL_OPT_LEFT,CTRL_OPT_LEFT",SHIFT_CTRL_OPT_CMD_L,,,
+13,Navigate forwards to a button inside a banner landmark,READING,JAWS,B,DOWN,TAB,,
+13,Navigate forwards to a button inside a banner landmark,READING,NVDA,B,DOWN,TAB,,
+14,Navigate forwards to a button inside a banner landmark,INTERACTION,JAWS,TAB,,,,
+14,Navigate forwards to a button inside a banner landmark,INTERACTION,NVDA,TAB,,,,
+15,Navigate forwards to a button inside a banner landmark,INTERACTION,VOICEOVER_MACOS,"CTRL_OPT_RIGHT,CTRL_OPT_RIGHT",CTRL_OPT_CMD_J,TAB,,
+16,Navigate backwards to a button inside a banner landmark,READING,JAWS,UP,SHIFT_B,SHIFT_TAB,,
+16,Navigate backwards to a button inside a banner landmark,READING,NVDA,UP,SHIFT_B,SHIFT_TAB,,
+17,Navigate backwards to a button inside a banner landmark,INTERACTION,JAWS,SHIFT_TAB,,,,
+17,Navigate backwards to a button inside a banner landmark,INTERACTION,NVDA,SHIFT_TAB,,,,
+18,Navigate backwards to a button inside a banner landmark,INTERACTION,VOICEOVER_MACOS,"CTRL_OPT_LEFT,CTRL_OPT_LEFT",SHIFT_CTRL_OPT_CMD_J,SHIFT_TAB,,
+19,Navigate forwards to an image inside a banner landmark,READING,JAWS,G,,,,
+19,Navigate forwards to an image inside a banner landmark,READING,NVDA,G,,,,
+20,Navigate forwards to an image inside a banner landmark,INTERACTION,VOICEOVER_MACOS,CTRL_OPT_CMD_G,,,,
+21,Navigate backwards to an image inside a banner landmark,READING,JAWS,SHIFT_G,,,,
+21,Navigate backwards to an image inside a banner landmark,READING,NVDA,SHIFT_G,,,,
+22,Navigate backwards to an image inside a banner landmark,INTERACTION,VOICEOVER_MACOS,SHIFT_CTRL_OPT_CMD_G,,,,
+23,Navigate forwards to a heading inside a banner landmark,READING,JAWS,H,ONE,,,
+23,Navigate forwards to a heading inside a banner landmark,READING,NVDA,H,ONE,,,
+24,Navigate forwards to a heading inside a banner landmark,INTERACTION,VOICEOVER_MACOS,CTRL_OPT_CMD_H,,,,
+25,Navigate backwards to a heading inside a banner landmark,READING,JAWS,SHIFT_H,SHIFT_ONE,,,
+25,Navigate backwards to a heading inside a banner landmark,READING,NVDA,SHIFT_H,SHIFT_ONE,,,
+26,Navigate backwards to a heading inside a banner landmark,INTERACTION,VOICEOVER_MACOS,SHIFT_CTRL_OPT_CMD_H,,,,

__test__/__mocks__/tests/banner/data/js/setFocusAfterBanner.js

@@ -0,0 +1,2 @@
+// sets focus on a link after the banner landmark
+testPageDocument.getElementById('afterlink').focus();

__test__/__mocks__/tests/banner/data/js/setFocusAfterBannerAndHideBottomLink.js

@@ -0,0 +1,3 @@
+// sets focus on a link after the banner landmark, and hides the last link inside the landmark region
+testPageDocument.getElementById('bottom').setAttribute('hidden', '');
+testPageDocument.getElementById('afterlink').focus();

__test__/__mocks__/tests/banner/data/js/setFocusBeforeBanner.js

@@ -0,0 +1,2 @@
+// sets focus on a link before the banner landmark
+testPageDocument.getElementById('beforelink').focus();

__test__/__mocks__/tests/banner/data/js/setFocusBeforeBannerAndHideTopLink.js

@@ -0,0 +1,3 @@
+// sets focus on a link before the banner landmark, and hides the first link inside the landmark region
+testPageDocument.getElementById('top').setAttribute('hidden', '');
+testPageDocument.getElementById('beforelink').focus();

__test__/__mocks__/tests/banner/data/js/setFocusOnBottomLink.js

@@ -0,0 +1,2 @@
+// sets focus on the 'Bottom' link
+testPageDocument.getElementById('bottom').focus();

__test__/__mocks__/tests/banner/data/js/setFocusOnTopLink.js

@@ -0,0 +1,2 @@
+// sets focus on the 'Top' link
+testPageDocument.getElementById('top').focus();

__test__/__mocks__/tests/banner/data/references.csv

@@ -0,0 +1,11 @@
+refId,value
+author,Isabel Del Castillo
+title,Banner Landmark
+authorEmail,[email protected]
+reference,reference/2021-10-24_135455/banner.html
+designPattern,https://w3c.github.io/aria-practices/#aria_lh_banner
+example,https://w3c.github.io/aria-practices/examples/landmarks/banner.html
+banner,https://w3c.github.io/aria/#banner
+aria-haspopup,https://w3c.github.io/aria/#aria-haspopup
+aria-describedby,https://w3c.github.io/aria/#aria-describedby
+aria-expanded,https://w3c.github.io/aria/#aria-expanded