From 2933425453883441e531f97aecb566692b4974e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Z=C3=BChlcke?= Date: Mon, 24 Oct 2022 18:09:57 +0200 Subject: [PATCH] Update validation and formatting scripts (#10) * Update validation and formatting scripts * Update npm script names * Add a test script which runs all the tests * Apply prettier to all files in the repository * Improve logging in validateRules. * Log all json validation errors rather than just the first one. This also cleans up the iterator in the duplicate check loop. * Remove unused fields 'last_updated' and 'schema'. This prepares the list for stricter JSON validation disallowing additional unknown properties. --- .github/workflows/test.yml | 9 +- README.md | 2 + cookie-banner-rules-list.json | 208 +++++++++------------------------- package.json | 6 +- test/validateRules.js | 21 ++-- 5 files changed, 76 insertions(+), 170 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4db79e..2bc4c73 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,13 +2,12 @@ name: Node.js CI on: push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] jobs: test: - runs-on: ubuntu-latest strategy: @@ -22,5 +21,5 @@ jobs: with: node-version: ${{ matrix.node-version }} - run: npm ci - - run: npm run validate - - run: npm run prettier + - run: npm run test-formatting + - run: npm run test-validate-rules diff --git a/README.md b/README.md index 9046b58..0b6f1ef 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,13 @@ See [CookieBannerRuleList.schema.json](./CookieBannerRuleList.schema.json) and [ Before submitting run the following commands to ensure the rule list is well formatted and valid: Install dependencies: + ``` npm install ``` Run tests: + ``` npm test ``` diff --git a/cookie-banner-rules-list.json b/cookie-banner-rules-list.json index 92af248..6f09294 100644 --- a/cookie-banner-rules-list.json +++ b/cookie-banner-rules-list.json @@ -7,10 +7,8 @@ "presence": "div#gdpr-new-container" }, "domain": "aliexpress.com", - "schema": 1661958902092, "cookies": {}, - "id": "80851a39-2183-49e4-99f4-16d6189bff1e", - "last_modified": 1661959171141 + "id": "80851a39-2183-49e4-99f4-16d6189bff1e" }, { "click": { @@ -18,10 +16,8 @@ "presence": "div#onetrust-consent-sdk" }, "domain": "soundcloud.com", - "schema": 1661957607051, "cookies": {}, - "id": "6c7366a0-4762-47b9-8eeb-04e86cc7a0cc", - "last_modified": 1661958901702 + "id": "6c7366a0-4762-47b9-8eeb-04e86cc7a0cc" }, { "click": { @@ -29,10 +25,8 @@ "presence": "div.a8c-cookie-banner" }, "domain": "gravatar.com", - "schema": 1661956519437, "cookies": {}, - "id": "4b835605-77e0-457a-8409-8d499ee1131c", - "last_modified": 1661957606664 + "id": "4b835605-77e0-457a-8409-8d499ee1131c" }, { "click": { @@ -40,20 +34,16 @@ "presence": "div#onetrust-banner-sdk" }, "domain": "fastly.com", - "schema": 1661955856767, "cookies": {}, - "id": "c63efcc1-06e0-42c5-955f-f5a61fd0baa2", - "last_modified": 1661956519048 + "id": "c63efcc1-06e0-42c5-955f-f5a61fd0baa2" }, { "click": { "presence": "div#uhfCookieAlert" }, "domain": "skype.com", - "schema": 1661956089353, "cookies": {}, - "id": "0c43a914-8bc7-4872-89f2-f1ad213d2a7f", - "last_modified": 1661956519043 + "id": "0c43a914-8bc7-4872-89f2-f1ad213d2a7f" }, { "click": { @@ -62,10 +52,8 @@ "presence": "div.truste_box_overlay_inner" }, "domain": "flickr.com", - "schema": 1661954382225, "cookies": {}, - "id": "22d33421-6872-41bc-9316-adba6a9af18e", - "last_modified": 1661955856388 + "id": "22d33421-6872-41bc-9316-adba6a9af18e" }, { "click": { @@ -74,10 +62,8 @@ "presence": "div#cookie-consent-banner" }, "domain": "europa.eu", - "schema": 1661954200065, "cookies": {}, - "id": "0c74749a-8c53-4bb0-b31a-ab2f89b7f493", - "last_modified": 1661954381849 + "id": "0c74749a-8c53-4bb0-b31a-ab2f89b7f493" }, { "click": { @@ -85,10 +71,8 @@ "presence": "div#onetrust-banner-sdk" }, "domain": "spotify.com", - "schema": 1661953620268, "cookies": {}, - "id": "443c2cd7-1d4f-486f-a263-02d57aa70cb3", - "last_modified": 1661953825139 + "id": "443c2cd7-1d4f-486f-a263-02d57aa70cb3" }, { "click": { @@ -96,10 +80,8 @@ "presence": "div#cookieBar" }, "domain": "google-analytics.com", - "schema": 1661952366874, "cookies": {}, - "id": "286028cd-7cb7-4c71-91a7-ae1a23df8a31", - "last_modified": 1661953619867 + "id": "286028cd-7cb7-4c71-91a7-ae1a23df8a31" }, { "click": { @@ -108,10 +90,8 @@ "presence": "div.evidon-banner" }, "domain": "dropbox.com", - "schema": 1661951534770, "cookies": {}, - "id": "7f82a27a-eaf8-4c16-b15e-66b5efe62fa2", - "last_modified": 1661952366489 + "id": "7f82a27a-eaf8-4c16-b15e-66b5efe62fa2" }, { "click": { @@ -120,10 +100,8 @@ "presence": "div#cmpbox" }, "domain": "mail.ru", - "schema": 1661869950587, "cookies": {}, - "id": "77424a06-c47f-4b06-9925-e4d5afc7c61a", - "last_modified": 1661873355741 + "id": "77424a06-c47f-4b06-9925-e4d5afc7c61a" }, { "click": { @@ -132,10 +110,8 @@ "presence": "div#cookie-banner" }, "domain": "azure.microsoft.com", - "schema": 1661781288397, "cookies": {}, - "id": "f899d35e-20af-4cfb-9856-143a80b86ba9", - "last_modified": 1661782001575 + "id": "f899d35e-20af-4cfb-9856-143a80b86ba9" }, { "click": { @@ -143,10 +119,8 @@ "presence": ".js-consent-banner" }, "domain": "askubuntu.com", - "schema": 1661539693723, "cookies": {}, - "id": "71443ce9-15b8-4e07-b51b-1f2158521ea9", - "last_modified": 1661539730639 + "id": "71443ce9-15b8-4e07-b51b-1f2158521ea9" }, { "click": { @@ -154,10 +128,8 @@ "presence": ".js-consent-banner" }, "domain": "serverfault.com", - "schema": 1661539634697, "cookies": {}, - "id": "98fcaf1f-aae9-4fb4-985a-e4f7230c3fdb", - "last_modified": 1661539730634 + "id": "98fcaf1f-aae9-4fb4-985a-e4f7230c3fdb" }, { "click": { @@ -165,10 +137,8 @@ "presence": ".js-consent-banner" }, "domain": "stackexchange.com", - "schema": 1661539497584, "cookies": {}, - "id": "6dca720b-4df0-4707-86c8-2a98748cba02", - "last_modified": 1661539730629 + "id": "6dca720b-4df0-4707-86c8-2a98748cba02" }, { "click": { @@ -176,10 +146,8 @@ "presence": ".js-consent-banner" }, "domain": "stackoverflow.com", - "schema": 1661539322069, "cookies": {}, - "id": "05e86dab-14e2-4fbe-9229-539752d57f8f", - "last_modified": 1661539497287 + "id": "05e86dab-14e2-4fbe-9229-539752d57f8f" }, { "click": { @@ -189,10 +157,8 @@ "presence": "div#onetrust-banner-sdk" }, "domain": "cnn.com", - "schema": 1661539313645, "cookies": {}, - "id": "3dde4bbf-a1fa-458d-afb1-e2ae5f459217", - "last_modified": 1661539497276 + "id": "3dde4bbf-a1fa-458d-afb1-e2ae5f459217" }, { "click": { @@ -200,15 +166,12 @@ "presence": "body" }, "domain": "privacy-mgmt.com", - "schema": 1661372690527, "cookies": {}, - "id": "d3d9bf51-2613-408a-afad-5d203fea7857", - "last_modified": 1661372852775 + "id": "d3d9bf51-2613-408a-afad-5d203fea7857" }, { "click": {}, "domain": "youtube.com", - "schema": 1661372391701, "cookies": { "optOut": [ { @@ -225,8 +188,7 @@ } ] }, - "id": "788ed05e-cf93-41a8-b2e5-c5571d855232", - "last_modified": 1661372428150 + "id": "788ed05e-cf93-41a8-b2e5-c5571d855232" }, { "click": { @@ -236,15 +198,12 @@ "presence": "#onetrust-pc-sdk" }, "domain": "dhl.com", - "schema": 1661367207642, "cookies": {}, - "id": "a224a333-229d-4f1f-af68-41a25a152037", - "last_modified": 1661368829788 + "id": "a224a333-229d-4f1f-af68-41a25a152037" }, { "click": {}, "domain": "google.com", - "schema": 1661360967749, "cookies": { "optOut": [ { @@ -261,13 +220,11 @@ } ] }, - "id": "ffe76ba2-d6aa-47f4-ab85-9368feeb4519", - "last_modified": 1661361726513 + "id": "ffe76ba2-d6aa-47f4-ab85-9368feeb4519" }, { "click": {}, "domain": "google.fr", - "schema": 1661360388718, "cookies": { "optOut": [ { @@ -284,13 +241,11 @@ } ] }, - "id": "114dde58-155f-45f6-9da6-b545f8f3f613", - "last_modified": 1661360967440 + "id": "114dde58-155f-45f6-9da6-b545f8f3f613" }, { "click": {}, "domain": "google.de", - "schema": 1661360299863, "cookies": { "optOut": [ { @@ -307,8 +262,7 @@ } ] }, - "id": "01c11d97-981e-4543-ae60-e468f1adbe98", - "last_modified": 1661360388419 + "id": "01c11d97-981e-4543-ae60-e468f1adbe98" }, { "click": { @@ -317,10 +271,8 @@ "presence": "div.uiLayer" }, "domain": "facebook.com", - "schema": 1661165103413, "cookies": {}, - "id": "d1d8ba36-ced7-4453-8b17-2e051e0ab1eb", - "last_modified": 1661165447807 + "id": "d1d8ba36-ced7-4453-8b17-2e051e0ab1eb" }, { "click": { @@ -329,10 +281,8 @@ "presence": "div#uhfCookieAlert" }, "domain": "microsoft.com", - "schema": 1661165137971, "cookies": {}, - "id": "caf8d44d-af29-47a9-aa66-ec4041655d33", - "last_modified": 1661165447803 + "id": "caf8d44d-af29-47a9-aa66-ec4041655d33" }, { "click": { @@ -341,10 +291,8 @@ "presence": "div#layers > div > div > div > div > div" }, "domain": "twitter.com", - "schema": 1661165167737, "cookies": {}, - "id": "05b3b417-c4c7-4ed0-a3cf-43053e8b33ab", - "last_modified": 1661165447799 + "id": "05b3b417-c4c7-4ed0-a3cf-43053e8b33ab" }, { "click": { @@ -353,10 +301,8 @@ "presence": "div.global-alert-banner" }, "domain": "linkedin.com", - "schema": 1661165204884, "cookies": {}, - "id": "f1ddc6f6-8927-4205-8003-1f6e84ae22b0", - "last_modified": 1661165447795 + "id": "f1ddc6f6-8927-4205-8003-1f6e84ae22b0" }, { "click": { @@ -366,10 +312,8 @@ "presence": "div#onetrust-banner-sdk" }, "domain": "vimeo.com", - "schema": 1661165282200, "cookies": {}, - "id": "dd0c1047-7dfa-4ad7-b249-3559e6a466f9", - "last_modified": 1661165447792 + "id": "dd0c1047-7dfa-4ad7-b249-3559e6a466f9" }, { "click": { @@ -378,10 +322,8 @@ "presence": "div#gdpr-banner" }, "domain": "ebay.com", - "schema": 1661165373411, "cookies": {}, - "id": "1e6d35e7-b907-4f5c-a09a-9f3336ef6e61", - "last_modified": 1661165447784 + "id": "1e6d35e7-b907-4f5c-a09a-9f3336ef6e61" }, { "click": { @@ -390,10 +332,8 @@ "presence": "div#gdpr-banner" }, "domain": "ebay.de", - "schema": 1661165397311, "cookies": {}, - "id": "d0356a58-6b55-4bbc-ab79-9ded68601af2", - "last_modified": 1661165447780 + "id": "d0356a58-6b55-4bbc-ab79-9ded68601af2" }, { "click": { @@ -401,10 +341,8 @@ "presence": "div.consent-banner" }, "domain": "twitch.tv", - "schema": 1661165425864, "cookies": {}, - "id": "542aafba-9f44-4321-88e5-ece333073360", - "last_modified": 1661165447776 + "id": "542aafba-9f44-4321-88e5-ece333073360" }, { "click": { @@ -413,7 +351,6 @@ "presence": "div.gdpr" }, "domain": "nytimes.com", - "schema": 1661164977100, "cookies": { "optOut": [ { @@ -422,8 +359,7 @@ } ] }, - "id": "488f07f9-0a01-4118-aab3-2b817e9fe6dc", - "last_modified": 1661165447772 + "id": "488f07f9-0a01-4118-aab3-2b817e9fe6dc" }, { "click": { @@ -433,7 +369,6 @@ "presence": "div#onetrust-banner-sdk" }, "domain": "getpocket.com", - "schema": 1661165237785, "cookies": { "optOut": [ { @@ -442,8 +377,7 @@ } ] }, - "id": "c2f6b05e-73dc-4df4-bb54-68215bb9f176", - "last_modified": 1661165447769 + "id": "c2f6b05e-73dc-4df4-bb54-68215bb9f176" }, { "click": { @@ -451,10 +385,8 @@ "presence": "div#truste-consent-track" }, "domain": "forbes.com", - "schema": 1661164859621, "cookies": {}, - "id": "30293090-f064-473a-ae0f-cd390507c1c7", - "last_modified": 1661164976804 + "id": "30293090-f064-473a-ae0f-cd390507c1c7" }, { "click": { @@ -462,10 +394,8 @@ "presence": "form#sp-cc" }, "domain": "amazon.de", - "schema": 1661164918986, "cookies": {}, - "id": "7957b964-6cc5-4d33-9200-aa35e649a2af", - "last_modified": 1661164976800 + "id": "7957b964-6cc5-4d33-9200-aa35e649a2af" }, { "click": { @@ -474,10 +404,8 @@ "presence": "div#cookie-disclosure" }, "domain": "netflix.com", - "schema": 1661164945628, "cookies": {}, - "id": "6037802d-9a37-4df2-bf35-9ad60c478725", - "last_modified": 1661164976796 + "id": "6037802d-9a37-4df2-bf35-9ad60c478725" }, { "click": { @@ -486,15 +414,12 @@ "presence": "div[role=presentation]" }, "domain": "instagram.com", - "schema": 1660934222335, "cookies": {}, - "id": "e7112653-2ab3-4ddf-b0b1-396052c2f001", - "last_modified": 1661164859316 + "id": "e7112653-2ab3-4ddf-b0b1-396052c2f001" }, { "click": {}, "domain": "bbc.com", - "schema": 1660929784754, "cookies": { "optOut": [ { @@ -511,13 +436,11 @@ } ] }, - "id": "e9a07bcd-28b5-4034-a663-c017e6a8208e", - "last_modified": 1660929852978 + "id": "e9a07bcd-28b5-4034-a663-c017e6a8208e" }, { "click": {}, "domain": "bbc.co.uk", - "schema": 1660918464179, "cookies": { "optOut": [ { @@ -534,13 +457,11 @@ } ] }, - "id": "1be89e75-e68b-4323-928e-3839995f2be3", - "last_modified": 1660929784461 + "id": "1be89e75-e68b-4323-928e-3839995f2be3" }, { "click": {}, "domain": "web.de", - "schema": 1660845073491, "cookies": { "optOut": [ { @@ -553,13 +474,11 @@ } ] }, - "id": "290db348-ced2-4f16-9954-da476d0aef01", - "last_modified": 1660918463869 + "id": "290db348-ced2-4f16-9954-da476d0aef01" }, { "click": {}, "domain": "bundesregierung.de", - "schema": 1660843213528, "cookies": { "optOut": [ { @@ -576,13 +495,11 @@ } ] }, - "id": "84f68b1a-18a2-478c-bc8e-9ec32d4e3e80", - "last_modified": 1660845073182 + "id": "84f68b1a-18a2-478c-bc8e-9ec32d4e3e80" }, { "click": {}, "domain": "bing.com", - "schema": 1660836870336, "cookies": { "optOut": [ { @@ -591,13 +508,11 @@ } ] }, - "id": "31dc6160-3495-4f4e-8c67-594527bd4051", - "last_modified": 1660837365450 + "id": "31dc6160-3495-4f4e-8c67-594527bd4051" }, { "click": {}, "domain": "paypal.com", - "schema": 1660837021903, "cookies": { "optOut": [ { @@ -606,13 +521,11 @@ } ] }, - "id": "3f1e21e7-327f-42de-ac40-25da7f4fa7c9", - "last_modified": 1660837365442 + "id": "3f1e21e7-327f-42de-ac40-25da7f4fa7c9" }, { "click": {}, "domain": "mydealz.de", - "schema": 1660837054566, "cookies": { "optOut": [ { @@ -625,13 +538,11 @@ } ] }, - "id": "069f4d94-8031-4b83-b8f9-89752c5c1353", - "last_modified": 1660837365438 + "id": "069f4d94-8031-4b83-b8f9-89752c5c1353" }, { "click": {}, "domain": "whatsapp.com", - "schema": 1660837148437, "cookies": { "optIn": [ { @@ -641,13 +552,11 @@ ], "optOut": [] }, - "id": "0f4b1486-7d4b-4f73-8ec9-122daca9ebf4", - "last_modified": 1660837365425 + "id": "0f4b1486-7d4b-4f73-8ec9-122daca9ebf4" }, { "click": {}, "domain": "wordpress.com", - "schema": 1660837179455, "cookies": { "optIn": [ { @@ -656,13 +565,11 @@ } ] }, - "id": "709f87b8-b010-4d32-b32d-3f265963c6a5", - "last_modified": 1660837365419 + "id": "709f87b8-b010-4d32-b32d-3f265963c6a5" }, { "click": {}, "domain": "bitly.com", - "schema": 1660837207193, "cookies": { "optIn": [ { @@ -671,13 +578,11 @@ } ] }, - "id": "84a097a3-5cd1-448a-afcf-4be950bc5756", - "last_modified": 1660837365414 + "id": "84a097a3-5cd1-448a-afcf-4be950bc5756" }, { "click": {}, "domain": "yandex.com", - "schema": 1660837231206, "cookies": { "optOut": [ { @@ -686,13 +591,11 @@ } ] }, - "id": "37319f5d-9484-4da8-aee1-570a78688da3", - "last_modified": 1660837365409 + "id": "37319f5d-9484-4da8-aee1-570a78688da3" }, { "click": {}, "domain": "yandex.ru", - "schema": 1660837249273, "cookies": { "optOut": [ { @@ -701,13 +604,11 @@ } ] }, - "id": "128c2e2e-99c8-4f15-9843-dab9942421b0", - "last_modified": 1660837365401 + "id": "128c2e2e-99c8-4f15-9843-dab9942421b0" }, { "click": {}, "domain": "tumblr.com", - "schema": 1660837266873, "cookies": { "optOut": [ { @@ -720,13 +621,11 @@ } ] }, - "id": "e8f941e6-ac56-4dec-b682-a75e8bd52aae", - "last_modified": 1660837365396 + "id": "e8f941e6-ac56-4dec-b682-a75e8bd52aae" }, { "click": {}, "domain": "tiktok.com", - "schema": 1660837306986, "cookies": { "optOut": [ { @@ -735,13 +634,11 @@ } ] }, - "id": "d9166ae8-dcc7-4ca2-8b02-0884fb1d6f70", - "last_modified": 1660837365389 + "id": "d9166ae8-dcc7-4ca2-8b02-0884fb1d6f70" }, { "click": {}, "domain": "reddit.com", - "schema": 1660836803699, "cookies": { "optOut": [ { @@ -750,8 +647,7 @@ } ] }, - "id": "c5243e7c-eb86-4a9d-947c-7129e99fbd72", - "last_modified": 1660836869997 + "id": "c5243e7c-eb86-4a9d-947c-7129e99fbd72" }, { "click": { diff --git a/package.json b/package.json index ef47dce..660252c 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,10 @@ "version": "1.0.0", "description": "Rules List for how Firefox's Automated Cookie Banner Preference Manager is to interact with banners on a site by site basis", "scripts": { - "validate": "node test/validateRules.js", - "prettier": "prettier --check cookie-banner-rules-list.json" + "test": "npm run test-formatting && npm run test-validate-rules", + "test-validate-rules": "node test/validateRules.js", + "test-formatting": "prettier --check .", + "prettier": "prettier --write ." }, "repository": { "type": "git", diff --git a/test/validateRules.js b/test/validateRules.js index 54031f1..2b3796a 100644 --- a/test/validateRules.js +++ b/test/validateRules.js @@ -33,7 +33,12 @@ async function loadSchema(uri) { return response.json(); } -const ajv = new Ajv({ loadSchema }); +function exitWithError(reason) { + console.error(`❌ ${RULE_LIST_FILE} is invalid: ${reason}`); + exit(1); +} + +const ajv = new Ajv({ loadSchema, allErrors: true }); (async () => { // 1. Load and parse the rules list. @@ -46,7 +51,7 @@ const ajv = new Ajv({ loadSchema }); ruleList = JSON.parse(ruleListStr); } catch (error) { console.error("Error while parsing rule list", error); - exit(1); + exitWithError("Invalid JSON"); } // 2. Validate rules list against schema. @@ -56,7 +61,7 @@ const ajv = new Ajv({ loadSchema }); const valid = validate(ruleList); if (!valid) { console.info("Rule list validation error", validate.errors); - exit(1); + exitWithError("Schema validation error"); } // 3. Check for duplicate rules (id or domain) @@ -64,21 +69,23 @@ const ajv = new Ajv({ loadSchema }); const domainSet = new Set(); let foundDuplicates = false; - let i = 0; - ruleList.data.forEach((rule) => { + ruleList.data.forEach((rule, i) => { if (idSet.has(rule.id)) { console.error(`Duplicate id ${rule.id} for rule #${i}`); + foundDuplicates = true; } // Allow * domain rules which are global rules. if (domainSet.has(rule.domain) && rule.domain != "*") { console.error(`Duplicate domain ${rule.domain} for rule #${i}`); + foundDuplicates = true; } idSet.add(rule.id); domainSet.add(rule.domain); - i += 1; }); if (foundDuplicates) { - exit(1); + exitWithError("Found duplicate rules"); } + + console.info(`✅ ${RULE_LIST_FILE} is valid.`); })();