From f13322eb8dbb5349ae385cf7f5b98cab84a8b028 Mon Sep 17 00:00:00 2001 From: Allen Kinzalow Date: Fri, 19 Jul 2024 23:01:07 -0400 Subject: [PATCH 1/2] Sort output of world scraping --- .changeset/fluffy-dolls-thank.md | 5 +++ src/scrapers/worlds/worlds.types.ts | 6 +++ src/scrapers/worlds/worlds.utils.ts | 67 ++++++++++++++++++++--------- 3 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 .changeset/fluffy-dolls-thank.md create mode 100644 src/scrapers/worlds/worlds.types.ts diff --git a/.changeset/fluffy-dolls-thank.md b/.changeset/fluffy-dolls-thank.md new file mode 100644 index 0000000..3f89c20 --- /dev/null +++ b/.changeset/fluffy-dolls-thank.md @@ -0,0 +1,5 @@ +--- +"osrs-web-scraper": patch +--- + +Sort output of world scraping diff --git a/src/scrapers/worlds/worlds.types.ts b/src/scrapers/worlds/worlds.types.ts new file mode 100644 index 0000000..76a8a49 --- /dev/null +++ b/src/scrapers/worlds/worlds.types.ts @@ -0,0 +1,6 @@ +export type World = { + activity: string; + number: number; + region: string; + type: "deadman" | "yes" | "no"; +}; diff --git a/src/scrapers/worlds/worlds.utils.ts b/src/scrapers/worlds/worlds.utils.ts index b1235ee..d18e858 100644 --- a/src/scrapers/worlds/worlds.utils.ts +++ b/src/scrapers/worlds/worlds.utils.ts @@ -1,31 +1,58 @@ import { MediaWikiTemplate } from "@osrs-wiki/mediawiki-builder"; import { HTMLElement } from "node-html-parser"; +import { World } from "./worlds.types"; + export const WORLD_LIST_URL = "https://oldschool.runescape.com/a=13/slu"; +/** + * Convert html td rows to MediaWikiTemplate's + * @param worldRows An array of world td rows + * @returns An array of MediaWikiTemplate's + */ export const getWorldLines = (worldRows: HTMLElement) => { const worldRowNodes = worldRows.childNodes.filter( (node) => node instanceof HTMLElement && node.tagName === "TR" ); - const worldLines = worldRowNodes.map((node) => { - const tdNodes = node.childNodes.filter( - (node) => node instanceof HTMLElement && node.tagName === "TD" - ); - const worldLine = new MediaWikiTemplate("WorldLine", { collapsed: true }); - const worldNumber = - tdNodes[0].childNodes?.[1].textContent?.replaceAll(/^\D+/g, "") ?? ""; - const region = tdNodes[2].textContent; - const activity = tdNodes[4].textContent; - const members = activity.includes("Deadman") - ? "deadman" - : tdNodes[3].textContent === "Members" - ? "yes" - : "no"; - worldLine.add("", worldNumber); - worldLine.add("", region); - worldLine.add("mems", members); - worldLine.add("", activity); - return worldLine; - }); + const worldLines = worldRowNodes + .map((node) => { + const tdNodes = node.childNodes.filter( + (node) => node instanceof HTMLElement && node.tagName === "TD" + ); + const worldNumber = + parseInt( + tdNodes[0].childNodes?.[1].textContent?.replaceAll(/^\D+/g, "") ?? "1" + ) + 300; + const region = tdNodes[2].textContent; + const activity = tdNodes[4].textContent; + const members = activity.includes("Deadman") + ? "deadman" + : tdNodes[3].textContent === "Members" + ? "yes" + : "no"; + + return { + activity, + number: worldNumber, + region, + type: members, + }; + }) + .sort((a, b) => a.activity.localeCompare(b.activity)) + .map(getWorldTemplate); return worldLines; }; + +/** + * Convert a World to a MediaWikiTemplate + * @param world The World + * @returns MediaWikiTemplate + */ +export const getWorldTemplate = (world: World) => { + const worldLine = new MediaWikiTemplate("WorldLine", { collapsed: true }); + worldLine.add("", world.number.toString()); + worldLine.add("", world.region); + worldLine.add("mems", world.type); + worldLine.add("", world.activity); + return worldLine; +}; From ff896196085472a220476d951d6f6a8afee52053 Mon Sep 17 00:00:00 2001 From: Allen Kinzalow Date: Fri, 19 Jul 2024 23:02:31 -0400 Subject: [PATCH 2/2] Fix tests --- .../worlds/__tests__/__snapshots__/worlds.utils.test.ts.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scrapers/worlds/__tests__/__snapshots__/worlds.utils.test.ts.snap b/src/scrapers/worlds/__tests__/__snapshots__/worlds.utils.test.ts.snap index 739a47b..5a8fb6c 100644 --- a/src/scrapers/worlds/__tests__/__snapshots__/worlds.utils.test.ts.snap +++ b/src/scrapers/worlds/__tests__/__snapshots__/worlds.utils.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`world scraper utils getWorldLines 1`] = ` -"{{WorldLine|101|United States|mems=deadman|111-126 Deadman}} +"{{WorldLine|401|United States|mems=deadman|111-126 Deadman}} " `;