Skip to content

Commit

Permalink
[#82] Gracefully handle news post category fetching failures
Browse files Browse the repository at this point in the history
  • Loading branch information
allenkinzalow committed Mar 2, 2024
1 parent 48fe7b9 commit a86815b
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 6 deletions.
5 changes: 5 additions & 0 deletions .changeset/famous-windows-grin.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"osrs-web-scraper": patch
---

Gracefully handle news post category fetching failures
9 changes: 3 additions & 6 deletions src/scrapers/news/sections/newsHeader/newsHeader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import Parser from "rss-parser";

import {
NEWS_RSS_LINK,

Check warning on line 14 in src/scrapers/news/sections/newsHeader/newsHeader.ts

View workflow job for this annotation

GitHub Actions / lint_build_test

'NEWS_RSS_LINK' is defined but never used
getLatestRSSCateogry,
getNewsCategory,
getNewsUrlIdentifier,

Check warning on line 17 in src/scrapers/news/sections/newsHeader/newsHeader.ts

View workflow job for this annotation

GitHub Actions / lint_build_test

'getNewsUrlIdentifier' is defined but never used
} from "./newsHeader.utils";
Expand All @@ -24,12 +25,8 @@ import { NewsSection } from "../types";

const newsHeader: NewsSection = {
format: async (html, url, title) => {
const rss = await new Parser().parseURL(NEWS_RSS_LINK);
const urlIdentifier = getNewsUrlIdentifier(url);
const item = rss.items.find(
(item) => getNewsUrlIdentifier(item.link) === urlIdentifier
);
const category = getNewsCategory(item?.categories?.[0] ?? "");
const rssCategory = await getLatestRSSCateogry(url);
const category = getNewsCategory(rssCategory ?? "");

const headerRoot = parse(html);

Expand Down
15 changes: 15 additions & 0 deletions src/scrapers/news/sections/newsHeader/newsHeader.utils.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import Parser from "rss-parser";

export const NEWS_RSS_LINK =
"https://secure.runescape.com/m=news/a=1/latest_news.rss?oldschool=true";

Expand Down Expand Up @@ -51,6 +53,19 @@ const updateCategories: { [key: string]: NewsCategory } = {
"Your Feedback": "yourfeedback",
};

export const getLatestRSSCateogry = async (url: string) => {
try {
const rss = await new Parser().parseURL(NEWS_RSS_LINK);
const urlIdentifier = getNewsUrlIdentifier(url);
const item = rss?.items?.find(
(item) => getNewsUrlIdentifier(item?.link) === urlIdentifier
);
return item?.categories?.[0];
} catch (error) {
return undefined;
}
};

export const getNewsCategory = (rawCategory: string) =>
updateCategories[rawCategory] ?? "game";

Expand Down

0 comments on commit a86815b

Please sign in to comment.