[#35][#15] Fix various new lines and improper header-3 (#36)

.changeset/modern-countries-rescue.md

@@ -0,0 +1,5 @@
+---
+"osrs-web-scraper": minor
+---
+
+Add transformer for converting single line bold text to header-3

.changeset/shy-boats-try.md

@@ -0,0 +1,5 @@
+---
+"osrs-web-scraper": minor
+---
+
+Add transformer for combining consecutive MediaWikiBreak's

src/scrapers/news/news.ts

@@ -2,7 +2,9 @@ import fs from "fs";
 
 import { newsContent, newsHeader } from "./sections";
 import {
+  NewsBreakTransformer,
   NewsFooterTransformer,
+  NewsHeaderTransformer,
   NewsImageCaptionTransformer,
 } from "./transformers";
 import { formatFileName } from "../../utils/images";
@@ -34,7 +36,9 @@ const news: ScrapingService<MediaWikiBuilder> = {
         .addContents(
           await newsContent.format(results.content, page.url(), results.title)
         )
+        .addTransformer(new NewsBreakTransformer())
         .addTransformer(new NewsImageCaptionTransformer())
+        .addTransformer(new NewsHeaderTransformer())
         .addTransformer(new NewsFooterTransformer());
 
       console.info("Writing newspost results to file...");

src/scrapers/news/transformers/__tests__/__snapshots__/headerTransformer.test.ts.snap

@@ -0,0 +1,14 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`NewsHeaderTransformer should change bold text to header-3 1`] = `
+"__TOC__
+
+===You can also discuss this update on our===
+''The Old School Team.''"
+`;
+
+exports[`NewsHeaderTransformer should not change bold text to header-3 1`] = `
+"__TOC__
+'''test'''
+''The Old School Team.''"
+`;

src/scrapers/news/transformers/__tests__/__snapshots__/imageCaptionTransformer.test.ts.snap

@@ -1,12 +1,16 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`NewsImageCaptionTransformer should combine the adjacent MediaWikiFile, MediaWikiBreak and MediaWikiImage 1`] = `"[[File:image|''caption'']]"`;
+exports[`NewsImageCaptionTransformer should combine the adjacent MediaWikiFile, MediaWikiBreak and MediaWikiImage 1`] = `
+"[[File:image|''caption'']]
+"
+`;
 
 exports[`NewsImageCaptionTransformer should combine the adjacent MediaWikiFile, MediaWikiBreak and MediaWikiImage with surrounding content 1`] = `
 "__TOC__
 
 [[test|test]]
 [[File:image|''caption'']]
+
 You can also discuss this update on our
 "
 `;

src/scrapers/news/transformers/__tests__/breakTransformer.ts

@@ -0,0 +1,24 @@
+import {
+  MediaWikiBreak,
+  MediaWikiBuilder,
+  MediaWikiContent,
+  MediaWikiText,
+  MediaWikiTOC,
+} from "../../../../utils/mediawiki";
+import NewsBreakTransformer from "../breakTransformer";
+
+describe("NewsBreakTransformer", () => {
+  it("should combine three lines breaks to one", () => {
+    const originalContent: MediaWikiContent[] = [
+      new MediaWikiTOC(),
+      new MediaWikiBreak(),
+      new MediaWikiBreak(),
+      new MediaWikiBreak(),
+      new MediaWikiText("The Old School Team.", { italics: true }),
+    ];
+    const transformed = new NewsBreakTransformer().transform(originalContent);
+    expect(
+      new MediaWikiBuilder().addContents(transformed).build()
+    ).toMatchSnapshot();
+  });
+});

src/scrapers/news/transformers/__tests__/headerTransformer.test.ts

@@ -0,0 +1,39 @@
+import {
+  MediaWikiBreak,
+  MediaWikiBuilder,
+  MediaWikiContent,
+  MediaWikiText,
+  MediaWikiTOC,
+} from "../../../../utils/mediawiki";
+import NewsHeaderTransformer from "../headerTransformer";
+
+describe("NewsHeaderTransformer", () => {
+  it("should change bold text to header-3", () => {
+    const originalContent: MediaWikiContent[] = [
+      new MediaWikiTOC(),
+      new MediaWikiBreak(),
+      new MediaWikiText("You can also discuss this update on our", {
+        bold: true,
+      }),
+      new MediaWikiBreak(),
+      new MediaWikiText("The Old School Team.", { italics: true }),
+    ];
+    const transformed = new NewsHeaderTransformer().transform(originalContent);
+    expect(
+      new MediaWikiBuilder().addContents(transformed).build()
+    ).toMatchSnapshot();
+  });
+
+  it("should not change bold text to header-3", () => {
+    const originalContent: MediaWikiContent[] = [
+      new MediaWikiTOC(),
+      new MediaWikiText("test", { bold: true }),
+      new MediaWikiBreak(),
+      new MediaWikiText("The Old School Team.", { italics: true }),
+    ];
+    const transformed = new NewsHeaderTransformer().transform(originalContent);
+    expect(
+      new MediaWikiBuilder().addContents(transformed).build()
+    ).toMatchSnapshot();
+  });
+});

src/scrapers/news/transformers/breakTransformer.ts

@@ -0,0 +1,35 @@
+import {
+  MediaWikiBreak,
+  MediaWikiContent,
+  MediaWikiTransformer,
+} from "../../../utils/mediawiki";
+
+class NewsBreakTransformer extends MediaWikiTransformer {
+  transform(content: MediaWikiContent[]): MediaWikiContent[] {
+    const transformedContent = [];
+    for (let index = 0; index < content.length; index++) {
+      const current = content[index];
+      if (
+        index > 0 &&
+        index < content.length - 1 &&
+        current instanceof MediaWikiBreak
+      ) {
+        const before = content[index - 1];
+        const after = content[index + 1];
+        if (
+          before instanceof MediaWikiBreak &&
+          after instanceof MediaWikiBreak
+        ) {
+          index++;
+        } else {
+          transformedContent.push(current);
+        }
+      } else {
+        transformedContent.push(current);
+      }
+    }
+    return transformedContent;
+  }
+}
+
+export default NewsBreakTransformer;

src/scrapers/news/transformers/headerTransformer.ts

@@ -0,0 +1,39 @@
+import {
+  MediaWikiBreak,
+  MediaWikiContent,
+  MediaWikiHeader,
+  MediaWikiText,
+  MediaWikiTransformer,
+} from "../../../utils/mediawiki";
+
+class NewsHeaderTransformer extends MediaWikiTransformer {
+  transform(content: MediaWikiContent[]): MediaWikiContent[] {
+    const transformedContent = [];
+    for (let index = 0; index < content.length; index++) {
+      const current = content[index];
+      if (
+        index > 0 &&
+        index < content.length - 1 &&
+        current instanceof MediaWikiText &&
+        current.value.length <= 70 &&
+        current.styling?.bold
+      ) {
+        const before = content[index - 1];
+        const after = content[index + 1];
+        if (
+          before instanceof MediaWikiBreak &&
+          after instanceof MediaWikiBreak
+        ) {
+          transformedContent.push(new MediaWikiHeader(current.value, 3));
+        } else {
+          transformedContent.push(current);
+        }
+      } else {
+        transformedContent.push(current);
+      }
+    }
+    return transformedContent;
+  }
+}
+
+export default NewsHeaderTransformer;

src/scrapers/news/transformers/imageCaptionTransformer.ts

@@ -28,6 +28,7 @@ class NewsImageCaptionTransformer extends MediaWikiTransformer {
               caption: second,
             })
           );
+          transformedContent.push(new MediaWikiBreak());
           index += 2;
         } else {
           transformedContent.push(current);

src/scrapers/news/transformers/index.ts

@@ -1,2 +1,4 @@
+export { default as NewsBreakTransformer } from "./breakTransformer";
 export { default as NewsFooterTransformer } from "./footerTransformer";
+export { default as NewsHeaderTransformer } from "./headerTransformer";
 export { default as NewsImageCaptionTransformer } from "./imageCaptionTransformer";

src/utils/mediawiki/builder.ts

@@ -23,14 +23,16 @@ class MediaWikiBuilder {
   }
 
   addContent(content: MediaWikiContent): MediaWikiBuilder {
-    if (content !== null) {
+    if (content !== null && content !== undefined) {
       this.content.push(content);
     }
     return this;
   }
 
   addContents(contents: MediaWikiContent[]): MediaWikiBuilder {
-    this.content = this.content.concat(contents);
+    this.content = this.content.concat(
+      contents.filter((content) => content !== undefined && content !== null)
+    );
     return this;
   }