From f4e99b826b3de59d37da0a69dfc7f10054636b61 Mon Sep 17 00:00:00 2001 From: Flameish Date: Sat, 14 Sep 2019 21:53:59 +0200 Subject: [PATCH] 2.1.0 --- src/grabber/Download.java | 17 ++++- src/grabber/HostSettings.java | 54 ++++++++++++++- src/grabber/Shared.java | 1 - src/grabber/autoFetchChapters.java | 56 ++++++++++++++-- src/gui/GUI.form | 104 +++++++++++++++++++---------- src/gui/GUI.java | 96 +++++++++++++++++++++----- 6 files changed, 263 insertions(+), 65 deletions(-) diff --git a/src/grabber/Download.java b/src/grabber/Download.java index 4d5fecd..7df3869 100644 --- a/src/grabber/Download.java +++ b/src/grabber/Download.java @@ -34,6 +34,7 @@ public class Download { boolean getImages; boolean allChapters; boolean invertOrder; + public boolean autoChapterToChapter; long startTime = System.nanoTime(); int firstChapter; int lastChapter; @@ -58,6 +59,9 @@ public Download(GUI myGUI) { currHostSettings = new HostSettings(host, tocUrl); blacklistedTags = currHostSettings.blacklistedTags; // Functions + if (HostSettings.autoChapterToChapterWebsitesList.contains(gui.autoHostSelection.getSelectedItem().toString())) { + autoChapterToChapter = true; + } autoFetchChapters.getChapters(this); autoFetchChapters.getMetadata(this); } @@ -126,8 +130,19 @@ public void startAutoDownload() { } // Getting the chapters again if it was stopped previously if (autoFetchChapters.killTask) autoFetchChapters.getChapters(this); + autoFetchChapters.killTask = false; - autoFetchChapters.downloadChapters(this); + if (autoChapterToChapter) { + String[] chapterInfo = { + gui.autoFirstChapterURL.getText(), + gui.autoLastChapterURL.getText(), + currHostSettings.nextChapterBtn + }; + chapterContainer = currHostSettings.chapterContainer; + autoFetchChapters.processChaptersToChapters(chapterInfo, this); + } else { + autoFetchChapters.downloadChapters(this); + } if (!successfulFilenames.isEmpty() && !autoFetchChapters.killTask) { switch ((String) gui.exportSelection.getSelectedItem()) { case "Calibre": diff --git a/src/grabber/HostSettings.java b/src/grabber/HostSettings.java index e855662..aa76974 100644 --- a/src/grabber/HostSettings.java +++ b/src/grabber/HostSettings.java @@ -1,5 +1,4 @@ package grabber; - import java.util.Arrays; import java.util.List; @@ -12,12 +11,21 @@ public class HostSettings { "Royal Road", "Gravity Tales", "Volare Novels", - "Light Novels Translations", + "Creative Novels", "WordExcerpt", - "BoxNovel", "LiberSpark", + "BoxNovel", + "Wordrain", + "Comrademao", + "Light Novels Translations", "Chrysanthemum Garden" }; + private static String[] autoChapterToChapterWebsites = { + "Comrademao", + "Creative Novels" + }; + public static List autoChapterToChapterWebsitesList = Arrays.asList(autoChapterToChapterWebsites); + public String chapterLinkSelecter; String titleHostName; @@ -26,6 +34,7 @@ public class HostSettings { public List blacklistedTags; int ordinalIndexForBaseNovel; String chapterContainer; + String nextChapterBtn; String bookTitleSelector; String bookAuthorSelector; String bookSubjectSelector; @@ -150,6 +159,45 @@ public HostSettings(String domain, String urla) { bookAuthorSelector = ""; bookSubjectSelector = ".novel-container a[href^=https://chrysanthemumgarden.com/genre/]"; break; + case "comrademao": + host = "https://comrademao.com/"; + ordinalIndexForBaseNovel = 0; + chapterLinkSelecter = ""; + chapterContainer = "article div"; + nextChapterBtn = "a.btn.btn-default.btn-sm:eq(2)"; + titleHostName = ""; + blacklistedTags = Arrays.asList("div#CollapseRaw"); + bookTitleSelector = "div.wrap-content h4"; + bookCoverSelector = "div.wrap-thumbnail img"; + bookAuthorSelector = "div.author"; + bookSubjectSelector = "a[rel=tag]"; + break; + case "creativenovels": + host = "https://creativenovels.com/"; + ordinalIndexForBaseNovel = 0; + chapterLinkSelecter = ""; + chapterContainer = ".entry-content"; + nextChapterBtn = "a.x-btn.nextkey"; + titleHostName = ""; + blacklistedTags = Arrays.asList(".mNS", ".support-placement"); + bookTitleSelector = "div.e45344-16.x-text.bK_C"; + bookCoverSelector = "img.book_cover"; + bookAuthorSelector = "div.e45344-17.x-text.bK_C a"; + bookSubjectSelector = "div.genre_novel"; + break; + case "wordrain": + host = "https://wordrain69.com/"; + ordinalIndexForBaseNovel = 5; + chapterLinkSelecter = "ul.main.version-chap li a:not([title])"; + chapterContainer = ".text-left"; + nextChapterBtn = ""; + titleHostName = "-Wordrain69"; + //blacklistedTags = Arrays.asList(""); + bookTitleSelector = ".post-title h1"; + bookCoverSelector = ".summary_image img"; + bookAuthorSelector = ".author-content a"; + bookSubjectSelector = ".genres-content a"; + break; } } } diff --git a/src/grabber/Shared.java b/src/grabber/Shared.java index 994c00f..5cb199b 100644 --- a/src/grabber/Shared.java +++ b/src/grabber/Shared.java @@ -246,7 +246,6 @@ static void saveChapterWithHTML(String url, int chapterNumber, String chapterNam // Getting the next chapter URL from the "nextChapterBtn" href for Chapter-To-Chapter. if (!currGrab.nextChapterBtn.equals("NOT_SET")) currGrab.nextChapterURL = doc.select(currGrab.nextChapterBtn).first().absUrl("href"); - Element chapterContent = doc.select(chapterContainer).first(); // Remove unwanted tags from chapter container. if (!(currGrab.blacklistedTags == null)) { diff --git a/src/grabber/autoFetchChapters.java b/src/grabber/autoFetchChapters.java index 2af54f2..453f9c0 100644 --- a/src/grabber/autoFetchChapters.java +++ b/src/grabber/autoFetchChapters.java @@ -34,11 +34,13 @@ static void getChapters(Download currGrab) { // Connect to webpage Document doc = Jsoup.connect(currGrab.currHostSettings.url).get(); // Get chapter links and names. - Elements chapterItems = doc.select(currGrab.currHostSettings.chapterLinkSelecter); - Elements links = chapterItems.select("a[href]"); - for (Element chapterLink : links) { - currGrab.chapterLinks.add(chapterLink.attr("abs:href")); - currGrab.chaptersNames.add(chapterLink.text()); + if (!currGrab.autoChapterToChapter) { + Elements chapterItems = doc.select(currGrab.currHostSettings.chapterLinkSelecter); + Elements links = chapterItems.select("a[href]"); + for (Element chapterLink : links) { + currGrab.chapterLinks.add(chapterLink.attr("abs:href")); + currGrab.chaptersNames.add(chapterLink.text()); + } } currGrab.tocDoc = doc; } catch (IllegalArgumentException | IOException e) { @@ -141,11 +143,15 @@ static void getMetadata(Download currGrab) { currGrab.bookSubjects = null; currGrab.gui.autoBookSubjects.setText(""); } - // Chapter + // Chapter number if (!currGrab.chapterLinks.isEmpty()) { currGrab.gui.autoChapterAmount.setText(String.valueOf(currGrab.chapterLinks.size())); currGrab.gui.autoGetNumberButton.setEnabled(true); } + if (currGrab.autoChapterToChapter) { + currGrab.gui.autoChapterAmount.setText("Unknown"); + currGrab.gui.autoGetNumberButton.setEnabled(false); + } // Cover if (!currGrab.currHostSettings.bookCoverSelector.isEmpty()) { if (doc.select(currGrab.currHostSettings.bookCoverSelector) != null && !doc.select(currGrab.currHostSettings.bookCoverSelector).isEmpty()) { @@ -219,6 +225,44 @@ private static void processAllChapters(Download currGrab) { } } + /** + * Handles downloading chapter to chapter. + */ + static void processChaptersToChapters(String[] args, Download currGrab) { + currGrab.gui.appendText(currGrab.window, "[INFO]Connecting..."); + currGrab.gui.setMaxProgress(currGrab.window, 9001); + String nextChapter = args[0]; + String lastChapter = args[1]; + currGrab.nextChapterBtn = args[2]; + int chapterNumber = 0; + while (true) { + chapterNumber++; + Shared.saveChapterWithHTML(nextChapter, chapterNumber, "Chapter " + chapterNumber, currGrab.chapterContainer, currGrab); + nextChapter = currGrab.nextChapterURL; + if (nextChapter.equals(lastChapter) || (nextChapter + "/").equals(lastChapter)) { + chapterNumber++; + Shared.sleep(currGrab.waitTime); + currGrab.nextChapterBtn = "NOT_SET"; + Shared.saveChapterWithHTML(nextChapter, chapterNumber, "Chapter " + chapterNumber, currGrab.chapterContainer, currGrab); + break; + } + if (killTask) { + currGrab.gui.appendText(currGrab.window, "[INFO]Stopped."); + Path chaptersFolder = Paths.get(currGrab.saveLocation + "/chapters"); + Path imagesFolder = Paths.get(currGrab.saveLocation + "/images"); + try { + if (Files.exists(imagesFolder)) Shared.deleteFolderAndItsContent(imagesFolder); + if (Files.exists(chaptersFolder)) Shared.deleteFolderAndItsContent(chaptersFolder); + } catch (IOException e) { + currGrab.gui.appendText(currGrab.window, e.getMessage()); + e.printStackTrace(); + } + return; + } + Shared.sleep(currGrab.waitTime); + } + } + /** * Displays chapter name and chapter number. * TODO: pass doc directly. diff --git a/src/gui/GUI.form b/src/gui/GUI.form index 300c572..a6c9a64 100644 --- a/src/gui/GUI.form +++ b/src/gui/GUI.form @@ -33,7 +33,7 @@ - + @@ -53,7 +53,7 @@ - + @@ -63,7 +63,7 @@ - + @@ -71,48 +71,39 @@ - - - - + - - - - + - - - - + - + - + - + - + - + @@ -130,19 +121,44 @@ - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + - @@ -156,7 +172,7 @@ - + @@ -214,12 +230,12 @@ - + - + @@ -231,9 +247,25 @@ + + + + + + + + + + + + + + + + - + @@ -243,22 +275,20 @@ - + - + - + + - + - + - - - - + diff --git a/src/gui/GUI.java b/src/gui/GUI.java index 27195bf..7f25fe5 100644 --- a/src/gui/GUI.java +++ b/src/gui/GUI.java @@ -32,7 +32,7 @@ import java.util.concurrent.Executors; public class GUI extends JFrame { - public static String versionNumber = "2.0.0"; + public static String versionNumber = "2.1.0"; public static String appdataPath = System.getProperty("user.home") + File.separator + "AppData" + File.separator + "Roaming" + File.separator + "Novel-Grabber"; public static DefaultListModel listModelChapterLinks = new DefaultListModel<>(); public static DefaultListModel listModelCheckerLinks = new DefaultListModel<>(); @@ -118,6 +118,11 @@ public class GUI extends JFrame { private JButton stopButton; private JButton manStopButton; private JButton manJsoupInfoButton; + public JTextField autoLastChapterURL; + public JTextField autoFirstChapterURL; + private JLabel autoLastChapterLbl; + private JLabel autoFirstChapterLbl; + private JLabel NovelUrlLbl; private JButton autoEditMetadataButton; @@ -171,26 +176,49 @@ public GUI() { if (chapterListURL.getText().isEmpty()) { showPopup("URL field is empty.", "warning"); chapterListURL.requestFocusInWindow(); - } else if (saveLocation.getText().isEmpty()) { + return; + } + if (saveLocation.getText().isEmpty()) { showPopup("Save directory field is empty.", "warning"); saveLocation.requestFocusInWindow(); - } else if ((!chapterAllCheckBox.isSelected()) && (!toLastChapter.isSelected()) - && (((Integer) firstChapter.getValue() < 1) - || ((Integer) lastChapter.getValue()) < 1)) { - showPopup("Chapter numbers can't be lower than 1.", "warning"); - } else if ((!chapterAllCheckBox.isSelected()) && (!toLastChapter.isSelected()) - && ((Integer) lastChapter.getValue()) < (Integer) firstChapter.getValue()) { - showPopup("Last chapter can't be lower than first chapter.", "warning"); - } else if ((!chapterAllCheckBox.isSelected()) && (toLastChapter.isSelected()) - && ((Integer) firstChapter.getValue()) < 1) { - showPopup("First chapter number can't be lower than 1.", "warning"); - } else if (waitTime.getText().isEmpty()) { + return; + } + if (!auto.autoChapterToChapter) { + if ((!chapterAllCheckBox.isSelected()) && (!toLastChapter.isSelected()) + && (((Integer) firstChapter.getValue() < 1) + || ((Integer) lastChapter.getValue()) < 1)) { + showPopup("Chapter numbers can't be lower than 1.", "warning"); + return; + } + if ((!chapterAllCheckBox.isSelected()) && (!toLastChapter.isSelected()) + && ((Integer) lastChapter.getValue()) < (Integer) firstChapter.getValue()) { + showPopup("Last chapter can't be lower than first chapter.", "warning"); + return; + } + if ((!chapterAllCheckBox.isSelected()) && (toLastChapter.isSelected()) + && ((Integer) firstChapter.getValue()) < 1) { + showPopup("First chapter number can't be lower than 1.", "warning"); + return; + } + } else { + if (autoFirstChapterURL.getText().isEmpty()) { + showPopup("First chapter URL is empty.", "warning"); + return; + } + if (autoLastChapterURL.getText().isEmpty()) { + showPopup("Last chapter URL is empty.", "warning"); + return; + } + } + if (waitTime.getText().isEmpty()) { showPopup("Wait time cannot be empty.", "warning"); - } else if (!waitTime.getText().matches("\\d+") && !waitTime.getText().isEmpty()) { + return; + } + if (!waitTime.getText().matches("\\d+") && !waitTime.getText().isEmpty()) { showPopup("Wait time must contain numbers.", "warning"); - } else if ((!saveLocation.getText().isEmpty()) - && (!chapterListURL.getText().isEmpty()) - ) { + return; + } + if ((!saveLocation.getText().isEmpty()) && (!chapterListURL.getText().isEmpty())) { grabChaptersButton.setEnabled(false); grabChaptersButton.setVisible(false); stopButton.setEnabled(true); @@ -216,6 +244,7 @@ public GUI() { autoBusyLabel.setVisible(true); auto = new Download(this); if (!auto.chapterLinks.isEmpty()) grabChaptersButton.setEnabled(true); + if (auto.autoChapterToChapter) grabChaptersButton.setEnabled(true); autoBusyLabel.setVisible(false); } })); @@ -482,6 +511,35 @@ public void actionPerformed(ActionEvent e) { } } }); + autoHostSelection.addItemListener(new ItemListener() { + @Override + public void itemStateChanged(ItemEvent e) { + String selection = autoHostSelection.getSelectedItem().toString(); + if (HostSettings.autoChapterToChapterWebsitesList.contains(selection)) { + chapterAllCheckBox.setEnabled(false); + firstChapter.setEnabled(false); + lastChapter.setEnabled(false); + toLastChapter.setEnabled(false); + checkInvertOrder.setEnabled(false); + autoGetNumberButton.setEnabled(false); + autoFirstChapterLbl.setVisible(true); + autoFirstChapterURL.setVisible(true); + autoLastChapterLbl.setVisible(true); + autoLastChapterURL.setVisible(true); + } else { + chapterAllCheckBox.setEnabled(true); + firstChapter.setEnabled(true); + lastChapter.setEnabled(true); + toLastChapter.setEnabled(true); + autoGetNumberButton.setEnabled(true); + checkInvertOrder.setEnabled(true); + autoFirstChapterLbl.setVisible(false); + autoFirstChapterURL.setVisible(false); + autoLastChapterLbl.setVisible(false); + autoLastChapterURL.setVisible(false); + } + } + }); } public static void main(String[] args) { @@ -769,6 +827,10 @@ private void createUIComponents() { button1.setBorder(BorderFactory.createEmptyBorder()); button1.setContentAreaFilled(false); + autoCheckAvailability = new JButton(new ImageIcon(getClass().getResource("/images/check_icon.png"))); + autoCheckAvailability.setBorder(BorderFactory.createEmptyBorder()); + autoCheckAvailability.setContentAreaFilled(false); + autoVisitButton = new JButton(new ImageIcon(getClass().getResource("/images/website_icon.png"))); autoVisitButton.setBorder(BorderFactory.createEmptyBorder()); autoVisitButton.setContentAreaFilled(false);