Skip to content

Commit

Permalink
Merge pull request #1081 from TeamNewPipe/fix/sc/search-next-page
Browse files Browse the repository at this point in the history 
[SoundCloud] Detect whether there are any more search results
  • Loading branch information
@Stypox
Stypox authored Aug 6, 2023
2 parents 7c70fef + 2947257 commit 8fb6ba3
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 54 deletions.
37 changes: 15 additions & 22 deletions ...c/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
package org.schabi.newpipe.extractor.services.soundcloud;

import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
Expand All @@ -28,6 +23,7 @@
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
Expand All @@ -38,7 +34,9 @@
import java.util.List;
import java.util.Map;

import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

public final class SoundcloudParsingHelper {
private static String clientId;
Expand Down Expand Up @@ -200,6 +198,7 @@ public static String getUsersFromApiMinItems(final int minItems,
*
* @return the next streams url, empty if don't have
*/
@Nonnull
public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
final String apiUrl) throws IOException,
ReCaptchaException, ParsingException {
Expand All @@ -221,17 +220,7 @@ public static String getUsersFromApi(final ChannelInfoItemsCollector collector,
}
}

String nextPageUrl;
try {
nextPageUrl = responseObject.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
} catch (final Exception ignored) {
nextPageUrl = "";
}

return nextPageUrl;
return getNextPageUrl(responseObject);
}

/**
Expand Down Expand Up @@ -261,6 +250,7 @@ public static String getStreamsFromApiMinItems(final int minItems,
*
* @return the next streams url, empty if don't have
*/
@Nonnull
public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
final String apiUrl,
final boolean charts) throws IOException,
Expand Down Expand Up @@ -288,17 +278,20 @@ public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
}
}

String nextPageUrl;
return getNextPageUrl(responseObject);
}

@Nonnull
private static String getNextPageUrl(@Nonnull final JsonObject response) {
try {
nextPageUrl = responseObject.getString("next_href");
String nextPageUrl = response.getString("next_href");
if (!nextPageUrl.contains("client_id=")) {
nextPageUrl += "&client_id=" + SoundcloudParsingHelper.clientId();
}
return nextPageUrl;
} catch (final Exception ignored) {
nextPageUrl = "";
return "";
}

return nextPageUrl;
}

public static String getStreamsFromApi(final StreamInfoItemsCollector collector,
Expand Down
28 changes: 9 additions & 19 deletions .../schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;

import java.io.IOException;
Expand All @@ -33,22 +34,7 @@ public SoundcloudCommentsExtractor(final StreamingService service,
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionException,
IOException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(getUrl());

final JsonObject json;
try {
json = JsonParser.object().from(response.responseBody());
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json", e);
}

final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(
getServiceId());

collectStreamsFrom(collector, json.getArray("collection"));

return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return getPage(getUrl());
}

@Override
Expand All @@ -57,9 +43,14 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws Extractio
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
return getPage(page.getUrl());
}

@Nonnull
private InfoItemsPage<CommentsInfoItem> getPage(@Nonnull final String url)
throws ParsingException, IOException, ReCaptchaException {
final Downloader downloader = NewPipe.getDownloader();
final Response response = downloader.get(page.getUrl());
final Response response = downloader.get(url);

final JsonObject json;
try {
Expand All @@ -72,8 +63,7 @@ public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws Extractio
getServiceId());

collectStreamsFrom(collector, json.getArray("collection"));

return new InfoItemsPage<>(collector, new Page(json.getString("next_href")));
return new InfoItemsPage<>(collector, new Page(json.getString("next_href", null)));
}

@Override
Expand Down
47 changes: 34 additions & 13 deletions ...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@
import javax.annotation.Nonnull;

public class SoundcloudSearchExtractor extends SearchExtractor {
private JsonArray initialSearchCollection;
private JsonObject initialSearchObject;
private static final String COLLECTION = "collection";
private static final String TOTAL_RESULTS = "total_results";

public SoundcloudSearchExtractor(final StreamingService service,
final SearchQueryHandler linkHandler) {
Expand All @@ -60,9 +62,15 @@ public List<MetaInfo> getMetaInfo() {
@Nonnull
@Override
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
return new InfoItemsPage<>(
collectItems(initialSearchCollection),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
if (initialSearchObject.getInt(TOTAL_RESULTS) > ITEMS_PER_PAGE) {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)),
getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
} else {
return new InfoItemsPage<>(
collectItems(initialSearchObject.getArray(COLLECTION)), null);
}

}

@Override
Expand All @@ -74,17 +82,23 @@ public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,

final Downloader dl = getDownloader();
final JsonArray searchCollection;
final int totalResults;
try {
final String response = dl.get(page.getUrl(), getExtractorLocalization())
.responseBody();
searchCollection = JsonParser.object().from(response).getArray("collection");
final JsonObject result = JsonParser.object().from(response);
searchCollection = result.getArray(COLLECTION);
totalResults = result.getInt(TOTAL_RESULTS);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
if (getOffsetFromUrl(page.getUrl()) + ITEMS_PER_PAGE < totalResults) {
return new InfoItemsPage<>(collectItems(searchCollection),
getNextPageFromCurrentUrl(page.getUrl(),
currentOffset -> currentOffset + ITEMS_PER_PAGE));
}
return new InfoItemsPage<>(collectItems(searchCollection), null);
}

@Override
Expand All @@ -94,12 +108,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
final String url = getUrl();
try {
final String response = dl.get(url, getExtractorLocalization()).responseBody();
initialSearchCollection = JsonParser.object().from(response).getArray("collection");
initialSearchObject = JsonParser.object().from(response);
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}

if (initialSearchCollection.isEmpty()) {
if (initialSearchObject.getArray(COLLECTION).isEmpty()) {
throw new SearchExtractor.NothingFoundException("Nothing found");
}
}
Expand Down Expand Up @@ -133,13 +147,20 @@ private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(

private Page getNextPageFromCurrentUrl(final String currentUrl,
final IntUnaryOperator newPageOffsetCalculator)
throws MalformedURLException, UnsupportedEncodingException {
final int currentPageOffset = Integer.parseInt(
Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
throws ParsingException {
final int currentPageOffset = getOffsetFromUrl(currentUrl);

return new Page(
currentUrl.replace(
"&offset=" + currentPageOffset,
"&offset=" + newPageOffsetCalculator.applyAsInt(currentPageOffset)));
}

private int getOffsetFromUrl(final String url) throws ParsingException {
try {
return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
} catch (MalformedURLException | UnsupportedEncodingException e) {
throw new ParsingException("Could not get offset from page URL", e);
}
}
}
24 changes: 24 additions & 0 deletions ...rg/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.schabi.newpipe.extractor.services.soundcloud.search;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
Expand Down Expand Up @@ -181,4 +182,27 @@ void testIsVerified() throws IOException, ExtractionException {
assertTrue(verified);
}
}

public static class NoNextPage extends DefaultSearchExtractorTest {

private static SearchExtractor extractor;
private static final String QUERY = "Dan at hor#berlgbd";

@BeforeAll
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getSearchExtractor(QUERY);
extractor.fetchPage();
}

@Override public boolean expectedHasMoreItems() { return false; }
@Override public SearchExtractor extractor() throws Exception { return extractor; }
@Override public StreamingService expectedService() throws Exception { return SoundCloud; }
@Override public String expectedName() throws Exception { return QUERY; }
@Override public String expectedId() throws Exception { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}
}

0 comments on commit 8fb6ba3

Please sign in to comment.