Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Tweaked Cache-Control headers to respect site headers even more #252

Merged
merged 1 commit into from
Apr 2, 2024
Merged

Tweaked Cache-Control headers to respect site headers even more #252

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 62 additions & 65 deletions app/src/main/java/com/nononsenseapps/feeder/model/FeedParser.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class FeedParser(override val di: DI) : DIAware {
@VisibleForTesting
internal fun getSiteMetaDataInHtml(
url: URL,
html: String,
html: String
): Either<FeedParserError, SiteMetaData> {
if (!html.contains("<head>", ignoreCase = true)) {
// Probably a a feed URL and not a page
Expand All @@ -66,20 +66,20 @@ class FeedParser(override val di: DI) : DIAware {
MetaDataParseError(url = url.toString(), throwable = t).also {
Log.w(LOG_TAG, "Error when fetching site metadata", t)
}
},
}
) {
SiteMetaData(
url = url,
alternateFeedLinks = getAlternateFeedLinksInHtml(html, baseUrl = url),
feedImage = getFeedIconInHtml(html, baseUrl = url),
feedImage = getFeedIconInHtml(html, baseUrl = url)
)
}
}

@VisibleForTesting
internal fun getFeedIconInHtml(
html: String,
baseUrl: URL? = null,
baseUrl: URL? = null
): String? {
val doc =
html.byteInputStream().use {
Expand All @@ -90,14 +90,14 @@ class FeedParser(override val di: DI) : DIAware {
doc.getElementsByAttributeValue("rel", "apple-touch-icon") +
doc.getElementsByAttributeValue("rel", "icon") +
doc.getElementsByAttributeValue("rel", "shortcut icon")
)
)
.filter { it.hasAttr("href") }
.firstNotNullOfOrNull { e ->
when {
baseUrl != null ->
relativeLinkIntoAbsolute(
base = baseUrl,
link = e.attr("href"),
link = e.attr("href")
)

else -> sloppyLinkToStrictURLOrNull(e.attr("href"))?.toString()
Expand All @@ -110,7 +110,7 @@ class FeedParser(override val di: DI) : DIAware {
*/
private fun getAlternateFeedLinksInHtml(
html: String,
baseUrl: URL? = null,
baseUrl: URL? = null
): List<AlternateLink> {
val doc =
html.byteInputStream().use {
Expand Down Expand Up @@ -150,10 +150,10 @@ class FeedParser(override val di: DI) : DIAware {
AlternateLink(
type = e.attr("type"),
link =
relativeLinkIntoAbsoluteOrThrow(
base = baseUrl,
link = e.attr("href"),
),
relativeLinkIntoAbsoluteOrThrow(
base = baseUrl,
link = e.attr("href")
)
)
} catch (e: Exception) {
null
Expand All @@ -164,7 +164,7 @@ class FeedParser(override val di: DI) : DIAware {
sloppyLinkToStrictURLOrNull(e.attr("href"))?.let { l ->
AlternateLink(
type = e.attr("type"),
link = l,
link = l
)
}
}
Expand All @@ -174,7 +174,7 @@ class FeedParser(override val di: DI) : DIAware {
feeds.isNotEmpty() -> feeds
baseUrl?.host == "www.youtube.com" || baseUrl?.host == "youtube.com" ->
findFeedLinksForYoutube(
doc,
doc
)

else -> emptyList()
Expand All @@ -193,8 +193,8 @@ class FeedParser(override val di: DI) : DIAware {
listOf(
AlternateLink(
type = "atom",
link = URL("https://www.youtube.com/feeds/videos.xml?channel_id=$channelId"),
),
link = URL("https://www.youtube.com/feeds/videos.xml?channel_id=$channelId")
)
)
}
}
Expand All @@ -220,14 +220,14 @@ class FeedParser(override val di: DI) : DIAware {
// OkHttp string method handles BOM and Content-Type header in request
parseFeedResponse(
response.request.url.toUrl(),
it,
it
)
} ?: Either.Left(NoBody(url = response.request.url.toString()))
}

private fun parseFeedBytes(
url: URL,
body: ByteArray,
body: ByteArray
): ParsedFeed? {
return goFeedAdapter.parseBody(body)?.asFeed(url)
}
Expand All @@ -237,7 +237,7 @@ class FeedParser(override val di: DI) : DIAware {
*/
fun parseFeedResponse(
url: URL,
responseBody: ResponseBody,
responseBody: ResponseBody
): Either<FeedParserError, ParsedFeed> {
val primaryType = responseBody.contentType()?.type
val subType = responseBody.contentType()?.subtype ?: ""
Expand All @@ -246,7 +246,7 @@ class FeedParser(override val di: DI) : DIAware {
Either.catching(
onCatch = { t ->
RSSParseError(url = url.toString(), throwable = t)
},
}
) {
responseBody.byteStream().use { bs ->
parseFeedBytes(url, bs.readBytes())
Expand All @@ -257,8 +257,8 @@ class FeedParser(override val di: DI) : DIAware {
else -> return Either.Left(
UnsupportedContentType(
url = url.toString(),
mimeType = responseBody.contentType().toString(),
),
mimeType = responseBody.contentType().toString()
)
)
}
}
Expand All @@ -269,12 +269,12 @@ class FeedParser(override val di: DI) : DIAware {
@VisibleForTesting
internal fun parseFeedResponse(
url: URL,
body: String,
body: String
): Either<FeedParserError, ParsedFeed> {
return Either.catching(
onCatch = { t ->
RSSParseError(url = url.toString(), throwable = t)
},
}
) {
parseFeedBytes(url, body.toByteArray())
?: throw NullPointerException("Parsed feed is null")
Expand All @@ -299,7 +299,7 @@ private fun GoFeed.asFeed(url: URL): ParsedFeed =
favicon = null,
author = author?.asParsedAuthor(),
expired = null,
items = items?.mapNotNull { it?.let { FeederGoItem(it, author, url).asParsedArticle() } },
items = items?.mapNotNull { it?.let { FeederGoItem(it, author, url).asParsedArticle() } }
)

private fun FeederGoItem.asParsedArticle() =
Expand All @@ -316,7 +316,7 @@ private fun FeederGoItem.asParsedArticle() =
date_modified = updated,
author = author?.asParsedAuthor(),
tags = categories,
attachments = enclosures?.map { it.asParsedEnclosure() },
attachments = enclosures?.map { it.asParsedEnclosure() }
)

private fun GoEnclosure.asParsedEnclosure() =
Expand All @@ -325,37 +325,34 @@ private fun GoEnclosure.asParsedEnclosure() =
title = null,
mime_type = type,
size_in_bytes = length?.toLongOrNull(),
duration_in_seconds = null,
duration_in_seconds = null
)

private fun GoPerson.asParsedAuthor() =
ParsedAuthor(
name = name,
url = null,
avatar = null,
avatar = null
)

suspend fun OkHttpClient.getResponse(
url: URL,
forceNetwork: Boolean = false,
forceNetwork: Boolean = false
): Response {
val request =
Request.Builder()
.url(url)
.cacheControl(
CacheControl.Builder()
// The time between cache re-validations
.maxAge(
if (forceNetwork) {
0
} else {
// Matches fastest sync schedule
15
},
TimeUnit.MINUTES,
.run {
if (forceNetwork) {
cacheControl(
CacheControl.Builder()
.maxAge(1, TimeUnit.MINUTES)
.build()
)
.build(),
)
} else {
this
}
}
.build()

@Suppress("BlockingMethodInNonBlockingContext")
Expand Down Expand Up @@ -421,25 +418,25 @@ suspend fun OkHttpClient.curl(url: URL): Either<FeedParserError, String> {
onCatch = { throwable ->
FetchError(
throwable = throwable,
url = url.toString(),
url = url.toString()
)
},
}
) {
body.string()
}
} ?: Either.Left(
NoBody(
url = url.toString(),
),
url = url.toString()
)
)
}

else ->
Either.Left(
UnsupportedContentType(
url = url.toString(),
mimeType = contentType.toString(),
),
mimeType = contentType.toString()
)
)
}
}
Expand All @@ -448,21 +445,21 @@ suspend fun OkHttpClient.curl(url: URL): Either<FeedParserError, String> {
Either.Left(
UnsupportedContentType(
url = url.toString(),
mimeType = contentType.toString(),
),
mimeType = contentType.toString()
)
)
}
}
}

suspend fun <T> OkHttpClient.curlAndOnResponse(
url: URL,
block: (suspend (Response) -> Either<FeedParserError, T>),
block: (suspend (Response) -> Either<FeedParserError, T>)
): Either<FeedParserError, T> {
return Either.catching(
onCatch = { t ->
FetchError(url = url.toString(), throwable = t)
},
}
) {
getResponse(url)
}.flatMap { response ->
Expand All @@ -475,8 +472,8 @@ suspend fun <T> OkHttpClient.curlAndOnResponse(
HttpError(
url = url.toString(),
code = response.code,
message = response.message,
),
message = response.message
)
)
}
}
Expand All @@ -496,49 +493,49 @@ sealed class FeedParserError : Parcelable {
data class NotInitializedYet(
override val url: String = "",
override val description: String = "",
override val throwable: Throwable? = null,
override val throwable: Throwable? = null
) : FeedParserError()

@Parcelize
data class FetchError(
override val url: String,
override val throwable: Throwable?,
override val description: String = throwable?.message ?: "",
override val description: String = throwable?.message ?: ""
) : FeedParserError()

@Parcelize
data class NotHTML(
override val url: String,
override val description: String = "",
override val throwable: Throwable? = null,
override val throwable: Throwable? = null
) : FeedParserError()

@Parcelize
data class MetaDataParseError(
override val url: String,
override val throwable: Throwable?,
override val description: String = throwable?.message ?: "",
override val description: String = throwable?.message ?: ""
) : FeedParserError()

@Parcelize
data class RSSParseError(
override val throwable: Throwable?,
override val url: String,
override val description: String = throwable?.message ?: "",
override val description: String = throwable?.message ?: ""
) : FeedParserError()

@Parcelize
data class JsonFeedParseError(
override val throwable: Throwable?,
override val url: String,
override val description: String = throwable?.message ?: "",
override val description: String = throwable?.message ?: ""
) : FeedParserError()

@Parcelize
data class NoAlternateFeeds(
override val url: String,
override val description: String = "",
override val throwable: Throwable? = null,
override val throwable: Throwable? = null
) : FeedParserError()

@Parcelize
Expand All @@ -547,34 +544,34 @@ data class HttpError(
val code: Int,
val message: String,
override val description: String = "$code: $message",
override val throwable: Throwable? = null,
override val throwable: Throwable? = null
) : FeedParserError()

@Parcelize
data class UnsupportedContentType(
override val url: String,
val mimeType: String,
override val description: String = mimeType,
override val throwable: Throwable? = null,
override val throwable: Throwable? = null
) : FeedParserError()

@Parcelize
data class NoBody(
override val url: String,
override val description: String = "",
override val throwable: Throwable? = null,
override val throwable: Throwable? = null
) : FeedParserError()

@Parcelize
data class NoUrl(
override val description: String = "",
override val url: String = "",
override val throwable: Throwable? = null,
override val throwable: Throwable? = null
) : FeedParserError()

@Parcelize
data class FullTextDecodingFailure(
override val url: String,
override val throwable: Throwable?,
override val description: String = throwable?.message ?: "",
override val description: String = throwable?.message ?: ""
) : FeedParserError()
Loading