diff --git a/ChangeLog b/ChangeLog index 17dc7f2ad88..dfc36917bb6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,37 @@ +Version 2.0.47 (August 20, 2023) +-------------------------------- + +* Update rules for `webtoons.com` +* Use HTTP client from the standard library for third-party integrations +* Rename internal `url` package to `urllib` to avoid overlap with `net/url` +* Add Shaarli integration +* Add Shiori integration +* Add Apprise integration +* Add Readwise Reader integration +* Consider base path when generating third-party services API endpoint +* Use podcast duration tag as reading time +* Move internal packages to an `internal` folder + * For reference: +* Rename Miniflux package name to follow Go module naming convention + * For reference: +* Update RockyLinux image from 8 to 9 (used to build RPM package) +* Add force refresh in feed edit and feed entries page +* Use Odysee video duration as read time +* Upgrade to Go 1.21 +* Use details disclosure element to show the list of third-party services +* Use Web Share API for sharing entry +* Add a workaround for parsing some invalid date format +* Add Thunder Client API collection into contrib folder +* Add new API endpoint: `/entries/{entryID}/save` +* Trigger Docker and packages workflows only for semantic tags + * Go module versioning expect Git tags to start with the letter v. + * The goal is to keep the existing naming convention for generated artifacts and + have proper versioning for the Go module. +* Bump `golang.org/x/*` dependencies +* Bump `github.com/yuin/goldmark` +* Bump `github.com/tdewolff/minify/v2` +* Bump `github.com/mccutchen/go-httpbin/v2` + Version 2.0.46 (July 21, 2023) ------------------------------ diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 596d640b9f8..d10c4471b33 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -724,6 +724,41 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) { } } +func TestDefautSchedulerEntryFrequencyFactorValue(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := defaultSchedulerEntryFrequencyFactor + result := opts.SchedulerEntryFrequencyFactor() + + if result != expected { + t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected) + } +} + +func TestDefautSchedulerEntryFrequencyFactor(t *testing.T) { + os.Clearenv() + os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", "2") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := 2 + result := opts.SchedulerEntryFrequencyFactor() + + if result != expected { + t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected) + } +} + func TestPollingParsingErrorLimit(t *testing.T) { os.Clearenv() os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100") diff --git a/internal/config/options.go b/internal/config/options.go index 5ca6306cdfc..d4a56d4ba43 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -30,6 +30,7 @@ const ( defaultPollingScheduler = "round_robin" defaultSchedulerEntryFrequencyMinInterval = 5 defaultSchedulerEntryFrequencyMaxInterval = 24 * 60 + defaultSchedulerEntryFrequencyFactor = 1 defaultPollingParsingErrorLimit = 3 defaultRunMigrations = false defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable" @@ -118,6 +119,7 @@ type Options struct { pollingScheduler string schedulerEntryFrequencyMinInterval int schedulerEntryFrequencyMaxInterval int + schedulerEntryFrequencyFactor int pollingParsingErrorLimit int workerPoolSize int createAdmin bool @@ -191,6 +193,7 @@ func NewOptions() *Options { pollingScheduler: defaultPollingScheduler, schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval, schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval, + schedulerEntryFrequencyFactor: defaultSchedulerEntryFrequencyFactor, pollingParsingErrorLimit: defaultPollingParsingErrorLimit, workerPoolSize: defaultWorkerPoolSize, createAdmin: defaultCreateAdmin, @@ -368,6 +371,11 @@ func (o *Options) SchedulerEntryFrequencyMinInterval() int { return o.schedulerEntryFrequencyMinInterval } +// SchedulerEntryFrequencyFactor returns the factor for the entry frequency scheduler. +func (o *Options) SchedulerEntryFrequencyFactor() int { + return o.schedulerEntryFrequencyFactor +} + // PollingParsingErrorLimit returns the limit of errors when to stop polling. func (o *Options) PollingParsingErrorLimit() int { return o.pollingParsingErrorLimit @@ -628,6 +636,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "RUN_MIGRATIONS": o.runMigrations, "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval, "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": o.schedulerEntryFrequencyMinInterval, + "SCHEDULER_ENTRY_FREQUENCY_FACTOR": o.schedulerEntryFrequencyFactor, "SCHEDULER_SERVICE": o.schedulerService, "SERVER_TIMING_HEADER": o.serverTimingHeader, "WATCHDOG": o.watchdog, diff --git a/internal/config/parser.go b/internal/config/parser.go index c754de5c3bf..bb43bafa19e 100644 --- a/internal/config/parser.go +++ b/internal/config/parser.go @@ -135,6 +135,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval) case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval) + case "SCHEDULER_ENTRY_FREQUENCY_FACTOR": + p.opts.schedulerEntryFrequencyFactor = parseInt(value, defaultSchedulerEntryFrequencyFactor) case "POLLING_PARSING_ERROR_LIMIT": p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit) // kept for compatibility purpose diff --git a/internal/googlereader/handler.go b/internal/googlereader/handler.go index 214fd8845ef..d3c0dfcea1c 100644 --- a/internal/googlereader/handler.go +++ b/internal/googlereader/handler.go @@ -815,7 +815,7 @@ func (h *handler) streamItemContents(w http.ResponseWriter, r *http.Request) { Updated: time.Now().Unix(), Self: []contentHREF{ { - HREF: config.Opts.BaseURL() + route.Path(h.router, "StreamItemsContents"), + HREF: config.Opts.RootURL() + route.Path(h.router, "StreamItemsContents"), }, }, Author: user.Username, diff --git a/internal/model/feed.go b/internal/model/feed.go index 4add53b676f..642695671b4 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -115,10 +115,10 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int) { if weeklyCount == 0 { intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval() } else { - intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount))) + intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount*config.Opts.SchedulerEntryFrequencyFactor()))) + intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval()))) + intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval()))) } - intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval()))) - intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval()))) f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes)) default: f.NextCheckAt = time.Now() diff --git a/internal/model/feed_test.go b/internal/model/feed_test.go index 91850f509e4..34f89e909a8 100644 --- a/internal/model/feed_test.go +++ b/internal/model/feed_test.go @@ -152,3 +152,28 @@ func TestFeedScheduleNextCheckEntryCountBasedMinInterval(t *testing.T) { t.Error(`The next_check_at should not be before the now + min interval`) } } + +func TestFeedScheduleNextCheckEntryFrequencyFactor(t *testing.T) { + factor := 2 + os.Clearenv() + os.Setenv("POLLING_SCHEDULER", "entry_frequency") + os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", fmt.Sprintf("%d", factor)) + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + feed := &Feed{} + weeklyCount := 7 + feed.ScheduleNextCheck(weeklyCount) + + if feed.NextCheckAt.IsZero() { + t.Error(`The next_check_at must be set`) + } + + if feed.NextCheckAt.After(time.Now().Add(time.Minute * time.Duration(config.Opts.SchedulerEntryFrequencyMaxInterval()/factor))) { + t.Error(`The next_check_at should not be after the now + factor * count`) + } +} diff --git a/internal/reader/scraper/rules.go b/internal/reader/scraper/rules.go index b0682aead51..3a73438600c 100644 --- a/internal/reader/scraper/rules.go +++ b/internal/reader/scraper/rules.go @@ -50,7 +50,7 @@ var predefinedRules = map[string]string{ "universfreebox.com": "#corps_corps", "version2.dk": "section.body", "wdwnt.com": "div.entry-content", - "webtoons.com": ".viewer_img", + "webtoons.com": ".viewer_img,p.author_text", "wired.com": "main figure, article", "zeit.de": ".summary, .article-body", "zdnet.com": "div.storyBody", diff --git a/internal/storage/entry_query_builder.go b/internal/storage/entry_query_builder.go index 89fa5ffc591..173fe46addb 100644 --- a/internal/storage/entry_query_builder.go +++ b/internal/storage/entry_query_builder.go @@ -267,6 +267,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { f.crawler, f.user_agent, f.cookie, + f.hide_globally, f.no_media_player, fi.icon_id, u.timezone @@ -333,6 +334,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { &entry.Feed.Crawler, &entry.Feed.UserAgent, &entry.Feed.Cookie, + &entry.Feed.HideGlobally, &entry.Feed.NoMediaPlayer, &iconID, &tz, diff --git a/internal/ui/integration_pocket.go b/internal/ui/integration_pocket.go index 30c1e6bf041..752394f58b5 100644 --- a/internal/ui/integration_pocket.go +++ b/internal/ui/integration_pocket.go @@ -32,7 +32,7 @@ func (h *handler) pocketAuthorize(w http.ResponseWriter, r *http.Request) { sess := session.New(h.store, request.SessionID(r)) connector := pocket.NewConnector(config.Opts.PocketConsumerKey(integration.PocketConsumerKey)) - redirectURL := config.Opts.BaseURL() + route.Path(h.router, "pocketCallback") + redirectURL := config.Opts.RootURL() + route.Path(h.router, "pocketCallback") requestToken, err := connector.RequestToken(redirectURL) if err != nil { logger.Error("[Pocket:Authorize] %v", err) diff --git a/miniflux.1 b/miniflux.1 index 1d66a8c4898..7ae9b507b75 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -181,6 +181,11 @@ Minimum interval in minutes for the entry frequency scheduler\&. .br Default is 5 minutes\&. .TP +.B SCHEDULER_ENTRY_FREQUENCY_FACTOR +Factor to increase refresh frequency for the entry frequency scheduler\&. +.br +Default is 1\&. +.TP .B POLLING_PARSING_ERROR_LIMIT The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited. .br