Merge branch 'miniflux:main' into dev-keep-rss-content-and-web-content

ChangeLog

@@ -1,3 +1,37 @@
+Version 2.0.47 (August 20, 2023)
+--------------------------------
+
+* Update rules for `webtoons.com`
+* Use HTTP client from the standard library for third-party integrations
+* Rename internal `url` package to `urllib` to avoid overlap with `net/url`
+* Add Shaarli integration
+* Add Shiori integration
+* Add Apprise integration
+* Add Readwise Reader integration
+* Consider base path when generating third-party services API endpoint
+* Use podcast duration tag as reading time
+* Move internal packages to an `internal` folder
+    * For reference: <https://go.dev/doc/go1.4#internalpackages>
+* Rename Miniflux package name to follow Go module naming convention
+    * For reference: <https://go.dev/ref/mod#major-version-suffixes>
+* Update RockyLinux image from 8 to 9 (used to build RPM package)
+* Add force refresh in feed edit and feed entries page
+* Use Odysee video duration as read time
+* Upgrade to Go 1.21
+* Use details disclosure element to show the list of third-party services
+* Use Web Share API for sharing entry
+* Add a workaround for parsing some invalid date format
+* Add Thunder Client API collection into contrib folder
+* Add new API endpoint: `/entries/{entryID}/save`
+* Trigger Docker and packages workflows only for semantic tags
+    * Go module versioning expect Git tags to start with the letter v.
+    * The goal is to keep the existing naming convention for generated artifacts and 
+    have proper versioning for the Go module.
+* Bump `golang.org/x/*` dependencies
+* Bump `github.com/yuin/goldmark`
+* Bump `github.com/tdewolff/minify/v2`
+* Bump `github.com/mccutchen/go-httpbin/v2`
+
 Version 2.0.46 (July 21, 2023)
 ------------------------------
 

internal/config/config_test.go

@@ -724,6 +724,41 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
 	}
 }
 
+func TestDefautSchedulerEntryFrequencyFactorValue(t *testing.T) {
+	os.Clearenv()
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing failure: %v`, err)
+	}
+
+	expected := defaultSchedulerEntryFrequencyFactor
+	result := opts.SchedulerEntryFrequencyFactor()
+
+	if result != expected {
+		t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected)
+	}
+}
+
+func TestDefautSchedulerEntryFrequencyFactor(t *testing.T) {
+	os.Clearenv()
+	os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", "2")
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing failure: %v`, err)
+	}
+
+	expected := 2
+	result := opts.SchedulerEntryFrequencyFactor()
+
+	if result != expected {
+		t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected)
+	}
+}
+
 func TestPollingParsingErrorLimit(t *testing.T) {
 	os.Clearenv()
 	os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100")

internal/config/options.go

@@ -30,6 +30,7 @@ const (
 	defaultPollingScheduler                   = "round_robin"
 	defaultSchedulerEntryFrequencyMinInterval = 5
 	defaultSchedulerEntryFrequencyMaxInterval = 24 * 60
+	defaultSchedulerEntryFrequencyFactor      = 1
 	defaultPollingParsingErrorLimit           = 3
 	defaultRunMigrations                      = false
 	defaultDatabaseURL                        = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
@@ -118,6 +119,7 @@ type Options struct {
 	pollingScheduler                   string
 	schedulerEntryFrequencyMinInterval int
 	schedulerEntryFrequencyMaxInterval int
+	schedulerEntryFrequencyFactor      int
 	pollingParsingErrorLimit           int
 	workerPoolSize                     int
 	createAdmin                        bool
@@ -191,6 +193,7 @@ func NewOptions() *Options {
 		pollingScheduler:                   defaultPollingScheduler,
 		schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval,
 		schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval,
+		schedulerEntryFrequencyFactor:      defaultSchedulerEntryFrequencyFactor,
 		pollingParsingErrorLimit:           defaultPollingParsingErrorLimit,
 		workerPoolSize:                     defaultWorkerPoolSize,
 		createAdmin:                        defaultCreateAdmin,
@@ -368,6 +371,11 @@ func (o *Options) SchedulerEntryFrequencyMinInterval() int {
 	return o.schedulerEntryFrequencyMinInterval
 }
 
+// SchedulerEntryFrequencyFactor returns the factor for the entry frequency scheduler.
+func (o *Options) SchedulerEntryFrequencyFactor() int {
+	return o.schedulerEntryFrequencyFactor
+}
+
 // PollingParsingErrorLimit returns the limit of errors when to stop polling.
 func (o *Options) PollingParsingErrorLimit() int {
 	return o.pollingParsingErrorLimit
@@ -628,6 +636,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
 		"RUN_MIGRATIONS":                         o.runMigrations,
 		"SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval,
 		"SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": o.schedulerEntryFrequencyMinInterval,
+		"SCHEDULER_ENTRY_FREQUENCY_FACTOR":       o.schedulerEntryFrequencyFactor,
 		"SCHEDULER_SERVICE":                      o.schedulerService,
 		"SERVER_TIMING_HEADER":                   o.serverTimingHeader,
 		"WATCHDOG":                               o.watchdog,

internal/config/parser.go

@@ -135,6 +135,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval)
 		case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL":
 			p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval)
+		case "SCHEDULER_ENTRY_FREQUENCY_FACTOR":
+			p.opts.schedulerEntryFrequencyFactor = parseInt(value, defaultSchedulerEntryFrequencyFactor)
 		case "POLLING_PARSING_ERROR_LIMIT":
 			p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
 		// kept for compatibility purpose

internal/googlereader/handler.go

@@ -815,7 +815,7 @@ func (h *handler) streamItemContents(w http.ResponseWriter, r *http.Request) {
 		Updated: time.Now().Unix(),
 		Self: []contentHREF{
 			{
-				HREF: config.Opts.BaseURL() + route.Path(h.router, "StreamItemsContents"),
+				HREF: config.Opts.RootURL() + route.Path(h.router, "StreamItemsContents"),
 			},
 		},
 		Author: user.Username,

internal/model/feed.go

@@ -115,10 +115,10 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int) {
 		if weeklyCount == 0 {
 			intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval()
 		} else {
-			intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount)))
+			intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount*config.Opts.SchedulerEntryFrequencyFactor())))
+			intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
+			intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
 		}
-		intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
-		intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
 		f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes))
 	default:
 		f.NextCheckAt = time.Now()

internal/model/feed_test.go

@@ -152,3 +152,28 @@ func TestFeedScheduleNextCheckEntryCountBasedMinInterval(t *testing.T) {
 		t.Error(`The next_check_at should not be before the now + min interval`)
 	}
 }
+
+func TestFeedScheduleNextCheckEntryFrequencyFactor(t *testing.T) {
+	factor := 2
+	os.Clearenv()
+	os.Setenv("POLLING_SCHEDULER", "entry_frequency")
+	os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", fmt.Sprintf("%d", factor))
+
+	var err error
+	parser := config.NewParser()
+	config.Opts, err = parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing failure: %v`, err)
+	}
+	feed := &Feed{}
+	weeklyCount := 7
+	feed.ScheduleNextCheck(weeklyCount)
+
+	if feed.NextCheckAt.IsZero() {
+		t.Error(`The next_check_at must be set`)
+	}
+
+	if feed.NextCheckAt.After(time.Now().Add(time.Minute * time.Duration(config.Opts.SchedulerEntryFrequencyMaxInterval()/factor))) {
+		t.Error(`The next_check_at should not be after the now + factor * count`)
+	}
+}

internal/reader/scraper/rules.go

@@ -50,7 +50,7 @@ var predefinedRules = map[string]string{
 	"universfreebox.com":   "#corps_corps",
 	"version2.dk":          "section.body",
 	"wdwnt.com":            "div.entry-content",
-	"webtoons.com":         ".viewer_img",
+	"webtoons.com":         ".viewer_img,p.author_text",
 	"wired.com":            "main figure, article",
 	"zeit.de":              ".summary, .article-body",
 	"zdnet.com":            "div.storyBody",

internal/storage/entry_query_builder.go

@@ -267,6 +267,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
 			f.crawler,
 			f.user_agent,
 			f.cookie,
+			f.hide_globally,
 			f.no_media_player,
 			fi.icon_id,
 			u.timezone
@@ -333,6 +334,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
 			&entry.Feed.Crawler,
 			&entry.Feed.UserAgent,
 			&entry.Feed.Cookie,
+			&entry.Feed.HideGlobally,
 			&entry.Feed.NoMediaPlayer,
 			&iconID,
 			&tz,

internal/ui/integration_pocket.go

@@ -32,7 +32,7 @@ func (h *handler) pocketAuthorize(w http.ResponseWriter, r *http.Request) {
 
 	sess := session.New(h.store, request.SessionID(r))
 	connector := pocket.NewConnector(config.Opts.PocketConsumerKey(integration.PocketConsumerKey))
-	redirectURL := config.Opts.BaseURL() + route.Path(h.router, "pocketCallback")
+	redirectURL := config.Opts.RootURL() + route.Path(h.router, "pocketCallback")
 	requestToken, err := connector.RequestToken(redirectURL)
 	if err != nil {
 		logger.Error("[Pocket:Authorize] %v", err)

miniflux.1

@@ -181,6 +181,11 @@ Minimum interval in minutes for the entry frequency scheduler\&.
 .br
 Default is 5 minutes\&.
 .TP
+.B SCHEDULER_ENTRY_FREQUENCY_FACTOR
+Factor to increase refresh frequency for the entry frequency scheduler\&.
+.br
+Default is 1\&.
+.TP
 .B POLLING_PARSING_ERROR_LIMIT
 The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited.
 .br