Merge branch 'release/0.3.0'

.gitignore

@@ -1,7 +1,13 @@
 # ignore main binary
 /felix
 /felix.exe
+
+#ignore runtime and build artifacts
 /config.yml
+/coverage.txt
+/felix.db
+/dist
+
 
 # ignore Jetbrains IDE project files
 .idea

.goreleaser.yml

@@ -0,0 +1,21 @@
+builds:
+  - binary: felix
+    goos:
+      - linux
+    goarch:
+      - amd64
+    ldflags: -s -w -X main.Version={{.Version}} -X main.GitSummary={{.Commit}} -X main.BuildDate={{.Date}}
+    env:
+      - CGO_ENABLED=0
+
+archive:
+  format: tar.gz
+  format_overrides:
+    - goos: windows
+      format: zip
+  files:
+    - LICENSE
+    - README.md
+    - CHANGELOG.md
+    - VERSION
+    - config.example.yml

.travis.yml

@@ -6,3 +6,4 @@ script: ./coverage.sh
 
 after_success:
   - if [ -z "$TRAVIS_TAG" ]; then bash <(curl -s https://codecov.io/bash); else echo "Tag build detected. Not uploading coverage (codecov)."; fi
+  - test -n "$TRAVIS_TAG" && curl -sL https://git.io/goreleaser | bash

CHANGELOG.md

@@ -8,6 +8,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 Currently no changes or additions.
 
+## [0.3.0] - 2017-10-12
+
+### Added
+
+- Made some previously hard-coded values configurable:
+    - `FeedOutputMaxAge`, the maximum age for links included in the output feed (default: 6h).
+    - `Port`, the TCP port the output feed should listen on (default: 6554).
+- Added overview diagram to README.md.
+- Automated binary releases (linux x86_64 only for now).
+
+### Fixed
+
+- Only report new links the first time they are found.
+- Do not upload coverage for tag CI builds.
+
 ## [0.2.0] - 2017-10-01
 
 ### Added
@@ -32,3 +47,4 @@ This is the first working release.
 [Unreleased]: https://github.com/martinplaner/felix/tree/develop
 [0.1.0]: https://github.com/martinplaner/felix/releases/tag/v0.1.0
 [0.2.0]: https://github.com/martinplaner/felix/releases/tag/v0.2.0
+[0.3.0]: https://github.com/martinplaner/felix/releases/tag/v0.3.0

VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

config.example.yml

@@ -1,4 +1,5 @@
 fetchInterval: 1h5m
+feedOutputMaxAge: 6h
 cleanupInterval: 10m
 cleanupMaxAge: 12h
 

internal/felix/config.go

@@ -18,19 +18,23 @@ import (
 const (
 	DefaultFeedFetchInterval = 65 * time.Minute
 	DefaultUserAgent         = "felix"
+	DefaultPort              = 6554
+	DefaultFeedOutputMaxAge  = 6 * time.Hour
 	DefaultCleanupInterval   = 1 * time.Hour
 	DefaultCleanupMaxAge     = 24 * time.Hour
 )
 
 // Config contains the configuration
 type Config struct {
-	FetchInterval   time.Duration  `yaml:"fetchInterval"`
-	UserAgent       string         `yaml:"userAgent"`
-	CleanupInterval time.Duration  `yaml:"cleanupInterval"`
-	CleanupMaxAge   time.Duration  `yaml:"cleanupMaxAge"`
-	Feeds           []FeedConfig   `yaml:"feeds"`
-	ItemFilters     []FilterConfig `yaml:"itemFilters"`
-	LinkFilters     []FilterConfig `yaml:"linkFilters"`
+	FetchInterval    time.Duration  `yaml:"fetchInterval"`
+	UserAgent        string         `yaml:"userAgent"`
+	Port             int            `yaml:"port"`
+	FeedOutputMaxAge time.Duration  `yaml:"feedOutputMaxAge"`
+	CleanupInterval  time.Duration  `yaml:"cleanupInterval"`
+	CleanupMaxAge    time.Duration  `yaml:"cleanupMaxAge"`
+	Feeds            []FeedConfig   `yaml:"feeds"`
+	ItemFilters      []FilterConfig `yaml:"itemFilters"`
+	LinkFilters      []FilterConfig `yaml:"linkFilters"`
 }
 
 var emptyConfig = Config{}
@@ -109,10 +113,12 @@ type LinkFilenameAsTitleFilterConfig struct {
 func NewConfig() Config {
 	// TODO: return value or pointer?
 	return Config{
-		UserAgent:       DefaultUserAgent,
-		FetchInterval:   DefaultFeedFetchInterval,
-		CleanupInterval: DefaultCleanupInterval,
-		CleanupMaxAge:   DefaultCleanupMaxAge,
+		UserAgent:        DefaultUserAgent,
+		Port:             DefaultPort,
+		FetchInterval:    DefaultFeedFetchInterval,
+		FeedOutputMaxAge: DefaultFeedOutputMaxAge,
+		CleanupInterval:  DefaultCleanupInterval,
+		CleanupMaxAge:    DefaultCleanupMaxAge,
 	}
 }
 

internal/felix/fetcher.go

@@ -29,8 +29,8 @@ type Attempter interface {
 	Inc(key string) error
 }
 
-// NextFetchFunc returns if the fetcher should continue to fetch and if so, how long to wait before the next attempt.
-type NextFetchFunc func(url string) (bool, time.Duration)
+// A NextAttemptFunc returns if and when the next attempt is scheduled for the given key.
+type NextAttemptFunc func(last time.Time, attempts int) (bool, time.Duration)
 
 // NewFetcher creates a new Fetcher.
 func NewFetcher(url string, source Source, scanner Scanner, attempt Attempter, items chan<- Item, links chan<- Link) *Fetcher {
@@ -124,7 +124,7 @@ L:
 // attempt is the default Datastore-backed Attempter
 type attempt struct {
 	ds   Datastore
-	next func(last time.Time, attempts int) (bool, time.Duration)
+	next NextAttemptFunc
 }
 
 func (a attempt) Next(key string) (bool, time.Duration, error) {
@@ -141,23 +141,26 @@ func (a attempt) Inc(key string) error {
 	return a.ds.IncAttempt(key)
 }
 
-// PeriodicAttempter creates a new Attempter for periodic attempts with a fixed interval.
-func PeriodicAttempter(ds Datastore, fetchInterval time.Duration) Attempter {
-	next := func(last time.Time, attempts int) (bool, time.Duration) {
-		nextTry := last.Add(fetchInterval)
-		untilNext := time.Until(nextTry)
-		return true, untilNext
-	}
-
+// creates a new Attempter with the given NextFunc.
+func NewAttempter(ds Datastore, next NextAttemptFunc) Attempter {
 	return &attempt{
 		ds:   ds,
 		next: next,
 	}
 }
 
-// FibAttempter creates a new Attempter for attempts with a fibonacci based backoff interval, up to maxAttempts.
-// The interval length is defined by baseInterval & fib(attempt count).
-func FibAttempter(ds Datastore, baseInterval time.Duration, maxAttempts int) Attempter {
+// PeriodicNextAttemptFunc creates a new NextAttemptFunc for periodic attempts with a fixed interval.
+func PeriodicNextAttemptFunc(fetchInterval time.Duration) NextAttemptFunc {
+	return func(last time.Time, attempts int) (bool, time.Duration) {
+		nextTry := last.Add(fetchInterval)
+		untilNext := time.Until(nextTry)
+		return true, untilNext
+	}
+}
+
+// FibNextAttemptFunc creates a new NextAttemptFunc for attempts with a fibonacci based backoff interval, up to maxAttempts.
+// The interval length is defined by baseInterval * fib(attempt count).
+func FibNextAttemptFunc(baseInterval time.Duration, maxAttempts int) NextAttemptFunc {
 	var fib func(n int) int
 	fib = func(n int) int {
 		if n < 2 {
@@ -166,7 +169,7 @@ func FibAttempter(ds Datastore, baseInterval time.Duration, maxAttempts int) Att
 		return fib(n-2) + fib(n-1)
 	}
 
-	next := func(last time.Time, attempts int) (bool, time.Duration) {
+	return func(last time.Time, attempts int) (bool, time.Duration) {
 		if attempts >= maxAttempts {
 			return false, 0
 		}
@@ -176,9 +179,4 @@ func FibAttempter(ds Datastore, baseInterval time.Duration, maxAttempts int) Att
 		untilNext := time.Until(nextTry)
 		return true, untilNext
 	}
-
-	return &attempt{
-		ds:   ds,
-		next: next,
-	}
 }

internal/felix/fetcher_test.go

@@ -15,6 +15,23 @@ import (
 	"time"
 )
 
+func TestNewFetcher(t *testing.T) {
+	f := NewFetcher("", nil, nil, nil, nil, nil)
+	if f == nil {
+		t.Error("NewFetcher() == nil")
+	}
+}
+
+func TestFetcher_SetLogger(t *testing.T) {
+	f := NewFetcher("", nil, nil, nil, nil, nil)
+	l := NewLogger()
+	f.SetLogger(l)
+
+	if f.log != l {
+		t.Error("logger not set correctly")
+	}
+}
+
 func TestFetcher(t *testing.T) {
 	items := make(chan Item)
 	links := make(chan Link)
@@ -131,3 +148,76 @@ func (a *mockAttempter) Next(key string) (bool, time.Duration, error) {
 func (mockAttempter) Inc(key string) error {
 	return nil
 }
+
+func TestNextAttemptFunc(t *testing.T) {
+	testCases := []struct {
+		desc          string
+		next          NextAttemptFunc
+		last          time.Time
+		attempts      int
+		shouldAttempt bool
+		check         func(time.Duration) bool
+	}{
+		{
+			desc:          "first periodic attempt (no wait)",
+			next:          PeriodicNextAttemptFunc(1 * time.Hour),
+			last:          time.Time{},
+			attempts:      0,
+			shouldAttempt: true,
+			check:         func(d time.Duration) bool { return d < 0 },
+		},
+		{
+			desc:          "second periodic attempt (wait)",
+			next:          PeriodicNextAttemptFunc(1 * time.Hour),
+			last:          time.Now().Add(-5 * time.Minute),
+			attempts:      1,
+			shouldAttempt: true,
+			check:         func(d time.Duration) bool { return d > 0 },
+		},
+		{
+			desc:          "first fibonacci attempt (no wait)",
+			next:          FibNextAttemptFunc(1*time.Hour, 5),
+			last:          time.Time{},
+			attempts:      0,
+			shouldAttempt: true,
+			check:         func(d time.Duration) bool { return d < 0 },
+		},
+		{
+			desc:          "first fibonacci attempt (wait)",
+			next:          FibNextAttemptFunc(1*time.Hour, 5),
+			last:          time.Now().Add(-5 * time.Minute),
+			attempts:      1,
+			shouldAttempt: true,
+			check:         func(d time.Duration) bool { return d > 0 },
+		},
+		{
+			desc:          "fourth fibonacci attempt (2 * baseInterval)",
+			next:          FibNextAttemptFunc(1*time.Hour, 5),
+			last:          time.Now(),
+			attempts:      3,
+			shouldAttempt: true,
+			check:         func(d time.Duration) bool { return d > 118*time.Minute && d < 122*time.Minute },
+		},
+		{
+			desc:          "nth fibonacci attempt (attempts > maxAttempts)",
+			next:          FibNextAttemptFunc(1*time.Hour, 5),
+			last:          time.Now(),
+			attempts:      10,
+			shouldAttempt: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			shouldAttempt, waitFor := tc.next(tc.last, tc.attempts)
+
+			if shouldAttempt != tc.shouldAttempt {
+				t.Errorf("shouldAttempt = %v, expected %v", shouldAttempt, tc.shouldAttempt)
+			}
+
+			if tc.check != nil && !tc.check(waitFor) {
+				t.Errorf("check(waitFor = %v) failed", waitFor)
+			}
+		})
+	}
+}

internal/felix/http.go

@@ -24,12 +24,11 @@ func StringHandler(s string) http.Handler {
 	})
 }
 
-// FeedHandler serves the found links as an RSS feed.
-func FeedHandler(ds Datastore) http.Handler {
+// FeedHandler serves the found links (up to maxAge) as an RSS feed.
+func FeedHandler(ds Datastore, maxAge time.Duration) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 
-		// TODO: make configurable
-		links, err := ds.GetLinks(3 * time.Hour)
+		links, err := ds.GetLinks(maxAge)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return