From 267cfa22868fc738541a6d482d15d1a54ce0b939 Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Thu, 31 Oct 2024 14:07:56 +0100
Subject: [PATCH 1/9] bus: add usable hosts route

---
 api/host.go               | 10 +++++
 bus/bus.go                |  2 +
 bus/client/hosts.go       | 15 +++++++
 bus/routes.go             | 23 ++++++++++
 stores/hostdb.go          |  8 ++++
 stores/hostdb_test.go     | 93 +++++++++++++++++++++++++++++++++++++++
 stores/sql/database.go    |  5 +++
 stores/sql/main.go        | 86 ++++++++++++++++++++++++++++++++++++
 stores/sql/mysql/main.go  |  4 ++
 stores/sql/sqlite/main.go |  4 ++
 10 files changed, 250 insertions(+)

diff --git a/api/host.go b/api/host.go
index 66e974a8b..f5056d89c 100644
--- a/api/host.go
+++ b/api/host.go
@@ -102,6 +102,11 @@ type (
 		MaxLastScan     TimeRFC3339
 		Offset          int
 	}
+
+	UsableHostOptions struct {
+		Limit  int
+		Offset int
+	}
 )
 
 type (
@@ -121,6 +126,11 @@ type (
 		Subnets           []string             `json:"subnets"`
 	}
 
+	HostInfo struct {
+		PublicKey  types.PublicKey `json:"publicKey"`
+		SiamuxAddr string          `json:"siamuxAddr"`
+	}
+
 	HostInteractions struct {
 		TotalScans              uint64        `json:"totalScans"`
 		LastScan                time.Time     `json:"lastScan"`
diff --git a/bus/bus.go b/bus/bus.go
index 9b26776bf..25fb10d48 100644
--- a/bus/bus.go
+++ b/bus/bus.go
@@ -211,6 +211,7 @@ type (
 		UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error
 		UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error
 		UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error
+		UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error)
 	}
 
 	// A MetadataStore stores information about contracts and objects.
@@ -428,6 +429,7 @@ func (b *Bus) Handler() http.Handler {
 		"GET    /contract/:id/roots":     b.contractIDRootsHandlerGET,
 		"GET    /contract/:id/size":      b.contractSizeHandlerGET,
 
+		"GET    /hosts":                          b.hostsHandlerGET,
 		"POST   /hosts":                          b.hostsHandlerPOST,
 		"GET    /hosts/allowlist":                b.hostsAllowlistHandlerGET,
 		"PUT    /hosts/allowlist":                b.hostsAllowlistHandlerPUT,
diff --git a/bus/client/hosts.go b/bus/client/hosts.go
index 086c6f5ad..93881a149 100644
--- a/bus/client/hosts.go
+++ b/bus/client/hosts.go
@@ -3,6 +3,7 @@ package client
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"time"
 
 	"go.sia.tech/core/types"
@@ -91,3 +92,17 @@ func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKe
 	err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck)
 	return
 }
+
+// UsableHosts returns a list of hosts that are ready to be used. That means
+// they are deemed usable by the autopilot, they are not gouging, not blocked,
+// not offline, etc.
+func (c *Client) UsableHosts(ctx context.Context, opts api.UsableHostOptions) (hosts []api.HostInfo, err error) {
+	values := url.Values{}
+	values.Set("offset", fmt.Sprint(opts.Offset))
+	if opts.Limit != 0 {
+		values.Set("limit", fmt.Sprint(opts.Limit))
+	}
+
+	err = c.c.WithContext(ctx).GET("/hosts?"+values.Encode(), &hosts)
+	return
+}
diff --git a/bus/routes.go b/bus/routes.go
index 029b61ab6..5db06837f 100644
--- a/bus/routes.go
+++ b/bus/routes.go
@@ -484,6 +484,29 @@ func (b *Bus) walletPendingHandler(jc jape.Context) {
 	jc.Encode(events)
 }
 
+func (b *Bus) hostsHandlerGET(jc jape.Context) {
+	var offset int
+	if jc.DecodeForm("offset", &offset) != nil {
+		return
+	} else if offset < 0 {
+		jc.Error(api.ErrInvalidOffset, http.StatusBadRequest)
+		return
+	}
+
+	limit := -1
+	if jc.DecodeForm("limit", &limit) != nil {
+		return
+	} else if limit < -1 {
+		jc.Error(api.ErrInvalidLimit, http.StatusBadRequest)
+		return
+	}
+
+	hosts, err := b.store.UsableHosts(jc.Request.Context(), offset, limit)
+	if jc.Check("couldn't fetch hosts", err) != nil {
+		return
+	}
+	jc.Encode(hosts)
+}
 func (b *Bus) hostsHandlerPOST(jc jape.Context) {
 	var req api.HostsRequest
 	if jc.Decode(&req) != nil {
diff --git a/stores/hostdb.go b/stores/hostdb.go
index a80df54dd..606b66930 100644
--- a/stores/hostdb.go
+++ b/stores/hostdb.go
@@ -116,3 +116,11 @@ func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api
 		return tx.RecordPriceTables(ctx, priceTableUpdate)
 	})
 }
+
+func (s *SQLStore) UsableHosts(ctx context.Context, offset, limit int) (hosts []api.HostInfo, err error) {
+	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
+		hosts, err = tx.UsableHosts(ctx, offset, limit)
+		return err
+	})
+	return
+}
diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go
index 155bb2c0a..94a5318f5 100644
--- a/stores/hostdb_test.go
+++ b/stores/hostdb_test.go
@@ -446,6 +446,99 @@ func TestHosts(t *testing.T) {
 	}
 }
 
+func TestUsableHosts(t *testing.T) {
+	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
+	defer ss.Close()
+	ctx := context.Background()
+
+	// add autopilot
+	err := ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: api.DefaultAutopilotID})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// prepare hosts & contracts
+	//
+	// h1: usable
+	// h2: usable - best one
+	// h3: not usable - blocked
+	// h4: not usable - no host check
+	// h5: not usable - no contract
+	var hks []types.PublicKey
+	for i := 1; i <= 5; i++ {
+		// add host
+		hk := types.PublicKey{byte(i)}
+		addr := fmt.Sprintf("foo.com:100%d", i)
+		err := ss.addCustomTestHost(hk, addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		hks = append(hks, hk)
+
+		// add host check
+		if i != 4 {
+			hc := newTestHostCheck()
+			if i == 2 {
+				hc.ScoreBreakdown.Age = .2
+			}
+			err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, hk, hc)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// add contract
+		if i != 5 {
+			_, err = ss.addTestContract(types.FileContractID{byte(i)}, hk)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// block host
+		if i == 3 {
+			err = ss.UpdateHostBlocklistEntries(context.Background(), []string{addr}, nil, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	// set siamux port in settings
+	_, err = ss.DB().Exec(context.Background(), "UPDATE hosts SET settings = ? WHERE 1=1", sql.HostSettings(rhpv2.HostSettings{SiaMuxPort: "9983"}))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// assert h1 and h2 are usable and ordered by score
+	hosts, err := ss.UsableHosts(ctx, 0, -1)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 2 {
+		t.Fatal("unexpected", len(hosts))
+	} else if hosts[0].PublicKey != hks[1] || hosts[1].PublicKey != hks[0] {
+		t.Fatal("unexpected", hosts)
+	} else if hosts[0].SiamuxAddr != "foo.com:9983" || hosts[1].SiamuxAddr != "foo.com:9983" {
+		t.Fatal("unexpected", hosts)
+	}
+
+	// assert offset and limit
+	hosts, err = ss.UsableHosts(ctx, 1, 1)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 1 {
+		t.Fatal("unexpected", len(hosts))
+	} else if hosts[0].PublicKey != hks[0] {
+		t.Fatal("unexpected", hosts)
+	}
+	hosts, err = ss.UsableHosts(ctx, 2, 1)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 0 {
+		t.Fatal("unexpected", len(hosts))
+	}
+}
+
 // TestRecordScan is a test for recording scans.
 func TestRecordScan(t *testing.T) {
 	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
diff --git a/stores/sql/database.go b/stores/sql/database.go
index 717fc39e6..d223b94ef 100644
--- a/stores/sql/database.go
+++ b/stores/sql/database.go
@@ -367,6 +367,11 @@ type (
 		// the health of the updated slabs becomes invalid
 		UpdateSlabHealth(ctx context.Context, limit int64, minValidity, maxValidity time.Duration) (int64, error)
 
+		// UsableHosts returns a list of hosts that are ready to be used. That
+		// means they are deemed usable by the autopilot, they are not gouging,
+		// not blocked, not offline, etc.
+		UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error)
+
 		// WalletEvents returns all wallet events in the database.
 		WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error)
 
diff --git a/stores/sql/main.go b/stores/sql/main.go
index a5db9f4d4..8073e3a90 100644
--- a/stores/sql/main.go
+++ b/stores/sql/main.go
@@ -2159,6 +2159,92 @@ func UnspentSiacoinElements(ctx context.Context, tx sql.Tx) (elements []types.Si
 	return
 }
 
+func UsableHosts(ctx context.Context, tx sql.Tx, offset, limit int) ([]api.HostInfo, error) {
+	// handle input parameters
+	if offset < 0 {
+		return nil, ErrNegativeOffset
+	} else if limit == 0 || limit == -1 {
+		limit = math.MaxInt64
+	}
+
+	// only include allowed hosts
+	var whereExprs []string
+	var hasAllowlist bool
+	if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_allowlist_entries)").Scan(&hasAllowlist); err != nil {
+		return nil, fmt.Errorf("failed to check for allowlist: %w", err)
+	} else if hasAllowlist {
+		whereExprs = append(whereExprs, "EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = h.id)")
+	}
+
+	// exclude blocked hosts
+	var hasBlocklist bool
+	if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_blocklist_entries)").Scan(&hasBlocklist); err != nil {
+		return nil, fmt.Errorf("failed to check for blocklist: %w", err)
+	} else if hasBlocklist {
+		whereExprs = append(whereExprs, "NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = h.id)")
+	}
+
+	// fetch autopilot
+	var autopilotID int64
+	if err := tx.QueryRow(ctx, "SELECT id FROM autopilots WHERE identifier = ?", api.DefaultAutopilotID).Scan(&autopilotID); err != nil {
+		return nil, fmt.Errorf("failed to fetch autopilot id: %w", err)
+	}
+
+	// only include usable hosts
+	whereExprs = append(whereExprs, `
+EXISTS (
+	SELECT 1
+	FROM hosts h2
+	INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND hc.db_autopilot_id = ? AND h2.id = h.id
+	WHERE
+		hc.usability_blocked = 0 AND
+		hc.usability_offline = 0 AND
+		hc.usability_low_score = 0 AND
+		hc.usability_redundant_ip = 0 AND
+		hc.usability_gouging = 0 AND
+		hc.usability_not_accepting_contracts = 0 AND
+		hc.usability_not_announced = 0 AND
+		hc.usability_not_completing_scan = 0
+)`)
+
+	// query hosts
+	rows, err := tx.Query(ctx, fmt.Sprintf(`
+	SELECT
+	h.public_key,
+	COALESCE(h.net_address, ""),
+	COALESCE(h.settings->>'$.siamuxport', "") AS siamux_port
+	FROM hosts h
+	INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL
+	INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ?
+	WHERE %s
+	GROUP by h.id
+	ORDER BY hc.score_age * hc.score_collateral * hc.score_interactions * hc.score_storage_remaining * hc.score_uptime * hc.score_version * hc.score_prices DESC
+	LIMIT ? OFFSET ?`, strings.Join(whereExprs, "AND")), autopilotID, autopilotID, limit, offset)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch hosts: %w", err)
+	}
+	defer rows.Close()
+
+	var hosts []api.HostInfo
+	for rows.Next() {
+		var hk PublicKey
+		var addr, port string
+		err := rows.Scan(&hk, &addr, &port)
+		if err != nil {
+			return nil, fmt.Errorf("failed to scan host: %w", err)
+		}
+		host, _, err := net.SplitHostPort(addr)
+		if err != nil || host == "" {
+			continue
+		}
+		hosts = append(hosts, api.HostInfo{
+			PublicKey:  types.PublicKey(hk),
+			SiamuxAddr: net.JoinHostPort(host, port),
+		})
+	}
+	return hosts, nil
+}
+
 func WalletEvents(ctx context.Context, tx sql.Tx, offset, limit int) (events []wallet.Event, _ error) {
 	if limit == 0 || limit == -1 {
 		limit = math.MaxInt64
diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go
index 8cb792063..db18795b3 100644
--- a/stores/sql/mysql/main.go
+++ b/stores/sql/mysql/main.go
@@ -1242,6 +1242,10 @@ func (tx *MainDatabaseTx) UpdateSlabHealth(ctx context.Context, limit int64, min
 	return res.RowsAffected()
 }
 
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx, offset, limit)
+}
+
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {
 	return ssql.WalletEvents(ctx, tx.Tx, offset, limit)
 }
diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go
index f392695c4..877a48648 100644
--- a/stores/sql/sqlite/main.go
+++ b/stores/sql/sqlite/main.go
@@ -1242,6 +1242,10 @@ func (tx *MainDatabaseTx) UpdateSlabHealth(ctx context.Context, limit int64, min
 	return res.RowsAffected()
 }
 
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx, offset, limit)
+}
+
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {
 	return ssql.WalletEvents(ctx, tx.Tx, offset, limit)
 }

From 69bc2f8587a54dcbd1b413f129da15074f8bb1b9 Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Thu, 31 Oct 2024 14:13:57 +0100
Subject: [PATCH 2/9] bus: add newline

---
 bus/routes.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bus/routes.go b/bus/routes.go
index 5db06837f..bf91c4dfe 100644
--- a/bus/routes.go
+++ b/bus/routes.go
@@ -507,6 +507,7 @@ func (b *Bus) hostsHandlerGET(jc jape.Context) {
 	}
 	jc.Encode(hosts)
 }
+
 func (b *Bus) hostsHandlerPOST(jc jape.Context) {
 	var req api.HostsRequest
 	if jc.Decode(&req) != nil {

From 7c678279bc3bdabf480932d6a51b1e72900fa318 Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Thu, 31 Oct 2024 14:27:54 +0100
Subject: [PATCH 3/9] stores: fix only_full_groupby

---
 stores/sql/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stores/sql/main.go b/stores/sql/main.go
index 8073e3a90..8abf06779 100644
--- a/stores/sql/main.go
+++ b/stores/sql/main.go
@@ -2218,7 +2218,7 @@ EXISTS (
 	INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ?
 	WHERE %s
 	GROUP by h.id
-	ORDER BY hc.score_age * hc.score_collateral * hc.score_interactions * hc.score_storage_remaining * hc.score_uptime * hc.score_version * hc.score_prices DESC
+	ORDER BY MAX(hc.score_age) * MAX(hc.score_collateral) * MAX(hc.score_interactions) * MAX(hc.score_storage_remaining) * MAX(hc.score_uptime) * MAX(hc.score_version) * MAX(hc.score_prices) DESC
 	LIMIT ? OFFSET ?`, strings.Join(whereExprs, "AND")), autopilotID, autopilotID, limit, offset)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch hosts: %w", err)

From bf6aabe60867fc8ca7f0f46d017c16bc55f2191a Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Fri, 1 Nov 2024 17:35:41 +0100
Subject: [PATCH 4/9] bus: add gouging check and extend TestGouging

---
 api/host.go                       |  3 +++
 bus/routes.go                     | 16 +++++++++++-
 internal/test/e2e/gouging_test.go | 43 +++++++++++++++++++------------
 stores/sql/main.go                | 13 ++++++++--
 4 files changed, 56 insertions(+), 19 deletions(-)

diff --git a/api/host.go b/api/host.go
index f5056d89c..163c08ba5 100644
--- a/api/host.go
+++ b/api/host.go
@@ -129,6 +129,9 @@ type (
 	HostInfo struct {
 		PublicKey  types.PublicKey `json:"publicKey"`
 		SiamuxAddr string          `json:"siamuxAddr"`
+
+		Prices   HostPriceTable     `json:"prices"`
+		Settings rhpv2.HostSettings `json:"settings"`
 	}
 
 	HostInteractions struct {
diff --git a/bus/routes.go b/bus/routes.go
index 28141ed46..d9227bf06 100644
--- a/bus/routes.go
+++ b/bus/routes.go
@@ -505,7 +505,21 @@ func (b *Bus) hostsHandlerGET(jc jape.Context) {
 	if jc.Check("couldn't fetch hosts", err) != nil {
 		return
 	}
-	jc.Encode(hosts)
+
+	gp, err := b.gougingParams(jc.Request.Context())
+	if jc.Check("could not get gouging parameters", err) != nil {
+		return
+	}
+	gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, nil, nil)
+
+	filtered := hosts[:0]
+	for _, host := range hosts {
+		if gc.Check(&host.Settings, &host.Prices.HostPriceTable).Gouging() {
+			continue
+		}
+		filtered = append(filtered, host)
+	}
+	jc.Encode(filtered)
 }
 
 func (b *Bus) hostsHandlerPOST(jc jape.Context) {
diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go
index 54ea827a4..3ce176da7 100644
--- a/internal/test/e2e/gouging_test.go
+++ b/internal/test/e2e/gouging_test.go
@@ -10,7 +10,6 @@ import (
 	"time"
 
 	rhpv2 "go.sia.tech/core/rhp/v2"
-	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
 	"go.sia.tech/renterd/internal/test"
 	"lukechampine.com/frand"
@@ -30,9 +29,17 @@ func TestGouging(t *testing.T) {
 	cluster.MineBlocks(cfg.Period + 1)
 
 	// add hosts
-	tt.OKAll(cluster.AddHostsBlocking(int(test.AutopilotConfig.Contracts.Amount)))
+	n := int(test.AutopilotConfig.Contracts.Amount)
+	tt.OKAll(cluster.AddHostsBlocking(n))
 	cluster.WaitForAccounts()
 
+	// assert all hosts are usable
+	h, err := b.UsableHosts(context.Background(), api.UsableHostOptions{})
+	tt.OK(err)
+	if len(h) != n {
+		t.Fatal("unexpected number of hosts")
+	}
+
 	// assert that the current period is greater than the period
 	tt.Retry(10, time.Second, func() error {
 		if ap, _ := b.Autopilot(context.Background(), api.DefaultAutopilotID); ap.CurrentPeriod <= cfg.Period {
@@ -62,35 +69,39 @@ func TestGouging(t *testing.T) {
 		t.Fatal("unexpected data")
 	}
 
-	// update the gouging settings to limit the max storage price to 100H
+	// fetch current host settings
+	settings := cluster.hosts[0].settings.Settings()
+
+	// update host settings
+	updated := settings
+	updated.StoragePrice = updated.StoragePrice.Mul64(2)
+	tt.OK(cluster.hosts[0].UpdateSettings(updated))
+
+	// update gouging settings
 	gs := test.GougingSettings
-	gs.MaxStoragePrice = types.NewCurrency64(100)
+	gs.MaxStoragePrice = settings.StoragePrice
 	if err := b.UpdateGougingSettings(context.Background(), gs); err != nil {
 		t.Fatal(err)
 	}
 
-	// fetch current contract set
-	contracts, err := b.Contracts(context.Background(), api.ContractsOpts{ContractSet: cfg.Set})
-	tt.OK(err)
-
-	// update one host's settings so it's gouging
-	hk := contracts[0].HostKey
-	host := hostsMap[hk.String()]
-	settings := host.settings.Settings()
-	settings.StoragePrice = types.NewCurrency64(101) // gouging
-	tt.OK(host.UpdateSettings(settings))
-
 	// make sure the price table expires so the worker is forced to fetch it
 	// again, this is necessary for the host to be considered price gouging
 	time.Sleep(defaultHostSettings.PriceTableValidity)
 
+	// assert all hosts are usable
+	h, err = b.UsableHosts(context.Background(), api.UsableHostOptions{})
+	tt.OK(err)
+	if len(h) != n-1 {
+		t.Fatal("unexpected number of hosts", len(h))
+	}
+
 	// upload some data - should fail
 	tt.FailAll(w.UploadObject(context.Background(), bytes.NewReader(data), testBucket, path, api.UploadObjectOptions{}))
 
 	// update all host settings so they're gouging
 	for _, h := range cluster.hosts {
 		settings := h.settings.Settings()
-		settings.StoragePrice = types.NewCurrency64(101)
+		settings.StoragePrice = settings.StoragePrice.Mul64(2)
 		if err := h.UpdateSettings(settings); err != nil {
 			t.Fatal(err)
 		}
diff --git a/stores/sql/main.go b/stores/sql/main.go
index 8abf06779..253057155 100644
--- a/stores/sql/main.go
+++ b/stores/sql/main.go
@@ -17,6 +17,7 @@ import (
 	dsql "database/sql"
 
 	rhpv2 "go.sia.tech/core/rhp/v2"
+	rhpv3 "go.sia.tech/core/rhp/v3"
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
@@ -2212,7 +2213,10 @@ EXISTS (
 	SELECT
 	h.public_key,
 	COALESCE(h.net_address, ""),
-	COALESCE(h.settings->>'$.siamuxport', "") AS siamux_port
+	COALESCE(h.settings->>'$.siamuxport', "") AS siamux_port,
+	h.price_table,
+	h.price_table_expiry,
+	h.settings
 	FROM hosts h
 	INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL
 	INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ?
@@ -2229,7 +2233,10 @@ EXISTS (
 	for rows.Next() {
 		var hk PublicKey
 		var addr, port string
-		err := rows.Scan(&hk, &addr, &port)
+		var pt PriceTable
+		var hs HostSettings
+		var pte dsql.NullTime
+		err := rows.Scan(&hk, &addr, &port, &pt, &pte, &hs)
 		if err != nil {
 			return nil, fmt.Errorf("failed to scan host: %w", err)
 		}
@@ -2240,6 +2247,8 @@ EXISTS (
 		hosts = append(hosts, api.HostInfo{
 			PublicKey:  types.PublicKey(hk),
 			SiamuxAddr: net.JoinHostPort(host, port),
+			Prices:     api.HostPriceTable{HostPriceTable: rhpv3.HostPriceTable(pt), Expiry: pte.Time},
+			Settings:   rhpv2.HostSettings(hs),
 		})
 	}
 	return hosts, nil

From daf77ff4bc5b8c2542106b21a00c1182e6d94657 Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Tue, 5 Nov 2024 10:24:00 +0100
Subject: [PATCH 5/9] api: remove prices and settings from host info

---
 api/host.go               |  3 ---
 bus/bus.go                |  2 +-
 bus/routes.go             | 16 +++---------
 internal/test/host.go     | 15 ++++++-----
 stores/hostdb.go          |  5 ++--
 stores/hostdb_test.go     | 53 +++++++++++++++++++++++++++++++--------
 stores/sql/database.go    |  3 ++-
 stores/sql/main.go        | 19 +++++++++-----
 stores/sql/mysql/main.go  |  5 ++--
 stores/sql/sqlite/main.go |  5 ++--
 10 files changed, 81 insertions(+), 45 deletions(-)

diff --git a/api/host.go b/api/host.go
index 163c08ba5..f5056d89c 100644
--- a/api/host.go
+++ b/api/host.go
@@ -129,9 +129,6 @@ type (
 	HostInfo struct {
 		PublicKey  types.PublicKey `json:"publicKey"`
 		SiamuxAddr string          `json:"siamuxAddr"`
-
-		Prices   HostPriceTable     `json:"prices"`
-		Settings rhpv2.HostSettings `json:"settings"`
 	}
 
 	HostInteractions struct {
diff --git a/bus/bus.go b/bus/bus.go
index 38b6e4537..35bcc15b5 100644
--- a/bus/bus.go
+++ b/bus/bus.go
@@ -212,7 +212,7 @@ type (
 		UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error
 		UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error
 		UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error
-		UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error)
+		UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error)
 	}
 
 	// A MetadataStore stores information about contracts and objects.
diff --git a/bus/routes.go b/bus/routes.go
index d9227bf06..892535fd8 100644
--- a/bus/routes.go
+++ b/bus/routes.go
@@ -501,25 +501,17 @@ func (b *Bus) hostsHandlerGET(jc jape.Context) {
 		return
 	}
 
-	hosts, err := b.store.UsableHosts(jc.Request.Context(), offset, limit)
-	if jc.Check("couldn't fetch hosts", err) != nil {
-		return
-	}
-
 	gp, err := b.gougingParams(jc.Request.Context())
 	if jc.Check("could not get gouging parameters", err) != nil {
 		return
 	}
 	gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, nil, nil)
 
-	filtered := hosts[:0]
-	for _, host := range hosts {
-		if gc.Check(&host.Settings, &host.Prices.HostPriceTable).Gouging() {
-			continue
-		}
-		filtered = append(filtered, host)
+	hosts, err := b.store.UsableHosts(jc.Request.Context(), gc, offset, limit)
+	if jc.Check("couldn't fetch hosts", err) != nil {
+		return
 	}
-	jc.Encode(filtered)
+	jc.Encode(hosts)
 }
 
 func (b *Bus) hostsHandlerPOST(jc jape.Context) {
diff --git a/internal/test/host.go b/internal/test/host.go
index f5128b8fc..ffde300f4 100644
--- a/internal/test/host.go
+++ b/internal/test/host.go
@@ -46,12 +46,15 @@ func NewHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSet
 
 func NewHostSettings() rhpv2.HostSettings {
 	return rhpv2.HostSettings{
-		AcceptingContracts: true,
-		Collateral:         types.Siacoins(1).Div64(1 << 40),
-		MaxCollateral:      types.Siacoins(10000),
-		MaxDuration:        144 * 7 * 12, // 12w
-		Version:            "1.5.10",
-		RemainingStorage:   1 << 42, // 4 TiB
+		AcceptingContracts:         true,
+		Collateral:                 types.Siacoins(1).Div64(1 << 40),
+		EphemeralAccountExpiry:     time.Hour * 24,
+		MaxCollateral:              types.Siacoins(10000),
+		MaxDuration:                144 * 7 * 12, // 12w
+		MaxEphemeralAccountBalance: types.Siacoins(1000),
+		Version:                    "1.5.10",
+		RemainingStorage:           1 << 42, // 4 TiB
+		SiaMuxPort:                 "9983",
 	}
 }
 
diff --git a/stores/hostdb.go b/stores/hostdb.go
index 606b66930..1f467578e 100644
--- a/stores/hostdb.go
+++ b/stores/hostdb.go
@@ -8,6 +8,7 @@ import (
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/gouging"
 	sql "go.sia.tech/renterd/stores/sql"
 )
 
@@ -117,9 +118,9 @@ func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api
 	})
 }
 
-func (s *SQLStore) UsableHosts(ctx context.Context, offset, limit int) (hosts []api.HostInfo, err error) {
+func (s *SQLStore) UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) (hosts []api.HostInfo, err error) {
 	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
-		hosts, err = tx.UsableHosts(ctx, offset, limit)
+		hosts, err = tx.UsableHosts(ctx, gc, offset, limit)
 		return err
 	})
 	return
diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go
index 94a5318f5..2e49dea4f 100644
--- a/stores/hostdb_test.go
+++ b/stores/hostdb_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -14,6 +15,8 @@ import (
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/gouging"
+	"go.sia.tech/renterd/internal/test"
 	sql "go.sia.tech/renterd/stores/sql"
 )
 
@@ -447,7 +450,11 @@ func TestHosts(t *testing.T) {
 }
 
 func TestUsableHosts(t *testing.T) {
-	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
+	cfg := defaultTestSQLStoreConfig
+	cfg.persistent = true
+	cfg.dir = "/users/peterjan/testing2"
+	os.RemoveAll(cfg.dir)
+	ss := newTestSQLStore(t, cfg)
 	defer ss.Close()
 	ctx := context.Background()
 
@@ -457,6 +464,10 @@ func TestUsableHosts(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	cs := api.ConsensusState{Synced: true}
+	gs := test.GougingSettings
+	gc := gouging.NewChecker(gs, cs, nil, nil)
+
 	// prepare hosts & contracts
 	//
 	// h1: usable
@@ -475,6 +486,15 @@ func TestUsableHosts(t *testing.T) {
 		}
 		hks = append(hks, hk)
 
+		// add host scan
+		hs := test.NewHostSettings()
+		hs.MaxEphemeralAccountBalance = types.Siacoins(1)
+		pt := test.NewHostPriceTable()
+		s1 := newTestScan(hk, time.Now(), hs, pt, true, nil, nil)
+		if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {
+			t.Fatal(err)
+		}
+
 		// add host check
 		if i != 4 {
 			hc := newTestHostCheck()
@@ -504,14 +524,8 @@ func TestUsableHosts(t *testing.T) {
 		}
 	}
 
-	// set siamux port in settings
-	_, err = ss.DB().Exec(context.Background(), "UPDATE hosts SET settings = ? WHERE 1=1", sql.HostSettings(rhpv2.HostSettings{SiaMuxPort: "9983"}))
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// assert h1 and h2 are usable and ordered by score
-	hosts, err := ss.UsableHosts(ctx, 0, -1)
+	hosts, err := ss.UsableHosts(ctx, gc, 0, -1)
 	if err != nil {
 		t.Fatal(err)
 	} else if len(hosts) != 2 {
@@ -523,7 +537,7 @@ func TestUsableHosts(t *testing.T) {
 	}
 
 	// assert offset and limit
-	hosts, err = ss.UsableHosts(ctx, 1, 1)
+	hosts, err = ss.UsableHosts(ctx, gc, 1, 1)
 	if err != nil {
 		t.Fatal(err)
 	} else if len(hosts) != 1 {
@@ -531,12 +545,31 @@ func TestUsableHosts(t *testing.T) {
 	} else if hosts[0].PublicKey != hks[0] {
 		t.Fatal("unexpected", hosts)
 	}
-	hosts, err = ss.UsableHosts(ctx, 2, 1)
+	hosts, err = ss.UsableHosts(ctx, gc, 2, 1)
 	if err != nil {
 		t.Fatal(err)
 	} else if len(hosts) != 0 {
 		t.Fatal("unexpected", len(hosts))
 	}
+
+	// record a scan for h1 to make it gouging
+	hs := test.NewHostSettings()
+	pt := test.NewHostPriceTable()
+	pt.UploadBandwidthCost = gs.MaxUploadPrice
+	s1 := newTestScan(types.PublicKey{1}, time.Now(), hs, pt, true, nil, nil)
+	if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {
+		t.Fatal(err)
+	}
+
+	// assert it's no longer in the result set
+	hosts, err = ss.UsableHosts(ctx, gc, 0, -1)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 1 {
+		t.Fatal("unexpected", len(hosts))
+	} else if hosts[0].PublicKey != hks[1] {
+		t.Fatal("unexpected", hosts)
+	}
 }
 
 // TestRecordScan is a test for recording scans.
diff --git a/stores/sql/database.go b/stores/sql/database.go
index d223b94ef..eaf1e8810 100644
--- a/stores/sql/database.go
+++ b/stores/sql/database.go
@@ -10,6 +10,7 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/object"
 	"go.sia.tech/renterd/webhooks"
 )
@@ -370,7 +371,7 @@ type (
 		// UsableHosts returns a list of hosts that are ready to be used. That
 		// means they are deemed usable by the autopilot, they are not gouging,
 		// not blocked, not offline, etc.
-		UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error)
+		UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error)
 
 		// WalletEvents returns all wallet events in the database.
 		WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error)
diff --git a/stores/sql/main.go b/stores/sql/main.go
index 253057155..798ce6cda 100644
--- a/stores/sql/main.go
+++ b/stores/sql/main.go
@@ -22,6 +22,7 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/internal/sql"
 	"go.sia.tech/renterd/internal/utils"
 	"go.sia.tech/renterd/object"
@@ -2160,7 +2161,7 @@ func UnspentSiacoinElements(ctx context.Context, tx sql.Tx) (elements []types.Si
 	return
 }
 
-func UsableHosts(ctx context.Context, tx sql.Tx, offset, limit int) ([]api.HostInfo, error) {
+func UsableHosts(ctx context.Context, tx sql.Tx, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error) {
 	// handle input parameters
 	if offset < 0 {
 		return nil, ErrNegativeOffset
@@ -2215,7 +2216,6 @@ EXISTS (
 	COALESCE(h.net_address, ""),
 	COALESCE(h.settings->>'$.siamuxport', "") AS siamux_port,
 	h.price_table,
-	h.price_table_expiry,
 	h.settings
 	FROM hosts h
 	INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL
@@ -2235,20 +2235,27 @@ EXISTS (
 		var addr, port string
 		var pt PriceTable
 		var hs HostSettings
-		var pte dsql.NullTime
-		err := rows.Scan(&hk, &addr, &port, &pt, &pte, &hs)
+		err := rows.Scan(&hk, &addr, &port, &pt, &hs)
 		if err != nil {
 			return nil, fmt.Errorf("failed to scan host: %w", err)
 		}
+
+		// exclude hosts that are gouging
+		hss := rhpv2.HostSettings(hs)
+		hpt := rhpv3.HostPriceTable(pt)
+		if gc.Check(&hss, &hpt).Gouging() {
+			continue
+		}
+
+		// exclude hosts with invalid address
 		host, _, err := net.SplitHostPort(addr)
 		if err != nil || host == "" {
 			continue
 		}
+
 		hosts = append(hosts, api.HostInfo{
 			PublicKey:  types.PublicKey(hk),
 			SiamuxAddr: net.JoinHostPort(host, port),
-			Prices:     api.HostPriceTable{HostPriceTable: rhpv3.HostPriceTable(pt), Expiry: pte.Time},
-			Settings:   rhpv2.HostSettings(hs),
 		})
 	}
 	return hosts, nil
diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go
index db18795b3..5527082a8 100644
--- a/stores/sql/mysql/main.go
+++ b/stores/sql/mysql/main.go
@@ -15,6 +15,7 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/object"
 	ssql "go.sia.tech/renterd/stores/sql"
 	"go.sia.tech/renterd/webhooks"
@@ -1242,8 +1243,8 @@ func (tx *MainDatabaseTx) UpdateSlabHealth(ctx context.Context, limit int64, min
 	return res.RowsAffected()
 }
 
-func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) {
-	return ssql.UsableHosts(ctx, tx, offset, limit)
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx, gc, offset, limit)
 }
 
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {
diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go
index 877a48648..2990172d3 100644
--- a/stores/sql/sqlite/main.go
+++ b/stores/sql/sqlite/main.go
@@ -15,6 +15,7 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/internal/sql"
 	"go.sia.tech/renterd/object"
 	ssql "go.sia.tech/renterd/stores/sql"
@@ -1242,8 +1243,8 @@ func (tx *MainDatabaseTx) UpdateSlabHealth(ctx context.Context, limit int64, min
 	return res.RowsAffected()
 }
 
-func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) {
-	return ssql.UsableHosts(ctx, tx, offset, limit)
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx, gc, offset, limit)
 }
 
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {

From 52ea32a736c607188b18477e4c3db1bc2eff615e Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Tue, 5 Nov 2024 10:25:52 +0100
Subject: [PATCH 6/9] stores: remove debug code

---
 stores/hostdb_test.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go
index 2e49dea4f..889bb4270 100644
--- a/stores/hostdb_test.go
+++ b/stores/hostdb_test.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -450,11 +449,7 @@ func TestHosts(t *testing.T) {
 }
 
 func TestUsableHosts(t *testing.T) {
-	cfg := defaultTestSQLStoreConfig
-	cfg.persistent = true
-	cfg.dir = "/users/peterjan/testing2"
-	os.RemoveAll(cfg.dir)
-	ss := newTestSQLStore(t, cfg)
+	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
 	defer ss.Close()
 	ctx := context.Background()
 

From 879172306ca6a9f7779828b14d41c793c0d58c5c Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Tue, 5 Nov 2024 14:14:20 +0100
Subject: [PATCH 7/9] stores: cleanup PR

---
 stores/hostdb_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go
index 889bb4270..4b673e282 100644
--- a/stores/hostdb_test.go
+++ b/stores/hostdb_test.go
@@ -483,7 +483,6 @@ func TestUsableHosts(t *testing.T) {
 
 		// add host scan
 		hs := test.NewHostSettings()
-		hs.MaxEphemeralAccountBalance = types.Siacoins(1)
 		pt := test.NewHostPriceTable()
 		s1 := newTestScan(hk, time.Now(), hs, pt, true, nil, nil)
 		if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {

From 4c40c41b8c68e22c5d39126e240375aa73a319ee Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Fri, 8 Nov 2024 10:46:18 +0100
Subject: [PATCH 8/9] stores: perform gouging check higher up, remove
 offset+limit

---
 api/host.go                       |   5 --
 bus/bus.go                        |   2 +-
 bus/client/hosts.go               |  11 +--
 bus/routes.go                     |  25 +++----
 internal/test/e2e/gouging_test.go |   4 +-
 stores/hostdb.go                  |   9 ++-
 stores/hostdb_test.go             | 109 +++++++++++++++++++-----------
 stores/sql/database.go            |   3 +-
 stores/sql/main.go                |  39 +++++------
 stores/sql/mysql/main.go          |   5 +-
 stores/sql/sqlite/main.go         |   5 +-
 11 files changed, 110 insertions(+), 107 deletions(-)

diff --git a/api/host.go b/api/host.go
index 570c402f6..3ca2cfe3f 100644
--- a/api/host.go
+++ b/api/host.go
@@ -97,11 +97,6 @@ type (
 		MaxLastScan     TimeRFC3339
 		Offset          int
 	}
-
-	UsableHostOptions struct {
-		Limit  int
-		Offset int
-	}
 )
 
 type (
diff --git a/bus/bus.go b/bus/bus.go
index a2e8e0227..ac47aa62d 100644
--- a/bus/bus.go
+++ b/bus/bus.go
@@ -214,7 +214,7 @@ type (
 		UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error
 		UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error
 		UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error
-		UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error)
+		UsableHosts(ctx context.Context) ([]sql.HostInfo, error)
 	}
 
 	// A MetadataStore stores information about contracts and objects.
diff --git a/bus/client/hosts.go b/bus/client/hosts.go
index c6fc05f72..344e7a660 100644
--- a/bus/client/hosts.go
+++ b/bus/client/hosts.go
@@ -3,7 +3,6 @@ package client
 import (
 	"context"
 	"fmt"
-	"net/url"
 	"time"
 
 	"go.sia.tech/core/types"
@@ -80,13 +79,7 @@ func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKe
 // UsableHosts returns a list of hosts that are ready to be used. That means
 // they are deemed usable by the autopilot, they are not gouging, not blocked,
 // not offline, etc.
-func (c *Client) UsableHosts(ctx context.Context, opts api.UsableHostOptions) (hosts []api.HostInfo, err error) {
-	values := url.Values{}
-	values.Set("offset", fmt.Sprint(opts.Offset))
-	if opts.Limit != 0 {
-		values.Set("limit", fmt.Sprint(opts.Limit))
-	}
-
-	err = c.c.WithContext(ctx).GET("/hosts?"+values.Encode(), &hosts)
+func (c *Client) UsableHosts(ctx context.Context) (hosts []api.HostInfo, err error) {
+	err = c.c.WithContext(ctx).GET("/hosts", &hosts)
 	return
 }
diff --git a/bus/routes.go b/bus/routes.go
index 550235490..024558b09 100644
--- a/bus/routes.go
+++ b/bus/routes.go
@@ -485,19 +485,8 @@ func (b *Bus) walletPendingHandler(jc jape.Context) {
 }
 
 func (b *Bus) hostsHandlerGET(jc jape.Context) {
-	var offset int
-	if jc.DecodeForm("offset", &offset) != nil {
-		return
-	} else if offset < 0 {
-		jc.Error(api.ErrInvalidOffset, http.StatusBadRequest)
-		return
-	}
-
-	limit := -1
-	if jc.DecodeForm("limit", &limit) != nil {
-		return
-	} else if limit < -1 {
-		jc.Error(api.ErrInvalidLimit, http.StatusBadRequest)
+	hosts, err := b.store.UsableHosts(jc.Request.Context())
+	if jc.Check("couldn't fetch hosts", err) != nil {
 		return
 	}
 
@@ -507,11 +496,13 @@ func (b *Bus) hostsHandlerGET(jc jape.Context) {
 	}
 	gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, nil, nil)
 
-	hosts, err := b.store.UsableHosts(jc.Request.Context(), gc, offset, limit)
-	if jc.Check("couldn't fetch hosts", err) != nil {
-		return
+	var infos []api.HostInfo
+	for _, h := range hosts {
+		if !gc.Check(&h.HS, &h.PT).Gouging() {
+			infos = append(infos, h.HostInfo)
+		}
 	}
-	jc.Encode(hosts)
+	jc.Encode(infos)
 }
 
 func (b *Bus) hostsHandlerPOST(jc jape.Context) {
diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go
index 3ce176da7..895436474 100644
--- a/internal/test/e2e/gouging_test.go
+++ b/internal/test/e2e/gouging_test.go
@@ -34,7 +34,7 @@ func TestGouging(t *testing.T) {
 	cluster.WaitForAccounts()
 
 	// assert all hosts are usable
-	h, err := b.UsableHosts(context.Background(), api.UsableHostOptions{})
+	h, err := b.UsableHosts(context.Background())
 	tt.OK(err)
 	if len(h) != n {
 		t.Fatal("unexpected number of hosts")
@@ -89,7 +89,7 @@ func TestGouging(t *testing.T) {
 	time.Sleep(defaultHostSettings.PriceTableValidity)
 
 	// assert all hosts are usable
-	h, err = b.UsableHosts(context.Background(), api.UsableHostOptions{})
+	h, err = b.UsableHosts(context.Background())
 	tt.OK(err)
 	if len(h) != n-1 {
 		t.Fatal("unexpected number of hosts", len(h))
diff --git a/stores/hostdb.go b/stores/hostdb.go
index 1f467578e..2d966c0ee 100644
--- a/stores/hostdb.go
+++ b/stores/hostdb.go
@@ -8,7 +8,6 @@ import (
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
-	"go.sia.tech/renterd/internal/gouging"
 	sql "go.sia.tech/renterd/stores/sql"
 )
 
@@ -16,6 +15,10 @@ var (
 	ErrNegativeMaxDowntime = errors.New("max downtime can not be negative")
 )
 
+type Host struct {
+	api.HostInfo
+}
+
 // Host returns information about a host.
 func (s *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) {
 	hosts, err := s.Hosts(ctx, api.HostOptions{
@@ -118,9 +121,9 @@ func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api
 	})
 }
 
-func (s *SQLStore) UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) (hosts []api.HostInfo, err error) {
+func (s *SQLStore) UsableHosts(ctx context.Context) (hosts []sql.HostInfo, err error) {
 	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
-		hosts, err = tx.UsableHosts(ctx, gc, offset, limit)
+		hosts, err = tx.UsableHosts(ctx)
 		return err
 	})
 	return
diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go
index 4b673e282..2e95e78df 100644
--- a/stores/hostdb_test.go
+++ b/stores/hostdb_test.go
@@ -459,19 +459,14 @@ func TestUsableHosts(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cs := api.ConsensusState{Synced: true}
-	gs := test.GougingSettings
-	gc := gouging.NewChecker(gs, cs, nil, nil)
-
 	// prepare hosts & contracts
 	//
 	// h1: usable
-	// h2: usable - best one
-	// h3: not usable - blocked
-	// h4: not usable - no host check
-	// h5: not usable - no contract
+	// h2: not usable - blocked
+	// h3: not usable - no host check
+	// h4: not usable - no contract
 	var hks []types.PublicKey
-	for i := 1; i <= 5; i++ {
+	for i := 1; i <= 4; i++ {
 		// add host
 		hk := types.PublicKey{byte(i)}
 		addr := fmt.Sprintf("foo.com:100%d", i)
@@ -490,11 +485,8 @@ func TestUsableHosts(t *testing.T) {
 		}
 
 		// add host check
-		if i != 4 {
+		if i != 3 {
 			hc := newTestHostCheck()
-			if i == 2 {
-				hc.ScoreBreakdown.Age = .2
-			}
 			err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, hk, hc)
 			if err != nil {
 				t.Fatal(err)
@@ -502,7 +494,7 @@ func TestUsableHosts(t *testing.T) {
 		}
 
 		// add contract
-		if i != 5 {
+		if i != 4 {
 			_, err = ss.addTestContract(types.FileContractID{byte(i)}, hk)
 			if err != nil {
 				t.Fatal(err)
@@ -510,7 +502,7 @@ func TestUsableHosts(t *testing.T) {
 		}
 
 		// block host
-		if i == 3 {
+		if i == 2 {
 			err = ss.UpdateHostBlocklistEntries(context.Background(), []string{addr}, nil, false)
 			if err != nil {
 				t.Fatal(err)
@@ -518,52 +510,91 @@ func TestUsableHosts(t *testing.T) {
 		}
 	}
 
-	// assert h1 and h2 are usable and ordered by score
-	hosts, err := ss.UsableHosts(ctx, gc, 0, -1)
-	if err != nil {
-		t.Fatal(err)
-	} else if len(hosts) != 2 {
-		t.Fatal("unexpected", len(hosts))
-	} else if hosts[0].PublicKey != hks[1] || hosts[1].PublicKey != hks[0] {
-		t.Fatal("unexpected", hosts)
-	} else if hosts[0].SiamuxAddr != "foo.com:9983" || hosts[1].SiamuxAddr != "foo.com:9983" {
-		t.Fatal("unexpected", hosts)
-	}
-
-	// assert offset and limit
-	hosts, err = ss.UsableHosts(ctx, gc, 1, 1)
+	// assert h1 is usable
+	hosts, err := ss.UsableHosts(ctx)
 	if err != nil {
 		t.Fatal(err)
 	} else if len(hosts) != 1 {
 		t.Fatal("unexpected", len(hosts))
 	} else if hosts[0].PublicKey != hks[0] {
 		t.Fatal("unexpected", hosts)
+	} else if hosts[0].SiamuxAddr != "foo.com:9983" {
+		t.Fatal("unexpected", hosts)
 	}
-	hosts, err = ss.UsableHosts(ctx, gc, 2, 1)
-	if err != nil {
-		t.Fatal(err)
-	} else if len(hosts) != 0 {
-		t.Fatal("unexpected", len(hosts))
+
+	// create gouging checker
+	gs := test.GougingSettings
+	cs := api.ConsensusState{Synced: true}
+	gc := gouging.NewChecker(gs, cs, nil, nil)
+
+	// assert h1 is not gouging
+	h1 := hosts[0]
+	if gc.Check(&h1.HS, &h1.PT).Gouging() {
+		t.Fatal("unexpected")
 	}
 
 	// record a scan for h1 to make it gouging
 	hs := test.NewHostSettings()
 	pt := test.NewHostPriceTable()
 	pt.UploadBandwidthCost = gs.MaxUploadPrice
-	s1 := newTestScan(types.PublicKey{1}, time.Now(), hs, pt, true, nil, nil)
+	s1 := newTestScan(h1.PublicKey, time.Now(), hs, pt, true, nil, nil)
 	if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {
 		t.Fatal(err)
 	}
 
-	// assert it's no longer in the result set
-	hosts, err = ss.UsableHosts(ctx, gc, 0, -1)
+	// fetch it again
+	hosts, err = ss.UsableHosts(ctx)
 	if err != nil {
 		t.Fatal(err)
 	} else if len(hosts) != 1 {
 		t.Fatal("unexpected", len(hosts))
-	} else if hosts[0].PublicKey != hks[1] {
-		t.Fatal("unexpected", hosts)
 	}
+
+	// assert h1 is now gouging
+	h1 = hosts[0]
+	if !gc.Check(&h1.HS, &h1.PT).Gouging() {
+		t.Fatal("unexpected")
+	}
+
+	// create helper to assert number of usable hosts
+	assertNumUsableHosts := func(n int) {
+		t.Helper()
+		hosts, err = ss.UsableHosts(ctx)
+		if err != nil {
+			t.Fatal(err)
+		} else if len(hosts) != n {
+			t.Fatal("unexpected", len(hosts))
+		}
+	}
+
+	// unblock h2
+	if err := ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true); err != nil {
+		t.Fatal(err)
+	}
+
+	assertNumUsableHosts(2)
+
+	// add host check for h3
+	hc := newTestHostCheck()
+	err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, types.PublicKey{3}, hc)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertNumUsableHosts(3)
+
+	// add contract for h4
+	_, err = ss.addTestContract(types.FileContractID{byte(4)}, types.PublicKey{4})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertNumUsableHosts(4)
+
+	// add an allowlist
+	ss.UpdateHostAllowlistEntries(context.Background(), []types.PublicKey{{9}}, nil, false)
+
+	assertNumUsableHosts(0)
 }
 
 // TestRecordScan is a test for recording scans.
diff --git a/stores/sql/database.go b/stores/sql/database.go
index da30455a8..8c543855f 100644
--- a/stores/sql/database.go
+++ b/stores/sql/database.go
@@ -10,7 +10,6 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
-	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/object"
 	"go.sia.tech/renterd/webhooks"
 )
@@ -383,7 +382,7 @@ type (
 		// UsableHosts returns a list of hosts that are ready to be used. That
 		// means they are deemed usable by the autopilot, they are not gouging,
 		// not blocked, not offline, etc.
-		UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error)
+		UsableHosts(ctx context.Context) ([]HostInfo, error)
 
 		// WalletEvents returns all wallet events in the database.
 		WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error)
diff --git a/stores/sql/main.go b/stores/sql/main.go
index 4800d48ba..287920942 100644
--- a/stores/sql/main.go
+++ b/stores/sql/main.go
@@ -22,7 +22,6 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
-	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/internal/sql"
 	"go.sia.tech/renterd/internal/utils"
 	"go.sia.tech/renterd/object"
@@ -38,6 +37,12 @@ var (
 
 // helper types
 type (
+	HostInfo struct {
+		api.HostInfo
+		HS rhpv2.HostSettings
+		PT rhpv3.HostPriceTable
+	}
+
 	multipartUpload struct {
 		ID       int64
 		Key      string
@@ -2314,14 +2319,7 @@ func UnspentSiacoinElements(ctx context.Context, tx sql.Tx) (elements []types.Si
 	return
 }
 
-func UsableHosts(ctx context.Context, tx sql.Tx, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error) {
-	// handle input parameters
-	if offset < 0 {
-		return nil, ErrNegativeOffset
-	} else if limit == 0 || limit == -1 {
-		limit = math.MaxInt64
-	}
-
+func UsableHosts(ctx context.Context, tx sql.Tx) ([]HostInfo, error) {
 	// only include allowed hosts
 	var whereExprs []string
 	var hasAllowlist bool
@@ -2374,15 +2372,13 @@ EXISTS (
 	INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL
 	INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ?
 	WHERE %s
-	GROUP by h.id
-	ORDER BY MAX(hc.score_age) * MAX(hc.score_collateral) * MAX(hc.score_interactions) * MAX(hc.score_storage_remaining) * MAX(hc.score_uptime) * MAX(hc.score_version) * MAX(hc.score_prices) DESC
-	LIMIT ? OFFSET ?`, strings.Join(whereExprs, "AND")), autopilotID, autopilotID, limit, offset)
+	GROUP by h.id`, strings.Join(whereExprs, "AND")), autopilotID, autopilotID)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch hosts: %w", err)
 	}
 	defer rows.Close()
 
-	var hosts []api.HostInfo
+	var hosts []HostInfo
 	for rows.Next() {
 		var hk PublicKey
 		var addr, port string
@@ -2393,22 +2389,19 @@ EXISTS (
 			return nil, fmt.Errorf("failed to scan host: %w", err)
 		}
 
-		// exclude hosts that are gouging
-		hss := rhpv2.HostSettings(hs)
-		hpt := rhpv3.HostPriceTable(pt)
-		if gc.Check(&hss, &hpt).Gouging() {
-			continue
-		}
-
 		// exclude hosts with invalid address
 		host, _, err := net.SplitHostPort(addr)
 		if err != nil || host == "" {
 			continue
 		}
 
-		hosts = append(hosts, api.HostInfo{
-			PublicKey:  types.PublicKey(hk),
-			SiamuxAddr: net.JoinHostPort(host, port),
+		hosts = append(hosts, HostInfo{
+			api.HostInfo{
+				PublicKey:  types.PublicKey(hk),
+				SiamuxAddr: net.JoinHostPort(host, port),
+			},
+			rhpv2.HostSettings(hs),
+			rhpv3.HostPriceTable(pt),
 		})
 	}
 	return hosts, nil
diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go
index 23cd07486..b5dec9646 100644
--- a/stores/sql/mysql/main.go
+++ b/stores/sql/mysql/main.go
@@ -15,7 +15,6 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
-	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/object"
 	ssql "go.sia.tech/renterd/stores/sql"
 	"go.sia.tech/renterd/webhooks"
@@ -1256,8 +1255,8 @@ func (tx *MainDatabaseTx) UpsertContractSectors(ctx context.Context, contractSec
 	return nil
 }
 
-func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error) {
-	return ssql.UsableHosts(ctx, tx, gc, offset, limit)
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context) ([]ssql.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx)
 }
 
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {
diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go
index 671ba4e32..44215e18b 100644
--- a/stores/sql/sqlite/main.go
+++ b/stores/sql/sqlite/main.go
@@ -15,7 +15,6 @@ import (
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/renterd/api"
-	"go.sia.tech/renterd/internal/gouging"
 	"go.sia.tech/renterd/internal/sql"
 	"go.sia.tech/renterd/object"
 	ssql "go.sia.tech/renterd/stores/sql"
@@ -1260,8 +1259,8 @@ func (tx *MainDatabaseTx) UpsertContractSectors(ctx context.Context, contractSec
 	return nil
 }
 
-func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, gc gouging.Checker, offset, limit int) ([]api.HostInfo, error) {
-	return ssql.UsableHosts(ctx, tx, gc, offset, limit)
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context) ([]ssql.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx)
 }
 
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {

From f9a119c63c4c60c085b2495a00532543a9122cc7 Mon Sep 17 00:00:00 2001
From: PJ <peterjan.brone@gmail.com>
Date: Fri, 8 Nov 2024 10:51:08 +0100
Subject: [PATCH 9/9] all: cleanup PR

---
 internal/test/e2e/gouging_test.go | 2 +-
 stores/hostdb.go                  | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go
index 895436474..3ebdd672e 100644
--- a/internal/test/e2e/gouging_test.go
+++ b/internal/test/e2e/gouging_test.go
@@ -88,7 +88,7 @@ func TestGouging(t *testing.T) {
 	// again, this is necessary for the host to be considered price gouging
 	time.Sleep(defaultHostSettings.PriceTableValidity)
 
-	// assert all hosts are usable
+	// assert all but one host are usable
 	h, err = b.UsableHosts(context.Background())
 	tt.OK(err)
 	if len(h) != n-1 {
diff --git a/stores/hostdb.go b/stores/hostdb.go
index 2d966c0ee..b1da3c11b 100644
--- a/stores/hostdb.go
+++ b/stores/hostdb.go
@@ -15,10 +15,6 @@ var (
 	ErrNegativeMaxDowntime = errors.New("max downtime can not be negative")
 )
 
-type Host struct {
-	api.HostInfo
-}
-
 // Host returns information about a host.
 func (s *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) {
 	hosts, err := s.Hosts(ctx, api.HostOptions{