Add usable hosts endpoint (#1643)

Partially implements
#1642, I'm doing the
second part in another PR because it touches on quite a lot of parts.

api/host.go

@@ -116,6 +116,11 @@ type (
 		Subnets           []string             `json:"subnets"`
 	}
 
+	HostInfo struct {
+		PublicKey  types.PublicKey `json:"publicKey"`
+		SiamuxAddr string          `json:"siamuxAddr"`
+	}
+
 	HostInteractions struct {
 		TotalScans              uint64        `json:"totalScans"`
 		LastScan                time.Time     `json:"lastScan"`

bus/bus.go

@@ -214,6 +214,7 @@ type (
 		UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error
 		UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error
 		UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error
+		UsableHosts(ctx context.Context) ([]sql.HostInfo, error)
 	}
 
 	// A MetadataStore stores information about contracts and objects.
@@ -438,6 +439,7 @@ func (b *Bus) Handler() http.Handler {
 		"GET    /contract/:id/roots":     b.contractIDRootsHandlerGET,
 		"GET    /contract/:id/size":      b.contractSizeHandlerGET,
 
+		"GET    /hosts":                          b.hostsHandlerGET,
 		"POST   /hosts":                          b.hostsHandlerPOST,
 		"GET    /hosts/allowlist":                b.hostsAllowlistHandlerGET,
 		"PUT    /hosts/allowlist":                b.hostsAllowlistHandlerPUT,

bus/client/hosts.go

@@ -75,3 +75,11 @@ func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKe
 	err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck)
 	return
 }
+
+// UsableHosts returns a list of hosts that are ready to be used. That means
+// they are deemed usable by the autopilot, they are not gouging, not blocked,
+// not offline, etc.
+func (c *Client) UsableHosts(ctx context.Context) (hosts []api.HostInfo, err error) {
+	err = c.c.WithContext(ctx).GET("/hosts", &hosts)
+	return
+}

bus/routes.go

@@ -484,6 +484,27 @@ func (b *Bus) walletPendingHandler(jc jape.Context) {
 	jc.Encode(events)
 }
 
+func (b *Bus) hostsHandlerGET(jc jape.Context) {
+	hosts, err := b.store.UsableHosts(jc.Request.Context())
+	if jc.Check("couldn't fetch hosts", err) != nil {
+		return
+	}
+
+	gp, err := b.gougingParams(jc.Request.Context())
+	if jc.Check("could not get gouging parameters", err) != nil {
+		return
+	}
+	gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, nil, nil)
+
+	var infos []api.HostInfo
+	for _, h := range hosts {
+		if !gc.Check(&h.HS, &h.PT).Gouging() {
+			infos = append(infos, h.HostInfo)
+		}
+	}
+	jc.Encode(infos)
+}
+
 func (b *Bus) hostsHandlerPOST(jc jape.Context) {
 	var req api.HostsRequest
 	if jc.Decode(&req) != nil {

internal/test/e2e/gouging_test.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	rhpv2 "go.sia.tech/core/rhp/v2"
-	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
 	"go.sia.tech/renterd/internal/test"
 	"lukechampine.com/frand"
@@ -30,9 +29,17 @@ func TestGouging(t *testing.T) {
 	cluster.MineBlocks(cfg.Period + 1)
 
 	// add hosts
-	tt.OKAll(cluster.AddHostsBlocking(int(test.AutopilotConfig.Contracts.Amount)))
+	n := int(test.AutopilotConfig.Contracts.Amount)
+	tt.OKAll(cluster.AddHostsBlocking(n))
 	cluster.WaitForAccounts()
 
+	// assert all hosts are usable
+	h, err := b.UsableHosts(context.Background())
+	tt.OK(err)
+	if len(h) != n {
+		t.Fatal("unexpected number of hosts")
+	}
+
 	// assert that the current period is greater than the period
 	tt.Retry(10, time.Second, func() error {
 		if ap, _ := b.Autopilot(context.Background(), api.DefaultAutopilotID); ap.CurrentPeriod <= cfg.Period {
@@ -62,35 +69,39 @@ func TestGouging(t *testing.T) {
 		t.Fatal("unexpected data")
 	}
 
-	// update the gouging settings to limit the max storage price to 100H
+	// fetch current host settings
+	settings := cluster.hosts[0].settings.Settings()
+
+	// update host settings
+	updated := settings
+	updated.StoragePrice = updated.StoragePrice.Mul64(2)
+	tt.OK(cluster.hosts[0].UpdateSettings(updated))
+
+	// update gouging settings
 	gs := test.GougingSettings
-	gs.MaxStoragePrice = types.NewCurrency64(100)
+	gs.MaxStoragePrice = settings.StoragePrice
 	if err := b.UpdateGougingSettings(context.Background(), gs); err != nil {
 		t.Fatal(err)
 	}
 
-	// fetch current contract set
-	contracts, err := b.Contracts(context.Background(), api.ContractsOpts{ContractSet: cfg.Set})
-	tt.OK(err)
-
-	// update one host's settings so it's gouging
-	hk := contracts[0].HostKey
-	host := hostsMap[hk.String()]
-	settings := host.settings.Settings()
-	settings.StoragePrice = types.NewCurrency64(101) // gouging
-	tt.OK(host.UpdateSettings(settings))
-
 	// make sure the price table expires so the worker is forced to fetch it
 	// again, this is necessary for the host to be considered price gouging
 	time.Sleep(defaultHostSettings.PriceTableValidity)
 
+	// assert all but one host are usable
+	h, err = b.UsableHosts(context.Background())
+	tt.OK(err)
+	if len(h) != n-1 {
+		t.Fatal("unexpected number of hosts", len(h))
+	}
+
 	// upload some data - should fail
 	tt.FailAll(w.UploadObject(context.Background(), bytes.NewReader(data), testBucket, path, api.UploadObjectOptions{}))
 
 	// update all host settings so they're gouging
 	for _, h := range cluster.hosts {
 		settings := h.settings.Settings()
-		settings.StoragePrice = types.NewCurrency64(101)
+		settings.StoragePrice = settings.StoragePrice.Mul64(2)
 		if err := h.UpdateSettings(settings); err != nil {
 			t.Fatal(err)
 		}

internal/test/host.go

@@ -46,12 +46,15 @@ func NewHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSet
 
 func NewHostSettings() rhpv2.HostSettings {
 	return rhpv2.HostSettings{
-		AcceptingContracts: true,
-		Collateral:         types.Siacoins(1).Div64(1 << 40),
-		MaxCollateral:      types.Siacoins(10000),
-		MaxDuration:        144 * 7 * 12, // 12w
-		Version:            "1.5.10",
-		RemainingStorage:   1 << 42, // 4 TiB
+		AcceptingContracts:         true,
+		Collateral:                 types.Siacoins(1).Div64(1 << 40),
+		EphemeralAccountExpiry:     time.Hour * 24,
+		MaxCollateral:              types.Siacoins(10000),
+		MaxDuration:                144 * 7 * 12, // 12w
+		MaxEphemeralAccountBalance: types.Siacoins(1000),
+		Version:                    "1.5.10",
+		RemainingStorage:           1 << 42, // 4 TiB
+		SiaMuxPort:                 "9983",
 	}
 }
 

stores/hostdb.go

@@ -116,3 +116,11 @@ func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api
 		return tx.RecordPriceTables(ctx, priceTableUpdate)
 	})
 }
+
+func (s *SQLStore) UsableHosts(ctx context.Context) (hosts []sql.HostInfo, err error) {
+	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
+		hosts, err = tx.UsableHosts(ctx)
+		return err
+	})
+	return
+}

stores/hostdb_test.go

@@ -14,6 +14,8 @@ import (
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/gouging"
+	"go.sia.tech/renterd/internal/test"
 	sql "go.sia.tech/renterd/stores/sql"
 )
 
@@ -446,6 +448,155 @@ func TestHosts(t *testing.T) {
 	}
 }
 
+func TestUsableHosts(t *testing.T) {
+	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
+	defer ss.Close()
+	ctx := context.Background()
+
+	// add autopilot
+	err := ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: api.DefaultAutopilotID})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// prepare hosts & contracts
+	//
+	// h1: usable
+	// h2: not usable - blocked
+	// h3: not usable - no host check
+	// h4: not usable - no contract
+	var hks []types.PublicKey
+	for i := 1; i <= 4; i++ {
+		// add host
+		hk := types.PublicKey{byte(i)}
+		addr := fmt.Sprintf("foo.com:100%d", i)
+		err := ss.addCustomTestHost(hk, addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		hks = append(hks, hk)
+
+		// add host scan
+		hs := test.NewHostSettings()
+		pt := test.NewHostPriceTable()
+		s1 := newTestScan(hk, time.Now(), hs, pt, true, nil, nil)
+		if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {
+			t.Fatal(err)
+		}
+
+		// add host check
+		if i != 3 {
+			hc := newTestHostCheck()
+			err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, hk, hc)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// add contract
+		if i != 4 {
+			_, err = ss.addTestContract(types.FileContractID{byte(i)}, hk)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// block host
+		if i == 2 {
+			err = ss.UpdateHostBlocklistEntries(context.Background(), []string{addr}, nil, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	// assert h1 is usable
+	hosts, err := ss.UsableHosts(ctx)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 1 {
+		t.Fatal("unexpected", len(hosts))
+	} else if hosts[0].PublicKey != hks[0] {
+		t.Fatal("unexpected", hosts)
+	} else if hosts[0].SiamuxAddr != "foo.com:9983" {
+		t.Fatal("unexpected", hosts)
+	}
+
+	// create gouging checker
+	gs := test.GougingSettings
+	cs := api.ConsensusState{Synced: true}
+	gc := gouging.NewChecker(gs, cs, nil, nil)
+
+	// assert h1 is not gouging
+	h1 := hosts[0]
+	if gc.Check(&h1.HS, &h1.PT).Gouging() {
+		t.Fatal("unexpected")
+	}
+
+	// record a scan for h1 to make it gouging
+	hs := test.NewHostSettings()
+	pt := test.NewHostPriceTable()
+	pt.UploadBandwidthCost = gs.MaxUploadPrice
+	s1 := newTestScan(h1.PublicKey, time.Now(), hs, pt, true, nil, nil)
+	if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil {
+		t.Fatal(err)
+	}
+
+	// fetch it again
+	hosts, err = ss.UsableHosts(ctx)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 1 {
+		t.Fatal("unexpected", len(hosts))
+	}
+
+	// assert h1 is now gouging
+	h1 = hosts[0]
+	if !gc.Check(&h1.HS, &h1.PT).Gouging() {
+		t.Fatal("unexpected")
+	}
+
+	// create helper to assert number of usable hosts
+	assertNumUsableHosts := func(n int) {
+		t.Helper()
+		hosts, err = ss.UsableHosts(ctx)
+		if err != nil {
+			t.Fatal(err)
+		} else if len(hosts) != n {
+			t.Fatal("unexpected", len(hosts))
+		}
+	}
+
+	// unblock h2
+	if err := ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true); err != nil {
+		t.Fatal(err)
+	}
+
+	assertNumUsableHosts(2)
+
+	// add host check for h3
+	hc := newTestHostCheck()
+	err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, types.PublicKey{3}, hc)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertNumUsableHosts(3)
+
+	// add contract for h4
+	_, err = ss.addTestContract(types.FileContractID{byte(4)}, types.PublicKey{4})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertNumUsableHosts(4)
+
+	// add an allowlist
+	ss.UpdateHostAllowlistEntries(context.Background(), []types.PublicKey{{9}}, nil, false)
+
+	assertNumUsableHosts(0)
+}
+
 // TestRecordScan is a test for recording scans.
 func TestRecordScan(t *testing.T) {
 	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)

stores/sql/database.go

@@ -379,6 +379,11 @@ type (
 		// present in the database.
 		UpsertContractSectors(ctx context.Context, contractSectors []ContractSector) error
 
+		// UsableHosts returns a list of hosts that are ready to be used. That
+		// means they are deemed usable by the autopilot, they are not gouging,
+		// not blocked, not offline, etc.
+		UsableHosts(ctx context.Context) ([]HostInfo, error)
+
 		// WalletEvents returns all wallet events in the database.
 		WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error)