Add usable hosts endpoint

api/host.go

@@ -102,6 +102,11 @@ type (
 		MaxLastScan     TimeRFC3339
 		Offset          int
 	}
+
+	UsableHostOptions struct {
+		Limit  int
+		Offset int
+	}
 )
 
 type (
@@ -121,6 +126,11 @@ type (
 		Subnets           []string             `json:"subnets"`
 	}
 
+	HostInfo struct {
+		PublicKey  types.PublicKey `json:"publicKey"`
+		SiamuxAddr string          `json:"siamuxAddr"`
+	}
+
 	HostInteractions struct {
 		TotalScans              uint64        `json:"totalScans"`
 		LastScan                time.Time     `json:"lastScan"`

bus/bus.go

@@ -212,6 +212,7 @@ type (
 		UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error
 		UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error
 		UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error
+		UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error)
 	}
 
 	// A MetadataStore stores information about contracts and objects.
@@ -432,6 +433,7 @@ func (b *Bus) Handler() http.Handler {
 		"GET    /contract/:id/roots":     b.contractIDRootsHandlerGET,
 		"GET    /contract/:id/size":      b.contractSizeHandlerGET,
 
+		"GET    /hosts":                          b.hostsHandlerGET,
 		"POST   /hosts":                          b.hostsHandlerPOST,
 		"GET    /hosts/allowlist":                b.hostsAllowlistHandlerGET,
 		"PUT    /hosts/allowlist":                b.hostsAllowlistHandlerPUT,

bus/client/hosts.go

@@ -3,6 +3,7 @@ package client
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"time"
 
 	"go.sia.tech/core/types"
@@ -91,3 +92,17 @@ func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKe
 	err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck)
 	return
 }
+
+// UsableHosts returns a list of hosts that are ready to be used. That means
+// they are deemed usable by the autopilot, they are not gouging, not blocked,
+// not offline, etc.
+func (c *Client) UsableHosts(ctx context.Context, opts api.UsableHostOptions) (hosts []api.HostInfo, err error) {
+	values := url.Values{}
+	values.Set("offset", fmt.Sprint(opts.Offset))
+	if opts.Limit != 0 {
+		values.Set("limit", fmt.Sprint(opts.Limit))
+	}
+
+	err = c.c.WithContext(ctx).GET("/hosts?"+values.Encode(), &hosts)
+	return
+}

bus/routes.go

@@ -484,6 +484,30 @@ func (b *Bus) walletPendingHandler(jc jape.Context) {
 	jc.Encode(events)
 }
 
+func (b *Bus) hostsHandlerGET(jc jape.Context) {
+	var offset int
+	if jc.DecodeForm("offset", &offset) != nil {
+		return
+	} else if offset < 0 {
+		jc.Error(api.ErrInvalidOffset, http.StatusBadRequest)
+		return
+	}
+
+	limit := -1
+	if jc.DecodeForm("limit", &limit) != nil {
+		return
+	} else if limit < -1 {
+		jc.Error(api.ErrInvalidLimit, http.StatusBadRequest)
+		return
+	}
+
+	hosts, err := b.store.UsableHosts(jc.Request.Context(), offset, limit)
+	if jc.Check("couldn't fetch hosts", err) != nil {
+		return
+	}
+	jc.Encode(hosts)
+}
+
 func (b *Bus) hostsHandlerPOST(jc jape.Context) {
 	var req api.HostsRequest
 	if jc.Decode(&req) != nil {

stores/hostdb.go

@@ -116,3 +116,11 @@ func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api
 		return tx.RecordPriceTables(ctx, priceTableUpdate)
 	})
 }
+
+func (s *SQLStore) UsableHosts(ctx context.Context, offset, limit int) (hosts []api.HostInfo, err error) {
+	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
+		hosts, err = tx.UsableHosts(ctx, offset, limit)
+		return err
+	})
+	return
+}

stores/hostdb_test.go

@@ -446,6 +446,99 @@ func TestHosts(t *testing.T) {
 	}
 }
 
+func TestUsableHosts(t *testing.T) {
+	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
+	defer ss.Close()
+	ctx := context.Background()
+
+	// add autopilot
+	err := ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: api.DefaultAutopilotID})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// prepare hosts & contracts
+	//
+	// h1: usable
+	// h2: usable - best one
+	// h3: not usable - blocked
+	// h4: not usable - no host check
+	// h5: not usable - no contract
+	var hks []types.PublicKey
+	for i := 1; i <= 5; i++ {
+		// add host
+		hk := types.PublicKey{byte(i)}
+		addr := fmt.Sprintf("foo.com:100%d", i)
+		err := ss.addCustomTestHost(hk, addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		hks = append(hks, hk)
+
+		// add host check
+		if i != 4 {
+			hc := newTestHostCheck()
+			if i == 2 {
+				hc.ScoreBreakdown.Age = .2
+			}
+			err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, hk, hc)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// add contract
+		if i != 5 {
+			_, err = ss.addTestContract(types.FileContractID{byte(i)}, hk)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		// block host
+		if i == 3 {
+			err = ss.UpdateHostBlocklistEntries(context.Background(), []string{addr}, nil, false)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	// set siamux port in settings
+	_, err = ss.DB().Exec(context.Background(), "UPDATE hosts SET settings = ? WHERE 1=1", sql.HostSettings(rhpv2.HostSettings{SiaMuxPort: "9983"}))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// assert h1 and h2 are usable and ordered by score
+	hosts, err := ss.UsableHosts(ctx, 0, -1)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 2 {
+		t.Fatal("unexpected", len(hosts))
+	} else if hosts[0].PublicKey != hks[1] || hosts[1].PublicKey != hks[0] {
+		t.Fatal("unexpected", hosts)
+	} else if hosts[0].SiamuxAddr != "foo.com:9983" || hosts[1].SiamuxAddr != "foo.com:9983" {
+		t.Fatal("unexpected", hosts)
+	}
+
+	// assert offset and limit
+	hosts, err = ss.UsableHosts(ctx, 1, 1)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 1 {
+		t.Fatal("unexpected", len(hosts))
+	} else if hosts[0].PublicKey != hks[0] {
+		t.Fatal("unexpected", hosts)
+	}
+	hosts, err = ss.UsableHosts(ctx, 2, 1)
+	if err != nil {
+		t.Fatal(err)
+	} else if len(hosts) != 0 {
+		t.Fatal("unexpected", len(hosts))
+	}
+}
+
 // TestRecordScan is a test for recording scans.
 func TestRecordScan(t *testing.T) {
 	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)

stores/sql/database.go

@@ -367,6 +367,11 @@ type (
 		// the health of the updated slabs becomes invalid
 		UpdateSlabHealth(ctx context.Context, limit int64, minValidity, maxValidity time.Duration) (int64, error)
 
+		// UsableHosts returns a list of hosts that are ready to be used. That
+		// means they are deemed usable by the autopilot, they are not gouging,
+		// not blocked, not offline, etc.
+		UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error)
+
 		// WalletEvents returns all wallet events in the database.
 		WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error)
 

stores/sql/main.go

@@ -2159,6 +2159,92 @@ func UnspentSiacoinElements(ctx context.Context, tx sql.Tx) (elements []types.Si
 	return
 }
 
+func UsableHosts(ctx context.Context, tx sql.Tx, offset, limit int) ([]api.HostInfo, error) {
+	// handle input parameters
+	if offset < 0 {
+		return nil, ErrNegativeOffset
+	} else if limit == 0 || limit == -1 {
+		limit = math.MaxInt64
+	}
+
+	// only include allowed hosts
+	var whereExprs []string
+	var hasAllowlist bool
+	if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_allowlist_entries)").Scan(&hasAllowlist); err != nil {
+		return nil, fmt.Errorf("failed to check for allowlist: %w", err)
+	} else if hasAllowlist {
+		whereExprs = append(whereExprs, "EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = h.id)")
+	}
+
+	// exclude blocked hosts
+	var hasBlocklist bool
+	if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_blocklist_entries)").Scan(&hasBlocklist); err != nil {
+		return nil, fmt.Errorf("failed to check for blocklist: %w", err)
+	} else if hasBlocklist {
+		whereExprs = append(whereExprs, "NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = h.id)")
+	}
+
+	// fetch autopilot
+	var autopilotID int64
+	if err := tx.QueryRow(ctx, "SELECT id FROM autopilots WHERE identifier = ?", api.DefaultAutopilotID).Scan(&autopilotID); err != nil {
+		return nil, fmt.Errorf("failed to fetch autopilot id: %w", err)
+	}
+
+	// only include usable hosts
+	whereExprs = append(whereExprs, `
+EXISTS (
+	SELECT 1
+	FROM hosts h2
+	INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND hc.db_autopilot_id = ? AND h2.id = h.id
+	WHERE
+		hc.usability_blocked = 0 AND
+		hc.usability_offline = 0 AND
+		hc.usability_low_score = 0 AND
+		hc.usability_redundant_ip = 0 AND
+		hc.usability_gouging = 0 AND
+		hc.usability_not_accepting_contracts = 0 AND
+		hc.usability_not_announced = 0 AND
+		hc.usability_not_completing_scan = 0
+)`)
+
+	// query hosts
+	rows, err := tx.Query(ctx, fmt.Sprintf(`
+	SELECT
+	h.public_key,
+	COALESCE(h.net_address, ""),
+	COALESCE(h.settings->>'$.siamuxport', "") AS siamux_port
+	FROM hosts h
+	INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL
+	INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ?
+	WHERE %s
+	GROUP by h.id
+	ORDER BY MAX(hc.score_age) * MAX(hc.score_collateral) * MAX(hc.score_interactions) * MAX(hc.score_storage_remaining) * MAX(hc.score_uptime) * MAX(hc.score_version) * MAX(hc.score_prices) DESC
+	LIMIT ? OFFSET ?`, strings.Join(whereExprs, "AND")), autopilotID, autopilotID, limit, offset)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch hosts: %w", err)
+	}
+	defer rows.Close()
+
+	var hosts []api.HostInfo
+	for rows.Next() {
+		var hk PublicKey
+		var addr, port string
+		err := rows.Scan(&hk, &addr, &port)
+		if err != nil {
+			return nil, fmt.Errorf("failed to scan host: %w", err)
+		}
+		host, _, err := net.SplitHostPort(addr)
+		if err != nil || host == "" {
+			continue
+		}
+		hosts = append(hosts, api.HostInfo{
+			PublicKey:  types.PublicKey(hk),
+			SiamuxAddr: net.JoinHostPort(host, port),
+		})
+	}
+	return hosts, nil
+}
+
 func WalletEvents(ctx context.Context, tx sql.Tx, offset, limit int) (events []wallet.Event, _ error) {
 	if limit == 0 || limit == -1 {
 		limit = math.MaxInt64

stores/sql/mysql/main.go

@@ -1242,6 +1242,10 @@ func (tx *MainDatabaseTx) UpdateSlabHealth(ctx context.Context, limit int64, min
 	return res.RowsAffected()
 }
 
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx, offset, limit)
+}
+
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {
 	return ssql.WalletEvents(ctx, tx.Tx, offset, limit)
 }

stores/sql/sqlite/main.go

@@ -1242,6 +1242,10 @@ func (tx *MainDatabaseTx) UpdateSlabHealth(ctx context.Context, limit int64, min
 	return res.RowsAffected()
 }
 
+func (tx *MainDatabaseTx) UsableHosts(ctx context.Context, offset, limit int) ([]api.HostInfo, error) {
+	return ssql.UsableHosts(ctx, tx, offset, limit)
+}
+
 func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) {
 	return ssql.WalletEvents(ctx, tx.Tx, offset, limit)
 }