diff --git a/api/host.go b/api/host.go index cfa9a0b6c..3ca2cfe3f 100644 --- a/api/host.go +++ b/api/host.go @@ -116,6 +116,11 @@ type ( Subnets []string `json:"subnets"` } + HostInfo struct { + PublicKey types.PublicKey `json:"publicKey"` + SiamuxAddr string `json:"siamuxAddr"` + } + HostInteractions struct { TotalScans uint64 `json:"totalScans"` LastScan time.Time `json:"lastScan"` diff --git a/bus/bus.go b/bus/bus.go index 5b5e43939..ac47aa62d 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -214,6 +214,7 @@ type ( UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error + UsableHosts(ctx context.Context) ([]sql.HostInfo, error) } // A MetadataStore stores information about contracts and objects. @@ -438,6 +439,7 @@ func (b *Bus) Handler() http.Handler { "GET /contract/:id/roots": b.contractIDRootsHandlerGET, "GET /contract/:id/size": b.contractSizeHandlerGET, + "GET /hosts": b.hostsHandlerGET, "POST /hosts": b.hostsHandlerPOST, "GET /hosts/allowlist": b.hostsAllowlistHandlerGET, "PUT /hosts/allowlist": b.hostsAllowlistHandlerPUT, diff --git a/bus/client/hosts.go b/bus/client/hosts.go index f100f6723..344e7a660 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -75,3 +75,11 @@ func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKe err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck) return } + +// UsableHosts returns a list of hosts that are ready to be used. That means +// they are deemed usable by the autopilot, they are not gouging, not blocked, +// not offline, etc. +func (c *Client) UsableHosts(ctx context.Context) (hosts []api.HostInfo, err error) { + err = c.c.WithContext(ctx).GET("/hosts", &hosts) + return +} diff --git a/bus/routes.go b/bus/routes.go index f5e35c19e..024558b09 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -484,6 +484,27 @@ func (b *Bus) walletPendingHandler(jc jape.Context) { jc.Encode(events) } +func (b *Bus) hostsHandlerGET(jc jape.Context) { + hosts, err := b.store.UsableHosts(jc.Request.Context()) + if jc.Check("couldn't fetch hosts", err) != nil { + return + } + + gp, err := b.gougingParams(jc.Request.Context()) + if jc.Check("could not get gouging parameters", err) != nil { + return + } + gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState, nil, nil) + + var infos []api.HostInfo + for _, h := range hosts { + if !gc.Check(&h.HS, &h.PT).Gouging() { + infos = append(infos, h.HostInfo) + } + } + jc.Encode(infos) +} + func (b *Bus) hostsHandlerPOST(jc jape.Context) { var req api.HostsRequest if jc.Decode(&req) != nil { diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go index 54ea827a4..3ebdd672e 100644 --- a/internal/test/e2e/gouging_test.go +++ b/internal/test/e2e/gouging_test.go @@ -10,7 +10,6 @@ import ( "time" rhpv2 "go.sia.tech/core/rhp/v2" - "go.sia.tech/core/types" "go.sia.tech/renterd/api" "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" @@ -30,9 +29,17 @@ func TestGouging(t *testing.T) { cluster.MineBlocks(cfg.Period + 1) // add hosts - tt.OKAll(cluster.AddHostsBlocking(int(test.AutopilotConfig.Contracts.Amount))) + n := int(test.AutopilotConfig.Contracts.Amount) + tt.OKAll(cluster.AddHostsBlocking(n)) cluster.WaitForAccounts() + // assert all hosts are usable + h, err := b.UsableHosts(context.Background()) + tt.OK(err) + if len(h) != n { + t.Fatal("unexpected number of hosts") + } + // assert that the current period is greater than the period tt.Retry(10, time.Second, func() error { if ap, _ := b.Autopilot(context.Background(), api.DefaultAutopilotID); ap.CurrentPeriod <= cfg.Period { @@ -62,35 +69,39 @@ func TestGouging(t *testing.T) { t.Fatal("unexpected data") } - // update the gouging settings to limit the max storage price to 100H + // fetch current host settings + settings := cluster.hosts[0].settings.Settings() + + // update host settings + updated := settings + updated.StoragePrice = updated.StoragePrice.Mul64(2) + tt.OK(cluster.hosts[0].UpdateSettings(updated)) + + // update gouging settings gs := test.GougingSettings - gs.MaxStoragePrice = types.NewCurrency64(100) + gs.MaxStoragePrice = settings.StoragePrice if err := b.UpdateGougingSettings(context.Background(), gs); err != nil { t.Fatal(err) } - // fetch current contract set - contracts, err := b.Contracts(context.Background(), api.ContractsOpts{ContractSet: cfg.Set}) - tt.OK(err) - - // update one host's settings so it's gouging - hk := contracts[0].HostKey - host := hostsMap[hk.String()] - settings := host.settings.Settings() - settings.StoragePrice = types.NewCurrency64(101) // gouging - tt.OK(host.UpdateSettings(settings)) - // make sure the price table expires so the worker is forced to fetch it // again, this is necessary for the host to be considered price gouging time.Sleep(defaultHostSettings.PriceTableValidity) + // assert all but one host are usable + h, err = b.UsableHosts(context.Background()) + tt.OK(err) + if len(h) != n-1 { + t.Fatal("unexpected number of hosts", len(h)) + } + // upload some data - should fail tt.FailAll(w.UploadObject(context.Background(), bytes.NewReader(data), testBucket, path, api.UploadObjectOptions{})) // update all host settings so they're gouging for _, h := range cluster.hosts { settings := h.settings.Settings() - settings.StoragePrice = types.NewCurrency64(101) + settings.StoragePrice = settings.StoragePrice.Mul64(2) if err := h.UpdateSettings(settings); err != nil { t.Fatal(err) } diff --git a/internal/test/host.go b/internal/test/host.go index f5128b8fc..ffde300f4 100644 --- a/internal/test/host.go +++ b/internal/test/host.go @@ -46,12 +46,15 @@ func NewHost(hk types.PublicKey, pt rhpv3.HostPriceTable, settings rhpv2.HostSet func NewHostSettings() rhpv2.HostSettings { return rhpv2.HostSettings{ - AcceptingContracts: true, - Collateral: types.Siacoins(1).Div64(1 << 40), - MaxCollateral: types.Siacoins(10000), - MaxDuration: 144 * 7 * 12, // 12w - Version: "1.5.10", - RemainingStorage: 1 << 42, // 4 TiB + AcceptingContracts: true, + Collateral: types.Siacoins(1).Div64(1 << 40), + EphemeralAccountExpiry: time.Hour * 24, + MaxCollateral: types.Siacoins(10000), + MaxDuration: 144 * 7 * 12, // 12w + MaxEphemeralAccountBalance: types.Siacoins(1000), + Version: "1.5.10", + RemainingStorage: 1 << 42, // 4 TiB + SiaMuxPort: "9983", } } diff --git a/stores/hostdb.go b/stores/hostdb.go index a80df54dd..b1da3c11b 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -116,3 +116,11 @@ func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api return tx.RecordPriceTables(ctx, priceTableUpdate) }) } + +func (s *SQLStore) UsableHosts(ctx context.Context) (hosts []sql.HostInfo, err error) { + err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + hosts, err = tx.UsableHosts(ctx) + return err + }) + return +} diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 155bb2c0a..2e95e78df 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -14,6 +14,8 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/internal/gouging" + "go.sia.tech/renterd/internal/test" sql "go.sia.tech/renterd/stores/sql" ) @@ -446,6 +448,155 @@ func TestHosts(t *testing.T) { } } +func TestUsableHosts(t *testing.T) { + ss := newTestSQLStore(t, defaultTestSQLStoreConfig) + defer ss.Close() + ctx := context.Background() + + // add autopilot + err := ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: api.DefaultAutopilotID}) + if err != nil { + t.Fatal(err) + } + + // prepare hosts & contracts + // + // h1: usable + // h2: not usable - blocked + // h3: not usable - no host check + // h4: not usable - no contract + var hks []types.PublicKey + for i := 1; i <= 4; i++ { + // add host + hk := types.PublicKey{byte(i)} + addr := fmt.Sprintf("foo.com:100%d", i) + err := ss.addCustomTestHost(hk, addr) + if err != nil { + t.Fatal(err) + } + hks = append(hks, hk) + + // add host scan + hs := test.NewHostSettings() + pt := test.NewHostPriceTable() + s1 := newTestScan(hk, time.Now(), hs, pt, true, nil, nil) + if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil { + t.Fatal(err) + } + + // add host check + if i != 3 { + hc := newTestHostCheck() + err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, hk, hc) + if err != nil { + t.Fatal(err) + } + } + + // add contract + if i != 4 { + _, err = ss.addTestContract(types.FileContractID{byte(i)}, hk) + if err != nil { + t.Fatal(err) + } + } + + // block host + if i == 2 { + err = ss.UpdateHostBlocklistEntries(context.Background(), []string{addr}, nil, false) + if err != nil { + t.Fatal(err) + } + } + } + + // assert h1 is usable + hosts, err := ss.UsableHosts(ctx) + if err != nil { + t.Fatal(err) + } else if len(hosts) != 1 { + t.Fatal("unexpected", len(hosts)) + } else if hosts[0].PublicKey != hks[0] { + t.Fatal("unexpected", hosts) + } else if hosts[0].SiamuxAddr != "foo.com:9983" { + t.Fatal("unexpected", hosts) + } + + // create gouging checker + gs := test.GougingSettings + cs := api.ConsensusState{Synced: true} + gc := gouging.NewChecker(gs, cs, nil, nil) + + // assert h1 is not gouging + h1 := hosts[0] + if gc.Check(&h1.HS, &h1.PT).Gouging() { + t.Fatal("unexpected") + } + + // record a scan for h1 to make it gouging + hs := test.NewHostSettings() + pt := test.NewHostPriceTable() + pt.UploadBandwidthCost = gs.MaxUploadPrice + s1 := newTestScan(h1.PublicKey, time.Now(), hs, pt, true, nil, nil) + if err := ss.RecordHostScans(context.Background(), []api.HostScan{s1}); err != nil { + t.Fatal(err) + } + + // fetch it again + hosts, err = ss.UsableHosts(ctx) + if err != nil { + t.Fatal(err) + } else if len(hosts) != 1 { + t.Fatal("unexpected", len(hosts)) + } + + // assert h1 is now gouging + h1 = hosts[0] + if !gc.Check(&h1.HS, &h1.PT).Gouging() { + t.Fatal("unexpected") + } + + // create helper to assert number of usable hosts + assertNumUsableHosts := func(n int) { + t.Helper() + hosts, err = ss.UsableHosts(ctx) + if err != nil { + t.Fatal(err) + } else if len(hosts) != n { + t.Fatal("unexpected", len(hosts)) + } + } + + // unblock h2 + if err := ss.UpdateHostBlocklistEntries(context.Background(), nil, nil, true); err != nil { + t.Fatal(err) + } + + assertNumUsableHosts(2) + + // add host check for h3 + hc := newTestHostCheck() + err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, types.PublicKey{3}, hc) + if err != nil { + t.Fatal(err) + } + + assertNumUsableHosts(3) + + // add contract for h4 + _, err = ss.addTestContract(types.FileContractID{byte(4)}, types.PublicKey{4}) + if err != nil { + t.Fatal(err) + } + + assertNumUsableHosts(4) + + // add an allowlist + ss.UpdateHostAllowlistEntries(context.Background(), []types.PublicKey{{9}}, nil, false) + + assertNumUsableHosts(0) +} + // TestRecordScan is a test for recording scans. func TestRecordScan(t *testing.T) { ss := newTestSQLStore(t, defaultTestSQLStoreConfig) diff --git a/stores/sql/database.go b/stores/sql/database.go index 0c49a16c0..8c543855f 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -379,6 +379,11 @@ type ( // present in the database. UpsertContractSectors(ctx context.Context, contractSectors []ContractSector) error + // UsableHosts returns a list of hosts that are ready to be used. That + // means they are deemed usable by the autopilot, they are not gouging, + // not blocked, not offline, etc. + UsableHosts(ctx context.Context) ([]HostInfo, error) + // WalletEvents returns all wallet events in the database. WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) diff --git a/stores/sql/main.go b/stores/sql/main.go index 30c8ab831..287920942 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -17,6 +17,7 @@ import ( dsql "database/sql" rhpv2 "go.sia.tech/core/rhp/v2" + rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/coreutils/syncer" "go.sia.tech/coreutils/wallet" @@ -36,6 +37,12 @@ var ( // helper types type ( + HostInfo struct { + api.HostInfo + HS rhpv2.HostSettings + PT rhpv3.HostPriceTable + } + multipartUpload struct { ID int64 Key string @@ -2312,6 +2319,94 @@ func UnspentSiacoinElements(ctx context.Context, tx sql.Tx) (elements []types.Si return } +func UsableHosts(ctx context.Context, tx sql.Tx) ([]HostInfo, error) { + // only include allowed hosts + var whereExprs []string + var hasAllowlist bool + if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_allowlist_entries)").Scan(&hasAllowlist); err != nil { + return nil, fmt.Errorf("failed to check for allowlist: %w", err) + } else if hasAllowlist { + whereExprs = append(whereExprs, "EXISTS (SELECT 1 FROM host_allowlist_entry_hosts hbeh WHERE hbeh.db_host_id = h.id)") + } + + // exclude blocked hosts + var hasBlocklist bool + if err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM host_blocklist_entries)").Scan(&hasBlocklist); err != nil { + return nil, fmt.Errorf("failed to check for blocklist: %w", err) + } else if hasBlocklist { + whereExprs = append(whereExprs, "NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = h.id)") + } + + // fetch autopilot + var autopilotID int64 + if err := tx.QueryRow(ctx, "SELECT id FROM autopilots WHERE identifier = ?", api.DefaultAutopilotID).Scan(&autopilotID); err != nil { + return nil, fmt.Errorf("failed to fetch autopilot id: %w", err) + } + + // only include usable hosts + whereExprs = append(whereExprs, ` +EXISTS ( + SELECT 1 + FROM hosts h2 + INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND hc.db_autopilot_id = ? AND h2.id = h.id + WHERE + hc.usability_blocked = 0 AND + hc.usability_offline = 0 AND + hc.usability_low_score = 0 AND + hc.usability_redundant_ip = 0 AND + hc.usability_gouging = 0 AND + hc.usability_not_accepting_contracts = 0 AND + hc.usability_not_announced = 0 AND + hc.usability_not_completing_scan = 0 +)`) + + // query hosts + rows, err := tx.Query(ctx, fmt.Sprintf(` + SELECT + h.public_key, + COALESCE(h.net_address, ""), + COALESCE(h.settings->>'$.siamuxport', "") AS siamux_port, + h.price_table, + h.settings + FROM hosts h + INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL + INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ? + WHERE %s + GROUP by h.id`, strings.Join(whereExprs, "AND")), autopilotID, autopilotID) + if err != nil { + return nil, fmt.Errorf("failed to fetch hosts: %w", err) + } + defer rows.Close() + + var hosts []HostInfo + for rows.Next() { + var hk PublicKey + var addr, port string + var pt PriceTable + var hs HostSettings + err := rows.Scan(&hk, &addr, &port, &pt, &hs) + if err != nil { + return nil, fmt.Errorf("failed to scan host: %w", err) + } + + // exclude hosts with invalid address + host, _, err := net.SplitHostPort(addr) + if err != nil || host == "" { + continue + } + + hosts = append(hosts, HostInfo{ + api.HostInfo{ + PublicKey: types.PublicKey(hk), + SiamuxAddr: net.JoinHostPort(host, port), + }, + rhpv2.HostSettings(hs), + rhpv3.HostPriceTable(pt), + }) + } + return hosts, nil +} + func WalletEvents(ctx context.Context, tx sql.Tx, offset, limit int) (events []wallet.Event, _ error) { if limit == 0 || limit == -1 { limit = math.MaxInt64 diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index fb3580564..b5dec9646 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -1255,6 +1255,10 @@ func (tx *MainDatabaseTx) UpsertContractSectors(ctx context.Context, contractSec return nil } +func (tx *MainDatabaseTx) UsableHosts(ctx context.Context) ([]ssql.HostInfo, error) { + return ssql.UsableHosts(ctx, tx) +} + func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) { return ssql.WalletEvents(ctx, tx.Tx, offset, limit) } diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index b4b04c376..44215e18b 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -1259,6 +1259,10 @@ func (tx *MainDatabaseTx) UpsertContractSectors(ctx context.Context, contractSec return nil } +func (tx *MainDatabaseTx) UsableHosts(ctx context.Context) ([]ssql.HostInfo, error) { + return ssql.UsableHosts(ctx, tx) +} + func (tx *MainDatabaseTx) WalletEvents(ctx context.Context, offset, limit int) ([]wallet.Event, error) { return ssql.WalletEvents(ctx, tx.Tx, offset, limit) }