From 0e98ce389a34ad9a4986d73e0031440fc4361593 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 11:33:21 +0100 Subject: [PATCH 1/8] sql: remove RecordPriceTables --- bus/bus.go | 1 - stores/hostdb.go | 6 ------ stores/sql/database.go | 4 ---- stores/sql/main.go | 37 ------------------------------------- stores/sql/mysql/main.go | 4 ---- stores/sql/sqlite/main.go | 4 ---- 6 files changed, 56 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index b32799b2d..7e48bc099 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -206,7 +206,6 @@ type ( HostBlocklist(ctx context.Context) ([]string, error) Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) RecordHostScans(ctx context.Context, scans []api.HostScan) error - RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error) ResetLostSectors(ctx context.Context, hk types.PublicKey) error UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error diff --git a/stores/hostdb.go b/stores/hostdb.go index b1da3c11b..88beaa843 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -111,12 +111,6 @@ func (s *SQLStore) RecordHostScans(ctx context.Context, scans []api.HostScan) er }) } -func (s *SQLStore) RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error { - return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - return tx.RecordPriceTables(ctx, priceTableUpdate) - }) -} - func (s *SQLStore) UsableHosts(ctx context.Context) (hosts []sql.HostInfo, err error) { err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { hosts, err = tx.UsableHosts(ctx) diff --git a/stores/sql/database.go b/stores/sql/database.go index 303cc8eed..726183072 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -264,10 +264,6 @@ type ( // therefore only useful for gouging checks. RecordHostScans(ctx context.Context, scans []api.HostScan) error - // RecordPriceTables records price tables for hosts in the database - // increasing the successful/failed interactions accordingly. - RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error - // RemoveContractSet removes the contract set with the given name from // the database. RemoveContractSet(ctx context.Context, contractSet string) error diff --git a/stores/sql/main.go b/stores/sql/main.go index 63883daf0..4a17aeffe 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -1722,43 +1722,6 @@ func RecordHostScans(ctx context.Context, tx sql.Tx, scans []api.HostScan) error return nil } -func RecordPriceTables(ctx context.Context, tx sql.Tx, priceTableUpdates []api.HostPriceTableUpdate) error { - if len(priceTableUpdates) == 0 { - return nil - } - - stmt, err := tx.Prepare(ctx, ` - UPDATE hosts SET - recent_downtime = CASE WHEN ? THEN recent_downtime = 0 ELSE recent_downtime END, - recent_scan_failures = CASE WHEN ? THEN recent_scan_failures = 0 ELSE recent_scan_failures END, - price_table = CASE WHEN ? THEN ? ELSE price_table END, - price_table_expiry = CASE WHEN ? THEN ? ELSE price_table_expiry END, - successful_interactions = CASE WHEN ? THEN successful_interactions + 1 ELSE successful_interactions END, - failed_interactions = CASE WHEN ? THEN failed_interactions + 1 ELSE failed_interactions END - WHERE public_key = ? - `) - if err != nil { - return fmt.Errorf("failed to prepare statement to update host with price table: %w", err) - } - defer stmt.Close() - - for _, ptu := range priceTableUpdates { - _, err := stmt.Exec(ctx, - ptu.Success, // recent_downtime - ptu.Success, // recent_scan_failures - ptu.Success, PriceTable(ptu.PriceTable.HostPriceTable), // price_table - ptu.Success, ptu.PriceTable.Expiry, // price_table_expiry - ptu.Success, // successful_interactions - !ptu.Success, // failed_interactions - PublicKey(ptu.HostKey), - ) - if err != nil { - return fmt.Errorf("failed to update host with price table: %w", err) - } - } - return nil -} - func RemoveContractSet(ctx context.Context, tx sql.Tx, contractSet string) error { _, err := tx.Exec(ctx, "DELETE FROM contract_sets WHERE name = ?", contractSet) if err != nil { diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index b5dec9646..a8fd4a358 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -778,10 +778,6 @@ func (tx *MainDatabaseTx) RecordHostScans(ctx context.Context, scans []api.HostS return ssql.RecordHostScans(ctx, tx, scans) } -func (tx *MainDatabaseTx) RecordPriceTables(ctx context.Context, priceTableUpdates []api.HostPriceTableUpdate) error { - return ssql.RecordPriceTables(ctx, tx, priceTableUpdates) -} - func (tx *MainDatabaseTx) RemoveContractSet(ctx context.Context, contractSet string) error { return ssql.RemoveContractSet(ctx, tx, contractSet) } diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 44215e18b..9be654edd 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -788,10 +788,6 @@ func (tx *MainDatabaseTx) RecordHostScans(ctx context.Context, scans []api.HostS return ssql.RecordHostScans(ctx, tx, scans) } -func (tx *MainDatabaseTx) RecordPriceTables(ctx context.Context, priceTableUpdates []api.HostPriceTableUpdate) error { - return ssql.RecordPriceTables(ctx, tx, priceTableUpdates) -} - func (tx *MainDatabaseTx) RemoveContractSet(ctx context.Context, contractSet string) error { return ssql.RemoveContractSet(ctx, tx, contractSet) } From ad1feaf988bc9be322eb663b40a605ed5d42d86c Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 11:48:20 +0100 Subject: [PATCH 2/8] bus: add v2 settings to Host type --- api/host.go | 3 ++- stores/sql/main.go | 15 +++++++---- stores/sql/mysql/migrations/main/schema.sql | 1 + stores/sql/sqlite/migrations/main/schema.sql | 23 ++++++++++++++++- stores/sql/types.go | 26 ++++++++++++++++++++ 5 files changed, 61 insertions(+), 7 deletions(-) diff --git a/api/host.go b/api/host.go index 059e06c5e..ce24d56f7 100644 --- a/api/host.go +++ b/api/host.go @@ -108,7 +108,8 @@ type ( PublicKey types.PublicKey `json:"publicKey"` NetAddress string `json:"netAddress"` PriceTable HostPriceTable `json:"priceTable"` - Settings rhpv2.HostSettings `json:"settings"` + Settings rhpv2.HostSettings `json:"settings,omitempty"` + V2Settings rhpv4.HostSettings `json:"v2Settings,omitempty"` Interactions HostInteractions `json:"interactions"` Scanned bool `json:"scanned"` Blocked bool `json:"blocked"` diff --git a/stores/sql/main.go b/stores/sql/main.go index 4a17aeffe..2a543a912 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -840,7 +840,7 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er rows, err = tx.Query(ctx, fmt.Sprintf(` SELECT h.id, h.created_at, h.last_announcement, h.public_key, h.net_address, h.price_table, h.price_table_expiry, - h.settings, h.total_scans, h.last_scan, h.last_scan_success, h.second_to_last_scan_success, + h.settings, h.v2_settings, h.total_scans, h.last_scan, h.last_scan_success, h.second_to_last_scan_success, h.uptime, h.downtime, h.successful_interactions, h.failed_interactions, COALESCE(h.lost_sectors, 0), h.scanned, %s FROM hosts h @@ -861,10 +861,13 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er var pte dsql.NullTime err := rows.Scan(&hostID, &h.KnownSince, &h.LastAnnouncement, (*PublicKey)(&h.PublicKey), &h.NetAddress, (*PriceTable)(&h.PriceTable.HostPriceTable), &pte, - (*HostSettings)(&h.Settings), &h.Interactions.TotalScans, (*UnixTimeMS)(&h.Interactions.LastScan), &h.Interactions.LastScanSuccess, - &h.Interactions.SecondToLastScanSuccess, (*DurationMS)(&h.Interactions.Uptime), (*DurationMS)(&h.Interactions.Downtime), - &h.Interactions.SuccessfulInteractions, &h.Interactions.FailedInteractions, &h.Interactions.LostSectors, - &h.Scanned, &h.Blocked, + (*HostSettings)(&h.Settings), (*V2HostSettings)(&h.V2Settings), + &h.Interactions.TotalScans, (*UnixTimeMS)(&h.Interactions.LastScan), + &h.Interactions.LastScanSuccess, &h.Interactions.SecondToLastScanSuccess, + (*DurationMS)(&h.Interactions.Uptime), + (*DurationMS)(&h.Interactions.Downtime), + &h.Interactions.SuccessfulInteractions, &h.Interactions.FailedInteractions, + &h.Interactions.LostSectors, &h.Scanned, &h.Blocked, ) if err != nil { return nil, fmt.Errorf("failed to scan host: %w", err) @@ -1686,6 +1689,7 @@ func RecordHostScans(ctx context.Context, tx sql.Tx, scans []api.HostScan) error uptime = CASE WHEN ? AND last_scan > 0 AND last_scan < ? THEN uptime + ? - last_scan ELSE uptime END, last_scan = ?, settings = CASE WHEN ? THEN ? ELSE settings END, + v2_settings = CASE WHEN ? THEN ? ELSE settings END, price_table = CASE WHEN ? AND (price_table_expiry IS NULL OR ? > price_table_expiry) THEN ? ELSE price_table END, price_table_expiry = CASE WHEN ? AND (price_table_expiry IS NULL OR ? > price_table_expiry) THEN ? ELSE price_table_expiry END, successful_interactions = CASE WHEN ? THEN successful_interactions + 1 ELSE successful_interactions END, @@ -1709,6 +1713,7 @@ func RecordHostScans(ctx context.Context, tx sql.Tx, scans []api.HostScan) error scan.Success, scanTime, scanTime, // uptime scanTime, // last_scan scan.Success, HostSettings(scan.Settings), // settings + scan.Success, V2HostSettings(scan.V2Settings), // settings scan.Success, now, PriceTable(scan.PriceTable), // price_table scan.Success, now, now, // price_table_expiry scan.Success, // successful_interactions diff --git a/stores/sql/mysql/migrations/main/schema.sql b/stores/sql/mysql/migrations/main/schema.sql index a4e9f6341..0f76fd6c3 100644 --- a/stores/sql/mysql/migrations/main/schema.sql +++ b/stores/sql/mysql/migrations/main/schema.sql @@ -43,6 +43,7 @@ CREATE TABLE `hosts` ( `created_at` datetime(3) DEFAULT NULL, `public_key` varbinary(32) NOT NULL, `settings` JSON, + `v2_settings` JSON NOT NULL DEFAULT '{}', `price_table` longtext, `price_table_expiry` datetime(3) DEFAULT NULL, `total_scans` bigint unsigned DEFAULT NULL, diff --git a/stores/sql/sqlite/migrations/main/schema.sql b/stores/sql/sqlite/migrations/main/schema.sql index d8960e56c..609afc865 100644 --- a/stores/sql/sqlite/migrations/main/schema.sql +++ b/stores/sql/sqlite/migrations/main/schema.sql @@ -1,5 +1,26 @@ -- dbHost -CREATE TABLE `hosts` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`public_key` blob NOT NULL UNIQUE,`settings` text,`price_table` text,`price_table_expiry` datetime,`total_scans` integer,`last_scan` integer,`last_scan_success` numeric,`second_to_last_scan_success` numeric,`scanned` numeric,`uptime` integer,`downtime` integer,`recent_downtime` integer,`recent_scan_failures` integer,`successful_interactions` real,`failed_interactions` real,`lost_sectors` integer,`last_announcement` datetime,`net_address` text); +CREATE TABLE `hosts` ( +`id` integer PRIMARY KEY AUTOINCREMENT, +`created_at` datetime, +`public_key` blob NOT NULL UNIQUE, +`settings` text, +`v2_settings` text NOT NULL DEFAULT '{}', +`price_table` text, +`price_table_expiry` datetime, +`total_scans` integer, +`last_scan` integer, +`last_scan_success` numeric, +`second_to_last_scan_success` numeric, +`scanned` numeric, +`uptime` integer, +`downtime` integer, +`recent_downtime` integer, +`recent_scan_failures` integer, +`successful_interactions` real, +`failed_interactions` real, +`lost_sectors` integer, +`last_announcement` datetime, +`net_address` text); CREATE INDEX `idx_hosts_recent_scan_failures` ON `hosts`(`recent_scan_failures`); CREATE INDEX `idx_hosts_recent_downtime` ON `hosts`(`recent_downtime`); CREATE INDEX `idx_hosts_scanned` ON `hosts`(`scanned`); diff --git a/stores/sql/types.go b/stores/sql/types.go index 2643dfcd6..8cf706c5d 100644 --- a/stores/sql/types.go +++ b/stores/sql/types.go @@ -14,6 +14,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" + rhpv4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" rhp4 "go.sia.tech/coreutils/rhp/v4" @@ -49,6 +50,7 @@ type ( DurationMS time.Duration Unsigned64 uint64 ChainProtocol chain.Protocol + V2HostSettings rhpv4.HostSettings StateElement struct { ID Hash256 @@ -79,6 +81,8 @@ var ( _ scannerValuer = (*UnixTimeMS)(nil) _ scannerValuer = (*DurationMS)(nil) _ scannerValuer = (*Unsigned64)(nil) + _ scannerValuer = (*ChainProtocol)(nil) + _ scannerValuer = (*V2HostSettings)(nil) ) // Scan scan value into AutopilotConfig, implements sql.Scanner interface. @@ -545,3 +549,25 @@ func (p ChainProtocol) Value() (driver.Value, error) { return nil, fmt.Errorf("invalid ChainProtocol value: %v", p) } } + +// Scan scan value into V2HostSettings, implements sql.Scanner interface. +func (hs *V2HostSettings) Scan(value interface{}) error { + var bytes []byte + switch value := value.(type) { + case string: + bytes = []byte(value) + case []byte: + bytes = value + default: + return errors.New(fmt.Sprint("failed to unmarshal Settings value:", value)) + } + return json.Unmarshal(bytes, hs) +} + +// Value returns a V2HostSettings value, implements driver.Valuer interface. +func (hs V2HostSettings) Value() (driver.Value, error) { + if hs == (V2HostSettings{}) { + return []byte("{}"), nil + } + return json.Marshal(hs) +} From e2c9ef1faa0195bf0534ba8a3a2b82acb58b8fd5 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 14:06:32 +0100 Subject: [PATCH 3/8] sql: move max duration gouging to usability checks --- api/autopilot.go | 1 + api/host.go | 9 ++- autopilot/contractor/contractor.go | 2 +- autopilot/contractor/evaluate.go | 10 +-- autopilot/contractor/hostfilter.go | 15 ++++- internal/gouging/gouging.go | 65 ++++---------------- internal/test/config.go | 10 +-- stores/hostdb_test.go | 2 +- stores/sql/main.go | 6 +- stores/sql/mysql/main.go | 8 +-- stores/sql/mysql/migrations/main/schema.sql | 2 +- stores/sql/sqlite/main.go | 8 +-- stores/sql/sqlite/migrations/main/schema.sql | 27 +++++++- 13 files changed, 84 insertions(+), 81 deletions(-) diff --git a/api/autopilot.go b/api/autopilot.go index 26dc74a82..631a50065 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -121,6 +121,7 @@ type ( Pruning uint64 `json:"pruning"` Upload uint64 `json:"upload"` } `json:"gouging"` + LowMaxDuration uint64 `json:"lowMaxDuration"` NotAcceptingContracts uint64 `json:"notAcceptingContracts"` NotScanned uint64 `json:"notScanned"` } `json:"unusable"` diff --git a/api/host.go b/api/host.go index ce24d56f7..4a6aee473 100644 --- a/api/host.go +++ b/api/host.go @@ -165,7 +165,6 @@ type ( } HostGougingBreakdown struct { - ContractErr string `json:"contractErr"` DownloadErr string `json:"downloadErr"` GougingErr string `json:"gougingErr"` PruneErr string `json:"pruneErr"` @@ -185,6 +184,7 @@ type ( HostUsabilityBreakdown struct { Blocked bool `json:"blocked"` Offline bool `json:"offline"` + LowMaxDuration bool `json:"lowMaxDuration"` LowScore bool `json:"lowScore"` RedundantIP bool `json:"redundantIP"` Gouging bool `json:"gouging"` @@ -222,6 +222,11 @@ func (h Host) IsOnline() bool { return h.Interactions.LastScanSuccess || h.Interactions.SecondToLastScanSuccess } +func (h Host) IsV2() bool { + // consider a host to be v2 if it has announced a v2 address + return len(h.V2SiamuxAddresses) > 0 +} + func (h Host) V2SiamuxAddr() string { // NOTE: eventually this can be smarter about picking an address but right now // we just prioritize IPv4 over IPv6 @@ -240,7 +245,6 @@ func (sb HostScoreBreakdown) String() string { func (hgb HostGougingBreakdown) Gouging() bool { for _, err := range []string{ - hgb.ContractErr, hgb.DownloadErr, hgb.GougingErr, hgb.PruneErr, @@ -256,7 +260,6 @@ func (hgb HostGougingBreakdown) Gouging() bool { func (hgb HostGougingBreakdown) String() string { var reasons []string for _, errStr := range []string{ - hgb.ContractErr, hgb.DownloadErr, hgb.GougingErr, hgb.PruneErr, diff --git a/autopilot/contractor/contractor.go b/autopilot/contractor/contractor.go index b79ae914f..5d5409947 100644 --- a/autopilot/contractor/contractor.go +++ b/autopilot/contractor/contractor.go @@ -1137,7 +1137,7 @@ func performHostChecks(ctx *mCtx, bus Bus, logger *zap.SugaredLogger) error { } for _, h := range scoredHosts { h.host.PriceTable.HostBlockHeight = cs.BlockHeight // ignore HostBlockHeight - hc := checkHost(ctx.GougingChecker(cs), h, minScore) + hc := checkHost(ctx.GougingChecker(cs), h, minScore, ctx.Period()) if err := bus.UpdateHostCheck(ctx, ctx.ApID(), h.host.PublicKey, *hc); err != nil { return fmt.Errorf("failed to update host check for host %v: %w", h.host.PublicKey, err) } diff --git a/autopilot/contractor/evaluate.go b/autopilot/contractor/evaluate.go index 70ce3ea5c..982de01dd 100644 --- a/autopilot/contractor/evaluate.go +++ b/autopilot/contractor/evaluate.go @@ -13,7 +13,7 @@ var ErrMissingRequiredFields = errors.New("missing required fields in configurat func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, period uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) { gc := gouging.NewChecker(gs, cs, &period, &cfg.Contracts.RenewWindow) for _, host := range hosts { - hc := checkHost(gc, scoreHost(host, cfg, gs, rs.Redundancy()), minValidScore) + hc := checkHost(gc, scoreHost(host, cfg, gs, rs.Redundancy()), minValidScore, period) if hc.UsabilityBreakdown.IsUsable() { usables++ } @@ -36,7 +36,7 @@ func EvaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, rs api.Redun resp.Hosts = uint64(len(hosts)) for i, host := range hosts { hosts[i].PriceTable.HostBlockHeight = cs.BlockHeight // ignore block height - hc := checkHost(gc, scoreHost(host, cfg, gs, rs.Redundancy()), minValidScore) + hc := checkHost(gc, scoreHost(host, cfg, gs, rs.Redundancy()), minValidScore, cfg.Contracts.Period) if hc.UsabilityBreakdown.IsUsable() { resp.Usable++ continue @@ -44,15 +44,15 @@ func EvaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, rs api.Redun if hc.UsabilityBreakdown.Blocked { resp.Unusable.Blocked++ } + if hc.UsabilityBreakdown.LowMaxDuration { + resp.Unusable.LowMaxDuration++ + } if hc.UsabilityBreakdown.NotAcceptingContracts { resp.Unusable.NotAcceptingContracts++ } if hc.UsabilityBreakdown.NotCompletingScan { resp.Unusable.NotScanned++ } - if hc.GougingBreakdown.ContractErr != "" { - resp.Unusable.Gouging.Contract++ - } if hc.GougingBreakdown.DownloadErr != "" { resp.Unusable.Gouging.Download++ } diff --git a/autopilot/contractor/hostfilter.go b/autopilot/contractor/hostfilter.go index 4a3e93b04..bc5d75f43 100644 --- a/autopilot/contractor/hostfilter.go +++ b/autopilot/contractor/hostfilter.go @@ -221,7 +221,7 @@ func isUpForRenewal(cfg api.AutopilotConfig, r api.Revision, blockHeight uint64) } // checkHost performs a series of checks on the host. -func checkHost(gc gouging.Checker, sh scoredHost, minScore float64) *api.HostCheck { +func checkHost(gc gouging.Checker, sh scoredHost, minScore float64, period uint64) *api.HostCheck { h := sh.host // prepare host breakdown fields @@ -256,6 +256,19 @@ func checkHost(gc gouging.Checker, sh scoredHost, minScore float64) *api.HostChe } else if minScore > 0 && !(sh.score > minScore) { ub.LowScore = true } + + // check contract gouging separately + checkContractGouging := func(maxDuration uint64) { + if period > maxDuration { + ub.LowMaxDuration = true + } + } + if h.IsV2() { + checkContractGouging(h.V2Settings.MaxContractDuration) + } else { + checkContractGouging(h.Settings.MaxDuration) + checkContractGouging(h.PriceTable.MaxDuration) + } } return &api.HostCheck{ diff --git a/internal/gouging/gouging.go b/internal/gouging/gouging.go index 3a8d812fe..b788e13a9 100644 --- a/internal/gouging/gouging.go +++ b/internal/gouging/gouging.go @@ -8,6 +8,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" + rhpv4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" "go.sia.tech/renterd/api" ) @@ -40,7 +41,8 @@ type ( } Checker interface { - Check(_ *rhpv2.HostSettings, _ *rhpv3.HostPriceTable) api.HostGougingBreakdown + Check(*rhpv2.HostSettings, *rhpv3.HostPriceTable) api.HostGougingBreakdown + CheckV2(rhpv4.HostSettings) api.HostGougingBreakdown CheckSettings(rhpv2.HostSettings) api.HostGougingBreakdown CheckUnusedDefaults(rhpv3.HostPriceTable) error BlocksUntilBlockHeightGouging(hostHeight uint64) int64 @@ -49,9 +51,6 @@ type ( checker struct { consensusState api.ConsensusState settings api.GougingSettings - - period *uint64 - renewWindow *uint64 } ) @@ -61,9 +60,6 @@ func NewChecker(gs api.GougingSettings, cs api.ConsensusState, period, renewWind return checker{ consensusState: cs, settings: gs, - - period: period, - renewWindow: renewWindow, } } @@ -99,10 +95,6 @@ func (gc checker) Check(hs *rhpv2.HostSettings, pt *rhpv3.HostPriceTable) api.Ho } return api.HostGougingBreakdown{ - ContractErr: errsToStr( - checkContractGougingRHPv2(gc.period, gc.renewWindow, hs), - checkContractGougingRHPv3(gc.period, gc.renewWindow, pt), - ), DownloadErr: errsToStr(checkDownloadGougingRHPv3(gc.settings, pt)), GougingErr: errsToStr( checkPriceGougingPT(gc.settings, gc.consensusState, pt), @@ -113,6 +105,16 @@ func (gc checker) Check(hs *rhpv2.HostSettings, pt *rhpv3.HostPriceTable) api.Ho } } +// TODO: write tests +func (gc checker) CheckV2(hs rhpv4.HostSettings) api.HostGougingBreakdown { + return api.HostGougingBreakdown{ + DownloadErr: "", + GougingErr: "", + PruneErr: "", + UploadErr: "", + } +} + func (gc checker) CheckSettings(hs rhpv2.HostSettings) api.HostGougingBreakdown { return gc.Check(&hs, nil) } @@ -250,47 +252,6 @@ func checkPriceGougingPT(gs api.GougingSettings, cs api.ConsensusState, pt *rhpv return nil } -func checkContractGougingRHPv2(period, renewWindow *uint64, hs *rhpv2.HostSettings) (err error) { - // period and renew window might be nil since we don't always have access to - // these settings when performing gouging checks - if hs == nil || period == nil || renewWindow == nil { - return nil - } - - err = checkContractGouging(*period, *renewWindow, hs.MaxDuration, hs.WindowSize) - if err != nil { - err = fmt.Errorf("%w: %v", ErrHostSettingsGouging, err) - } - return -} - -func checkContractGougingRHPv3(period, renewWindow *uint64, pt *rhpv3.HostPriceTable) (err error) { - // period and renew window might be nil since we don't always have access to - // these settings when performing gouging checks - if pt == nil || period == nil || renewWindow == nil { - return nil - } - err = checkContractGouging(*period, *renewWindow, pt.MaxDuration, pt.WindowSize) - if err != nil { - err = fmt.Errorf("%w: %v", ErrPriceTableGouging, err) - } - return -} - -func checkContractGouging(period, renewWindow, maxDuration, windowSize uint64) error { - // check MaxDuration - if period != 0 && period > maxDuration { - return fmt.Errorf("MaxDuration %v is lower than the period %v", maxDuration, period) - } - - // check WindowSize - if renewWindow != 0 && renewWindow < windowSize { - return fmt.Errorf("minimum WindowSize %v is greater than the renew window %v", windowSize, renewWindow) - } - - return nil -} - func checkPruneGougingRHPv2(gs api.GougingSettings, hs *rhpv2.HostSettings) error { if hs == nil { return nil diff --git a/internal/test/config.go b/internal/test/config.go index 14093cd74..9c6b2b273 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -34,11 +34,11 @@ var ( ContractSet = "testset" GougingSettings = api.GougingSettings{ - MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC - MaxContractPrice: types.Siacoins(10), // 10 SC per contract - MaxDownloadPrice: types.Siacoins(1).Mul64(1000), // 1000 SC per 1 TiB - MaxUploadPrice: types.Siacoins(1).Mul64(1000), // 1000 SC per 1 TiB - MaxStoragePrice: types.Siacoins(1000).Div64(144 * 30), // 1000 SC per month + MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC + MaxContractPrice: types.Siacoins(10), // 10 SC per contract + MaxDownloadPrice: types.Siacoins(1).Mul64(1000).Div64(1e12), // 1000 SC per 1 TB + MaxUploadPrice: types.Siacoins(1).Mul64(1000).Div64(1e12), // 1000 SC per 1 TB + MaxStoragePrice: types.Siacoins(1000).Div64(1e12).Div64(144 * 30), // 1000 SC per TB per month HostBlockHeightLeeway: 240, // amount of leeway given to host block height diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 759ae17ec..17d864737 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -1333,7 +1333,6 @@ func newTestHostCheck() api.HostCheck { return api.HostCheck{ GougingBreakdown: api.HostGougingBreakdown{ - ContractErr: "foo", DownloadErr: "bar", GougingErr: "baz", PruneErr: "qux", @@ -1351,6 +1350,7 @@ func newTestHostCheck() api.HostCheck { UsabilityBreakdown: api.HostUsabilityBreakdown{ Blocked: false, Offline: false, + LowMaxDuration: false, LowScore: false, RedundantIP: false, Gouging: false, diff --git a/stores/sql/main.go b/stores/sql/main.go index 2a543a912..35466e877 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -896,7 +896,7 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er } rows, err = tx.Query(ctx, fmt.Sprintf(` SELECT h.public_key, ap.identifier, hc.usability_blocked, hc.usability_offline, hc.usability_low_score, hc.usability_redundant_ip, - hc.usability_gouging, usability_not_accepting_contracts, hc.usability_not_announced, hc.usability_not_completing_scan, + hc.usability_gouging, hc.usability_low_max_duration, usability_not_accepting_contracts, hc.usability_not_announced, hc.usability_not_completing_scan, hc.score_age, hc.score_collateral, hc.score_interactions, hc.score_storage_remaining, hc.score_uptime, hc.score_version, hc.score_prices, hc.gouging_contract_err, hc.gouging_download_err, hc.gouging_gouging_err, hc.gouging_prune_err, hc.gouging_upload_err @@ -921,9 +921,9 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er var pk PublicKey var hc api.HostCheck err := rows.Scan(&pk, &ap, &hc.UsabilityBreakdown.Blocked, &hc.UsabilityBreakdown.Offline, &hc.UsabilityBreakdown.LowScore, &hc.UsabilityBreakdown.RedundantIP, - &hc.UsabilityBreakdown.Gouging, &hc.UsabilityBreakdown.NotAcceptingContracts, &hc.UsabilityBreakdown.NotAnnounced, &hc.UsabilityBreakdown.NotCompletingScan, + &hc.UsabilityBreakdown.Gouging, &hc.UsabilityBreakdown.LowMaxDuration, &hc.UsabilityBreakdown.NotAcceptingContracts, &hc.UsabilityBreakdown.NotAnnounced, &hc.UsabilityBreakdown.NotCompletingScan, &hc.ScoreBreakdown.Age, &hc.ScoreBreakdown.Collateral, &hc.ScoreBreakdown.Interactions, &hc.ScoreBreakdown.StorageRemaining, &hc.ScoreBreakdown.Uptime, - &hc.ScoreBreakdown.Version, &hc.ScoreBreakdown.Prices, &hc.GougingBreakdown.ContractErr, &hc.GougingBreakdown.DownloadErr, &hc.GougingBreakdown.GougingErr, + &hc.ScoreBreakdown.Version, &hc.ScoreBreakdown.Prices, &hc.GougingBreakdown.DownloadErr, &hc.GougingBreakdown.GougingErr, &hc.GougingBreakdown.PruneErr, &hc.GougingBreakdown.UploadErr) if err != nil { return nil, fmt.Errorf("failed to scan host: %w", err) diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index a8fd4a358..315fb6ea3 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -1148,7 +1148,7 @@ func (tx *MainDatabaseTx) UpdateHostBlocklistEntries(ctx context.Context, add, r func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, hk types.PublicKey, hc api.HostCheck) error { _, err := tx.Exec(ctx, ` INSERT INTO host_checks (created_at, db_autopilot_id, db_host_id, usability_blocked, usability_offline, usability_low_score, - usability_redundant_ip, usability_gouging, usability_not_accepting_contracts, usability_not_announced, usability_not_completing_scan, + usability_redundant_ip, usability_gouging, usability_low_max_duration, usability_not_accepting_contracts, usability_not_announced, usability_not_completing_scan, score_age, score_collateral, score_interactions, score_storage_remaining, score_uptime, score_version, score_prices, gouging_contract_err, gouging_download_err, gouging_gouging_err, gouging_prune_err, gouging_upload_err) VALUES (?, @@ -1158,16 +1158,16 @@ func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, ON DUPLICATE KEY UPDATE created_at = VALUES(created_at), db_autopilot_id = VALUES(db_autopilot_id), db_host_id = VALUES(db_host_id), usability_blocked = VALUES(usability_blocked), usability_offline = VALUES(usability_offline), usability_low_score = VALUES(usability_low_score), - usability_redundant_ip = VALUES(usability_redundant_ip), usability_gouging = VALUES(usability_gouging), usability_not_accepting_contracts = VALUES(usability_not_accepting_contracts), + usability_redundant_ip = VALUES(usability_redundant_ip), usability_gouging = VALUES(usability_gouging), usability_low_max_duration = VALUES(usability_low_max_duration), usability_not_accepting_contracts = VALUES(usability_not_accepting_contracts), usability_not_announced = VALUES(usability_not_announced), usability_not_completing_scan = VALUES(usability_not_completing_scan), score_age = VALUES(score_age), score_collateral = VALUES(score_collateral), score_interactions = VALUES(score_interactions), score_storage_remaining = VALUES(score_storage_remaining), score_uptime = VALUES(score_uptime), score_version = VALUES(score_version), score_prices = VALUES(score_prices), gouging_contract_err = VALUES(gouging_contract_err), gouging_download_err = VALUES(gouging_download_err), gouging_gouging_err = VALUES(gouging_gouging_err), gouging_prune_err = VALUES(gouging_prune_err), gouging_upload_err = VALUES(gouging_upload_err) `, time.Now(), autopilot, ssql.PublicKey(hk), hc.UsabilityBreakdown.Blocked, hc.UsabilityBreakdown.Offline, hc.UsabilityBreakdown.LowScore, - hc.UsabilityBreakdown.RedundantIP, hc.UsabilityBreakdown.Gouging, hc.UsabilityBreakdown.NotAcceptingContracts, hc.UsabilityBreakdown.NotAnnounced, hc.UsabilityBreakdown.NotCompletingScan, + hc.UsabilityBreakdown.RedundantIP, hc.UsabilityBreakdown.Gouging, hc.UsabilityBreakdown.LowMaxDuration, hc.UsabilityBreakdown.NotAcceptingContracts, hc.UsabilityBreakdown.NotAnnounced, hc.UsabilityBreakdown.NotCompletingScan, hc.ScoreBreakdown.Age, hc.ScoreBreakdown.Collateral, hc.ScoreBreakdown.Interactions, hc.ScoreBreakdown.StorageRemaining, hc.ScoreBreakdown.Uptime, hc.ScoreBreakdown.Version, hc.ScoreBreakdown.Prices, - hc.GougingBreakdown.ContractErr, hc.GougingBreakdown.DownloadErr, hc.GougingBreakdown.GougingErr, hc.GougingBreakdown.PruneErr, hc.GougingBreakdown.UploadErr, + hc.GougingBreakdown.DownloadErr, hc.GougingBreakdown.GougingErr, hc.GougingBreakdown.PruneErr, hc.GougingBreakdown.UploadErr, ) if err != nil { return fmt.Errorf("failed to insert host check: %w", err) diff --git a/stores/sql/mysql/migrations/main/schema.sql b/stores/sql/mysql/migrations/main/schema.sql index 0f76fd6c3..825448c62 100644 --- a/stores/sql/mysql/migrations/main/schema.sql +++ b/stores/sql/mysql/migrations/main/schema.sql @@ -396,6 +396,7 @@ CREATE TABLE `host_checks` ( `usability_low_score` boolean NOT NULL DEFAULT false, `usability_redundant_ip` boolean NOT NULL DEFAULT false, `usability_gouging` boolean NOT NULL DEFAULT false, + `usability_low_max_duration` boolean NOT NULL DEFAULT false, `usability_not_accepting_contracts` boolean NOT NULL DEFAULT false, `usability_not_announced` boolean NOT NULL DEFAULT false, `usability_not_completing_scan` boolean NOT NULL DEFAULT false, @@ -408,7 +409,6 @@ CREATE TABLE `host_checks` ( `score_version` double NOT NULL, `score_prices` double NOT NULL, - `gouging_contract_err` text, `gouging_download_err` text, `gouging_gouging_err` text, `gouging_prune_err` text, diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 9be654edd..5b268b094 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -1166,16 +1166,16 @@ func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, ON CONFLICT (db_autopilot_id, db_host_id) DO UPDATE SET created_at = EXCLUDED.created_at, db_autopilot_id = EXCLUDED.db_autopilot_id, db_host_id = EXCLUDED.db_host_id, usability_blocked = EXCLUDED.usability_blocked, usability_offline = EXCLUDED.usability_offline, usability_low_score = EXCLUDED.usability_low_score, - usability_redundant_ip = EXCLUDED.usability_redundant_ip, usability_gouging = EXCLUDED.usability_gouging, usability_not_accepting_contracts = EXCLUDED.usability_not_accepting_contracts, + usability_redundant_ip = EXCLUDED.usability_redundant_ip, usability_gouging = EXCLUDED.usability_gouging, usability_low_max_duration = EXCLUDED.usability_low_max_duration, usability_not_accepting_contracts = EXCLUDED.usability_not_accepting_contracts, usability_not_announced = EXCLUDED.usability_not_announced, usability_not_completing_scan = EXCLUDED.usability_not_completing_scan, score_age = EXCLUDED.score_age, score_collateral = EXCLUDED.score_collateral, score_interactions = EXCLUDED.score_interactions, score_storage_remaining = EXCLUDED.score_storage_remaining, score_uptime = EXCLUDED.score_uptime, score_version = EXCLUDED.score_version, - score_prices = EXCLUDED.score_prices, gouging_contract_err = EXCLUDED.gouging_contract_err, gouging_download_err = EXCLUDED.gouging_download_err, + score_prices = EXCLUDED.score_prices, gouging_download_err = EXCLUDED.gouging_download_err, gouging_gouging_err = EXCLUDED.gouging_gouging_err, gouging_prune_err = EXCLUDED.gouging_prune_err, gouging_upload_err = EXCLUDED.gouging_upload_err `, time.Now(), autopilot, ssql.PublicKey(hk), hc.UsabilityBreakdown.Blocked, hc.UsabilityBreakdown.Offline, hc.UsabilityBreakdown.LowScore, - hc.UsabilityBreakdown.RedundantIP, hc.UsabilityBreakdown.Gouging, hc.UsabilityBreakdown.NotAcceptingContracts, hc.UsabilityBreakdown.NotAnnounced, hc.UsabilityBreakdown.NotCompletingScan, + hc.UsabilityBreakdown.RedundantIP, hc.UsabilityBreakdown.Gouging, hc.UsabilityBreakdown.LowMaxDuration, hc.UsabilityBreakdown.NotAcceptingContracts, hc.UsabilityBreakdown.NotAnnounced, hc.UsabilityBreakdown.NotCompletingScan, hc.ScoreBreakdown.Age, hc.ScoreBreakdown.Collateral, hc.ScoreBreakdown.Interactions, hc.ScoreBreakdown.StorageRemaining, hc.ScoreBreakdown.Uptime, hc.ScoreBreakdown.Version, hc.ScoreBreakdown.Prices, - hc.GougingBreakdown.ContractErr, hc.GougingBreakdown.DownloadErr, hc.GougingBreakdown.GougingErr, hc.GougingBreakdown.PruneErr, hc.GougingBreakdown.UploadErr, + hc.GougingBreakdown.DownloadErr, hc.GougingBreakdown.GougingErr, hc.GougingBreakdown.PruneErr, hc.GougingBreakdown.UploadErr, ) if err != nil { return fmt.Errorf("failed to insert host check: %w", err) diff --git a/stores/sql/sqlite/migrations/main/schema.sql b/stores/sql/sqlite/migrations/main/schema.sql index 609afc865..6aae7894f 100644 --- a/stores/sql/sqlite/migrations/main/schema.sql +++ b/stores/sql/sqlite/migrations/main/schema.sql @@ -167,7 +167,32 @@ CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`cre CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); -- dbHostCheck -CREATE TABLE `host_checks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +CREATE TABLE `host_checks` ( +`id` INTEGER PRIMARY KEY AUTOINCREMENT, +`created_at` datetime, +`db_autopilot_id` INTEGER NOT NULL, +`db_host_id` INTEGER NOT NULL, +`usability_blocked` INTEGER NOT NULL DEFAULT 0, +`usability_offline` INTEGER NOT NULL DEFAULT 0, +`usability_low_score` INTEGER NOT NULL DEFAULT 0, +`usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, +`usability_gouging` INTEGER NOT NULL DEFAULT 0, +`usability_low_max_duration` INTEGER NOT NULL DEFAULT 0, +`usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, +`usability_not_announced` INTEGER NOT NULL DEFAULT 0, +`usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, +`score_age` REAL NOT NULL, +`score_collateral` REAL NOT NULL, +`score_interactions` REAL NOT NULL, +`score_storage_remaining` REAL NOT NULL, +`score_uptime` REAL NOT NULL, +`score_version` REAL NOT NULL, +`score_prices` REAL NOT NULL, +`gouging_download_err` TEXT, +`gouging_gouging_err` TEXT, +`gouging_prune_err` TEXT, +`gouging_upload_err` TEXT, +FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); CREATE UNIQUE INDEX `idx_host_checks_id` ON `host_checks` (`db_autopilot_id`, `db_host_id`); CREATE INDEX `idx_host_checks_usability_blocked` ON `host_checks` (`usability_blocked`); CREATE INDEX `idx_host_checks_usability_offline` ON `host_checks` (`usability_offline`); From 03afadff34e0eca33144e2526c2db7a702d786bb Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 15:06:38 +0100 Subject: [PATCH 4/8] gouging: implement CheckV2 --- internal/gouging/gouging.go | 55 +++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/internal/gouging/gouging.go b/internal/gouging/gouging.go index b788e13a9..43c8991d6 100644 --- a/internal/gouging/gouging.go +++ b/internal/gouging/gouging.go @@ -106,13 +106,56 @@ func (gc checker) Check(hs *rhpv2.HostSettings, pt *rhpv3.HostPriceTable) api.Ho } // TODO: write tests -func (gc checker) CheckV2(hs rhpv4.HostSettings) api.HostGougingBreakdown { - return api.HostGougingBreakdown{ - DownloadErr: "", - GougingErr: "", - PruneErr: "", - UploadErr: "", +func (gc checker) CheckV2(hs rhpv4.HostSettings) (gb api.HostGougingBreakdown) { + prices := hs.Prices + gs := gc.settings + + // upload gouging + var uploadErrs []error + if prices.StoragePrice.Cmp(gs.MaxStoragePrice) > 0 { + uploadErrs = append(uploadErrs, fmt.Errorf("%v: storage price exceeds max storage price: %v > %v", ErrPriceTableGouging, prices.StoragePrice, gs.MaxStoragePrice)) + } + if prices.IngressPrice.Cmp(gs.MaxUploadPrice) > 0 { + uploadErrs = append(uploadErrs, fmt.Errorf("%v: ingress price exceeds max upload price: %v > %v", ErrPriceTableGouging, prices.IngressPrice, gs.MaxUploadPrice)) + } + gb.UploadErr = errsToStr(uploadErrs...) + if gougingErr := errsToStr(uploadErrs...); gougingErr != "" { + gb.UploadErr = fmt.Sprintf("%v: %s", ErrPriceTableGouging, gougingErr) + } + + // download gouging + if prices.EgressPrice.Cmp(gs.MaxDownloadPrice) > 0 { + gb.DownloadErr = fmt.Sprintf("%v: egress price exceeds max download price: %v > %v", ErrPriceTableGouging, prices.EgressPrice, gs.MaxDownloadPrice) + } + + // prune gouging + maxFreeSectorCost := types.Siacoins(1).Div64((1 << 40) / rhpv4.SectorSize) // 1 SC / TiB + if prices.FreeSectorPrice.Cmp(maxFreeSectorCost) > 0 { + gb.PruneErr = fmt.Sprintf("%v: cost to free a sector exceeds max free sector cost: %v > %v", ErrPriceTableGouging, prices.FreeSectorPrice, maxFreeSectorCost) + } + + // general gouging + const sectorDuration = 144 * 3 + const sectorBatchSize = 25600 + + var errs []error + if prices.ContractPrice.Cmp(gs.MaxContractPrice) > 0 { + errs = append(errs, fmt.Errorf("contract price exceeds max contract price: %v > %v", prices.ContractPrice, gs.MaxContractPrice)) + } + if hs.MaxCollateral.IsZero() { + errs = append(errs, errors.New("max collateral is zero")) + } + if hs.MaxSectorDuration < sectorDuration { + errs = append(errs, fmt.Errorf("max sector duration is less than %v: %v", sectorDuration, hs.MaxSectorDuration)) + } + if hs.MaxSectorBatchSize < sectorBatchSize { + errs = append(errs, fmt.Errorf("max sector batch size is less than %v: %v", sectorBatchSize, hs.MaxSectorBatchSize)) + } + if gougingErr := errsToStr(errs...); gougingErr != "" { + gb.GougingErr = fmt.Sprintf("%v: %s", ErrPriceTableGouging, gougingErr) } + gb.GougingErr = errsToStr(errs...) + return } func (gc checker) CheckSettings(hs rhpv2.HostSettings) api.HostGougingBreakdown { From 77b9be5b17a958945d66710c391c5ed5d77f6d70 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 15:22:08 +0100 Subject: [PATCH 5/8] contractor: use V2 gouging check in checkHost --- autopilot/contractor/hostfilter.go | 22 ++++++++++++-------- bus/routes.go | 3 ++- internal/gouging/gouging.go | 6 +++--- internal/rhp/v3/rpc.go | 2 +- stores/hostdb_test.go | 4 ++-- stores/sql/main.go | 14 +++++++++---- stores/sql/mysql/migrations/main/schema.sql | 1 + stores/sql/sqlite/main.go | 2 +- stores/sql/sqlite/migrations/main/schema.sql | 1 + worker/host.go | 4 ++-- 10 files changed, 36 insertions(+), 23 deletions(-) diff --git a/autopilot/contractor/hostfilter.go b/autopilot/contractor/hostfilter.go index bc5d75f43..226ac2e05 100644 --- a/autopilot/contractor/hostfilter.go +++ b/autopilot/contractor/hostfilter.go @@ -249,15 +249,7 @@ func checkHost(gc gouging.Checker, sh scoredHost, minScore float64, period uint6 ub.NotAcceptingContracts = true } - // perform gouging and score checks - gb = gc.Check(&h.Settings, &h.PriceTable.HostPriceTable) - if gb.Gouging() { - ub.Gouging = true - } else if minScore > 0 && !(sh.score > minScore) { - ub.LowScore = true - } - - // check contract gouging separately + // max duration check checkContractGouging := func(maxDuration uint64) { if period > maxDuration { ub.LowMaxDuration = true @@ -269,6 +261,18 @@ func checkHost(gc gouging.Checker, sh scoredHost, minScore float64, period uint6 checkContractGouging(h.Settings.MaxDuration) checkContractGouging(h.PriceTable.MaxDuration) } + + // perform gouging and score checks + if h.IsV2() { + gb = gc.CheckV2(h.V2Settings) + } else { + gb = gc.CheckV1(&h.Settings, &h.PriceTable.HostPriceTable) + } + if gb.Gouging() { + ub.Gouging = true + } else if minScore > 0 && !(sh.score > minScore) { + ub.LowScore = true + } } return &api.HostCheck{ diff --git a/bus/routes.go b/bus/routes.go index a2df35b8c..0fe02f798 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -500,7 +500,8 @@ func (b *Bus) hostsHandlerGET(jc jape.Context) { var infos []api.HostInfo for _, h := range hosts { - if !gc.Check(&h.HS, &h.PT).Gouging() { + if (len(h.V2SiamuxAddresses) > 0 && !gc.CheckV2(h.V2HS).Gouging()) || + (len(h.V2SiamuxAddresses) == 0 && !gc.CheckV1(&h.HS, &h.PT).Gouging()) { infos = append(infos, h.HostInfo) } } diff --git a/internal/gouging/gouging.go b/internal/gouging/gouging.go index 43c8991d6..5ee60e86c 100644 --- a/internal/gouging/gouging.go +++ b/internal/gouging/gouging.go @@ -41,7 +41,7 @@ type ( } Checker interface { - Check(*rhpv2.HostSettings, *rhpv3.HostPriceTable) api.HostGougingBreakdown + CheckV1(*rhpv2.HostSettings, *rhpv3.HostPriceTable) api.HostGougingBreakdown CheckV2(rhpv4.HostSettings) api.HostGougingBreakdown CheckSettings(rhpv2.HostSettings) api.HostGougingBreakdown CheckUnusedDefaults(rhpv3.HostPriceTable) error @@ -89,7 +89,7 @@ func (gc checker) BlocksUntilBlockHeightGouging(hostHeight uint64) int64 { return int64(hostHeight) - int64(minHeight) } -func (gc checker) Check(hs *rhpv2.HostSettings, pt *rhpv3.HostPriceTable) api.HostGougingBreakdown { +func (gc checker) CheckV1(hs *rhpv2.HostSettings, pt *rhpv3.HostPriceTable) api.HostGougingBreakdown { if hs == nil && pt == nil { panic("gouging checker needs to be provided with at least host settings or a price table") // developer error } @@ -159,7 +159,7 @@ func (gc checker) CheckV2(hs rhpv4.HostSettings) (gb api.HostGougingBreakdown) { } func (gc checker) CheckSettings(hs rhpv2.HostSettings) api.HostGougingBreakdown { - return gc.Check(&hs, nil) + return gc.CheckV1(&hs, nil) } func (gc checker) CheckUnusedDefaults(pt rhpv3.HostPriceTable) error { diff --git a/internal/rhp/v3/rpc.go b/internal/rhp/v3/rpc.go index db9c57469..44885f57f 100644 --- a/internal/rhp/v3/rpc.go +++ b/internal/rhp/v3/rpc.go @@ -368,7 +368,7 @@ func rpcRenew(ctx context.Context, t *transportV3, gc gouging.Checker, rev types } // Perform gouging checks. - if breakdown := gc.Check(nil, &pt); breakdown.Gouging() { + if breakdown := gc.CheckV1(nil, &pt); breakdown.Gouging() { return rhpv2.ContractRevision{}, nil, types.Currency{}, types.Currency{}, fmt.Errorf("host gouging during renew: %v", breakdown) } diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 17d864737..1bd98c16c 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -528,7 +528,7 @@ func TestUsableHosts(t *testing.T) { // assert h1 is not gouging h1 := hosts[0] - if gc.Check(&h1.HS, &h1.PT).Gouging() { + if gc.CheckV1(&h1.HS, &h1.PT).Gouging() { t.Fatal("unexpected") } @@ -551,7 +551,7 @@ func TestUsableHosts(t *testing.T) { // assert h1 is now gouging h1 = hosts[0] - if !gc.Check(&h1.HS, &h1.PT).Gouging() { + if !gc.CheckV1(&h1.HS, &h1.PT).Gouging() { t.Fatal("unexpected") } diff --git a/stores/sql/main.go b/stores/sql/main.go index 35466e877..66a7a216a 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -18,6 +18,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" + rhpv4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" rhp4 "go.sia.tech/coreutils/rhp/v4" @@ -40,8 +41,9 @@ var ( type ( HostInfo struct { api.HostInfo - HS rhpv2.HostSettings - PT rhpv3.HostPriceTable + HS rhpv2.HostSettings + PT rhpv3.HostPriceTable + V2HS rhpv4.HostSettings } multipartUpload struct { @@ -2325,6 +2327,7 @@ EXISTS ( hc.usability_low_score = 0 AND hc.usability_redundant_ip = 0 AND hc.usability_gouging = 0 AND + hc.usability_low_max_duration = 0 AND hc.usability_not_accepting_contracts = 0 AND hc.usability_not_announced = 0 AND hc.usability_not_completing_scan = 0 @@ -2338,7 +2341,8 @@ EXISTS ( COALESCE(h.net_address, ""), COALESCE(h.settings->>'$.siamuxport', "") AS siamux_port, h.price_table, - h.settings + h.settings, + h.v2_settings FROM hosts h INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ? @@ -2357,7 +2361,8 @@ EXISTS ( var addr, port string var pt PriceTable var hs HostSettings - err := rows.Scan(&hostID, &hk, &addr, &port, &pt, &hs) + var v2Hs V2HostSettings + err := rows.Scan(&hostID, &hk, &addr, &port, &pt, &hs, &v2Hs) if err != nil { return nil, fmt.Errorf("failed to scan host: %w", err) } @@ -2375,6 +2380,7 @@ EXISTS ( }, rhpv2.HostSettings(hs), rhpv3.HostPriceTable(pt), + rhpv4.HostSettings(v2Hs), }) hostIDs = append(hostIDs, hostID) } diff --git a/stores/sql/mysql/migrations/main/schema.sql b/stores/sql/mysql/migrations/main/schema.sql index 825448c62..5dd75c361 100644 --- a/stores/sql/mysql/migrations/main/schema.sql +++ b/stores/sql/mysql/migrations/main/schema.sql @@ -421,6 +421,7 @@ CREATE TABLE `host_checks` ( INDEX `idx_host_checks_usability_low_score` (`usability_low_score`), INDEX `idx_host_checks_usability_redundant_ip` (`usability_redundant_ip`), INDEX `idx_host_checks_usability_gouging` (`usability_gouging`), + INDEX `idx_host_checks_usability_low_max_duration` (`usability_low_max_duration`), INDEX `idx_host_checks_usability_not_accepting_contracts` (`usability_not_accepting_contracts`), INDEX `idx_host_checks_usability_not_announced` (`usability_not_announced`), INDEX `idx_host_checks_usability_not_completing_scan` (`usability_not_completing_scan`), diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 5b268b094..1fc940038 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -1156,7 +1156,7 @@ func (tx *MainDatabaseTx) UpdateHostBlocklistEntries(ctx context.Context, add, r func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, hk types.PublicKey, hc api.HostCheck) error { _, err := tx.Exec(ctx, ` INSERT INTO host_checks (created_at, db_autopilot_id, db_host_id, usability_blocked, usability_offline, usability_low_score, - usability_redundant_ip, usability_gouging, usability_not_accepting_contracts, usability_not_announced, usability_not_completing_scan, + usability_redundant_ip, usability_gouging, usability_low_max_duration, usability_not_accepting_contracts, usability_not_announced, usability_not_completing_scan, score_age, score_collateral, score_interactions, score_storage_remaining, score_uptime, score_version, score_prices, gouging_contract_err, gouging_download_err, gouging_gouging_err, gouging_prune_err, gouging_upload_err) VALUES (?, diff --git a/stores/sql/sqlite/migrations/main/schema.sql b/stores/sql/sqlite/migrations/main/schema.sql index 6aae7894f..1f9220bf8 100644 --- a/stores/sql/sqlite/migrations/main/schema.sql +++ b/stores/sql/sqlite/migrations/main/schema.sql @@ -199,6 +199,7 @@ CREATE INDEX `idx_host_checks_usability_offline` ON `host_checks` (`usability_of CREATE INDEX `idx_host_checks_usability_low_score` ON `host_checks` (`usability_low_score`); CREATE INDEX `idx_host_checks_usability_redundant_ip` ON `host_checks` (`usability_redundant_ip`); CREATE INDEX `idx_host_checks_usability_gouging` ON `host_checks` (`usability_gouging`); +CREATE INDEX `idx_host_checks_usability_low_max_duration` ON `host_checks` (`usability_low_max_duration`); CREATE INDEX `idx_host_checks_usability_not_accepting_contracts` ON `host_checks` (`usability_not_accepting_contracts`); CREATE INDEX `idx_host_checks_usability_not_announced` ON `host_checks` (`usability_not_announced`); CREATE INDEX `idx_host_checks_usability_not_completing_scan` ON `host_checks` (`usability_not_completing_scan`); diff --git a/worker/host.go b/worker/host.go index 52d846a3d..0f2a16b6c 100644 --- a/worker/host.go +++ b/worker/host.go @@ -86,7 +86,7 @@ func (h *host) DownloadSector(ctx context.Context, w io.Writer, root types.Hash2 if err != nil { return amount, err } - if breakdown := gc.Check(nil, &hpt); breakdown.DownloadErr != "" { + if breakdown := gc.CheckV1(nil, &hpt); breakdown.DownloadErr != "" { return amount, fmt.Errorf("%w: %v", gouging.ErrPriceTableGouging, breakdown.DownloadErr) } @@ -234,7 +234,7 @@ func (h *host) priceTable(ctx context.Context, rev *types.FileContractRevision) if err != nil { return rhpv3.HostPriceTable{}, cost, err } - if breakdown := gc.Check(nil, &pt.HostPriceTable); breakdown.Gouging() { + if breakdown := gc.CheckV1(nil, &pt.HostPriceTable); breakdown.Gouging() { return rhpv3.HostPriceTable{}, cost, fmt.Errorf("%w: %v", gouging.ErrPriceTableGouging, breakdown) } return pt.HostPriceTable, cost, nil From b09998c679aadf056834f0a46e996025804095bf Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 15:29:07 +0100 Subject: [PATCH 6/8] stores: fix TestProcessChainUpdate --- stores/sql/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/sql/main.go b/stores/sql/main.go index 66a7a216a..bbbd4eca0 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -900,7 +900,7 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er SELECT h.public_key, ap.identifier, hc.usability_blocked, hc.usability_offline, hc.usability_low_score, hc.usability_redundant_ip, hc.usability_gouging, hc.usability_low_max_duration, usability_not_accepting_contracts, hc.usability_not_announced, hc.usability_not_completing_scan, hc.score_age, hc.score_collateral, hc.score_interactions, hc.score_storage_remaining, hc.score_uptime, - hc.score_version, hc.score_prices, hc.gouging_contract_err, hc.gouging_download_err, hc.gouging_gouging_err, + hc.score_version, hc.score_prices, hc.gouging_download_err, hc.gouging_gouging_err, hc.gouging_prune_err, hc.gouging_upload_err FROM ( SELECT h.id, h.public_key From 0d20bc447c395998ac4cce7ebc89b68bf315e233 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 15:43:27 +0100 Subject: [PATCH 7/8] stores: fix TestHosts --- stores/sql/mysql/main.go | 2 +- stores/sql/mysql/migrations/main/schema.sql | 2 +- stores/sql/sqlite/main.go | 2 +- stores/sql/sqlite/migrations/main/schema.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 315fb6ea3..04cff372a 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -1162,7 +1162,7 @@ func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, usability_not_announced = VALUES(usability_not_announced), usability_not_completing_scan = VALUES(usability_not_completing_scan), score_age = VALUES(score_age), score_collateral = VALUES(score_collateral), score_interactions = VALUES(score_interactions), score_storage_remaining = VALUES(score_storage_remaining), score_uptime = VALUES(score_uptime), score_version = VALUES(score_version), - score_prices = VALUES(score_prices), gouging_contract_err = VALUES(gouging_contract_err), gouging_download_err = VALUES(gouging_download_err), + score_prices = VALUES(score_prices), gouging_download_err = VALUES(gouging_download_err), gouging_gouging_err = VALUES(gouging_gouging_err), gouging_prune_err = VALUES(gouging_prune_err), gouging_upload_err = VALUES(gouging_upload_err) `, time.Now(), autopilot, ssql.PublicKey(hk), hc.UsabilityBreakdown.Blocked, hc.UsabilityBreakdown.Offline, hc.UsabilityBreakdown.LowScore, hc.UsabilityBreakdown.RedundantIP, hc.UsabilityBreakdown.Gouging, hc.UsabilityBreakdown.LowMaxDuration, hc.UsabilityBreakdown.NotAcceptingContracts, hc.UsabilityBreakdown.NotAnnounced, hc.UsabilityBreakdown.NotCompletingScan, diff --git a/stores/sql/mysql/migrations/main/schema.sql b/stores/sql/mysql/migrations/main/schema.sql index 5dd75c361..6634a3ca1 100644 --- a/stores/sql/mysql/migrations/main/schema.sql +++ b/stores/sql/mysql/migrations/main/schema.sql @@ -43,7 +43,7 @@ CREATE TABLE `hosts` ( `created_at` datetime(3) DEFAULT NULL, `public_key` varbinary(32) NOT NULL, `settings` JSON, - `v2_settings` JSON NOT NULL DEFAULT '{}', + `v2_settings` JSON, `price_table` longtext, `price_table_expiry` datetime(3) DEFAULT NULL, `total_scans` bigint unsigned DEFAULT NULL, diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 1fc940038..21fb27b6d 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -1158,7 +1158,7 @@ func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, INSERT INTO host_checks (created_at, db_autopilot_id, db_host_id, usability_blocked, usability_offline, usability_low_score, usability_redundant_ip, usability_gouging, usability_low_max_duration, usability_not_accepting_contracts, usability_not_announced, usability_not_completing_scan, score_age, score_collateral, score_interactions, score_storage_remaining, score_uptime, score_version, score_prices, - gouging_contract_err, gouging_download_err, gouging_gouging_err, gouging_prune_err, gouging_upload_err) + gouging_download_err, gouging_gouging_err, gouging_prune_err, gouging_upload_err) VALUES (?, (SELECT id FROM autopilots WHERE identifier = ?), (SELECT id FROM hosts WHERE public_key = ?), diff --git a/stores/sql/sqlite/migrations/main/schema.sql b/stores/sql/sqlite/migrations/main/schema.sql index 1f9220bf8..9e1ca1d93 100644 --- a/stores/sql/sqlite/migrations/main/schema.sql +++ b/stores/sql/sqlite/migrations/main/schema.sql @@ -4,7 +4,7 @@ CREATE TABLE `hosts` ( `created_at` datetime, `public_key` blob NOT NULL UNIQUE, `settings` text, -`v2_settings` text NOT NULL DEFAULT '{}', +`v2_settings` text, `price_table` text, `price_table_expiry` datetime, `total_scans` integer, From f623e42291a9951dc41f61d34a8a604f64b3ac18 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Nov 2024 16:48:43 +0100 Subject: [PATCH 8/8] sql: insert empty v2_settings in UpdateHost --- api/host.go | 8 ++------ stores/sql/mysql/chain.go | 5 +++-- stores/sql/sqlite/chain.go | 5 +++-- stores/sql/types.go | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/api/host.go b/api/host.go index 4a6aee473..d658a318e 100644 --- a/api/host.go +++ b/api/host.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" "strings" "time" @@ -228,11 +227,8 @@ func (h Host) IsV2() bool { } func (h Host) V2SiamuxAddr() string { - // NOTE: eventually this can be smarter about picking an address but right now - // we just prioritize IPv4 over IPv6 - sort.Slice(h.V2SiamuxAddresses, func(i, j int) bool { - return len(h.V2SiamuxAddresses[i]) < len(h.V2SiamuxAddresses[j]) - }) + // NOTE: eventually we can improve this by implementing a dialer wrapper that + // can be created from a slice of addresses and tries them in order. if len(h.V2SiamuxAddresses) > 0 { return h.V2SiamuxAddresses[0] } diff --git a/stores/sql/mysql/chain.go b/stores/sql/mysql/chain.go index 2870be00e..3ffaa8a17 100644 --- a/stores/sql/mysql/chain.go +++ b/stores/sql/mysql/chain.go @@ -210,8 +210,8 @@ func (c chainUpdateTx) UpdateHost(hk types.PublicKey, v1Addr string, v2Ha chain. // create the host var hostID int64 if res, err := c.tx.Exec(c.ctx, ` - INSERT INTO hosts (created_at, public_key, settings, price_table, total_scans, last_scan, last_scan_success, second_to_last_scan_success, scanned, uptime, downtime, recent_downtime, recent_scan_failures, successful_interactions, failed_interactions, lost_sectors, last_announcement, net_address) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO hosts (created_at, public_key, settings, v2_settings, price_table, total_scans, last_scan, last_scan_success, second_to_last_scan_success, scanned, uptime, downtime, recent_downtime, recent_scan_failures, successful_interactions, failed_interactions, lost_sectors, last_announcement, net_address) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE last_announcement = VALUES(last_announcement), net_address = VALUES(net_address), @@ -220,6 +220,7 @@ func (c chainUpdateTx) UpdateHost(hk types.PublicKey, v1Addr string, v2Ha chain. time.Now().UTC(), ssql.PublicKey(hk), ssql.HostSettings{}, + ssql.V2HostSettings{}, ssql.PriceTable{}, 0, 0, diff --git a/stores/sql/sqlite/chain.go b/stores/sql/sqlite/chain.go index e97b8c1aa..eb46844a9 100644 --- a/stores/sql/sqlite/chain.go +++ b/stores/sql/sqlite/chain.go @@ -213,8 +213,8 @@ func (c chainUpdateTx) UpdateHost(hk types.PublicKey, v1Addr string, v2Ha chain. // create the host var hostID int64 if err := c.tx.QueryRow(c.ctx, ` - INSERT INTO hosts (created_at, public_key, settings, price_table, total_scans, last_scan, last_scan_success, second_to_last_scan_success, scanned, uptime, downtime, recent_downtime, recent_scan_failures, successful_interactions, failed_interactions, lost_sectors, last_announcement, net_address) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO hosts (created_at, public_key, settings, v2_settings, price_table, total_scans, last_scan, last_scan_success, second_to_last_scan_success, scanned, uptime, downtime, recent_downtime, recent_scan_failures, successful_interactions, failed_interactions, lost_sectors, last_announcement, net_address) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(public_key) DO UPDATE SET last_announcement = EXCLUDED.last_announcement, net_address = EXCLUDED.net_address @@ -222,6 +222,7 @@ func (c chainUpdateTx) UpdateHost(hk types.PublicKey, v1Addr string, v2Ha chain. time.Now().UTC(), ssql.PublicKey(hk), ssql.HostSettings{}, + ssql.V2HostSettings{}, ssql.PriceTable{}, 0, 0, diff --git a/stores/sql/types.go b/stores/sql/types.go index 8cf706c5d..043667600 100644 --- a/stores/sql/types.go +++ b/stores/sql/types.go @@ -559,7 +559,7 @@ func (hs *V2HostSettings) Scan(value interface{}) error { case []byte: bytes = value default: - return errors.New(fmt.Sprint("failed to unmarshal Settings value:", value)) + return errors.New(fmt.Sprint("failed to unmarshal V2Settings value:", value)) } return json.Unmarshal(bytes, hs) }