From f357d2ef23f47f00a1acfd5f0d68d422f0ac4873 Mon Sep 17 00:00:00 2001 From: Christopher Schinnerl Date: Wed, 3 Jul 2024 09:23:26 +0200 Subject: [PATCH 1/6] Use unit SC/TB/month for maxStoragePrice when pinning it (#1358) --- internal/bus/pinmanager.go | 44 +++++++++----------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index 02e4df79b..21591b21c 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -265,15 +265,9 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin if err != nil { pm.logger.Warn("failed to convert max download price to currency") } else if !gs.MaxDownloadPrice.Equals(update) { - bkp := gs.MaxDownloadPrice gs.MaxDownloadPrice = update - if err := gs.Validate(); err != nil { - pm.logger.Warn("failed to update gouging setting, new download price makes the setting invalid", zap.Error(err)) - gs.MaxDownloadPrice = bkp - } else { - pm.logger.Infow("updating max download price", "old", bkp, "new", gs.MaxDownloadPrice, "rate", rate) - updated = true - } + pm.logger.Infow("updating max download price", "old", gs.MaxDownloadPrice, "new", update, "rate", rate) + updated = true } } @@ -283,33 +277,21 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin if err != nil { pm.logger.Warnw("failed to convert max RPC price to currency", zap.Error(err)) } else if !gs.MaxRPCPrice.Equals(update) { - bkp := gs.MaxRPCPrice + pm.logger.Infow("updating max RPC price", "old", gs.MaxRPCPrice, "new", update, "rate", rate) gs.MaxRPCPrice = update - if err := gs.Validate(); err != nil { - pm.logger.Warnw("failed to update gouging setting, new RPC price makes the setting invalid", zap.Error(err)) - gs.MaxRPCPrice = bkp - } else { - pm.logger.Infow("updating max RPC price", "old", bkp, "new", gs.MaxRPCPrice, "rate", rate) - updated = true - } + updated = true } } // update max storage price if pins.MaxStorage.IsPinned() { - update, err := convertCurrencyToSC(decimal.NewFromFloat(pins.MaxStorage.Value), rate) + maxStorageCurr, err := convertCurrencyToSC(decimal.NewFromFloat(pins.MaxStorage.Value), rate) if err != nil { pm.logger.Warnw("failed to convert max storage price to currency", zap.Error(err)) - } else if !gs.MaxStoragePrice.Equals(update) { - bkp := gs.MaxStoragePrice + } else if update := maxStorageCurr.Div64(1e12).Div64(144 * 30); !gs.MaxStoragePrice.Equals(update) { // convert from SC/TB/month to SC/byte/block + pm.logger.Infow("updating max storage price", "old", gs.MaxStoragePrice, "new", update, "rate", rate) gs.MaxStoragePrice = update - if err := gs.Validate(); err != nil { - pm.logger.Warnw("failed to update gouging setting, new storage price makes the setting invalid", zap.Error(err)) - gs.MaxStoragePrice = bkp - } else { - pm.logger.Infow("updating max storage price", "old", bkp, "new", gs.MaxStoragePrice, "rate", rate) - updated = true - } + updated = true } } @@ -319,15 +301,9 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin if err != nil { pm.logger.Warnw("failed to convert max upload price to currency", zap.Error(err)) } else if !gs.MaxUploadPrice.Equals(update) { - bkp := gs.MaxUploadPrice + pm.logger.Infow("updating max upload price", "old", gs.MaxUploadPrice, "new", update, "rate", rate) gs.MaxUploadPrice = update - if err := gs.Validate(); err != nil { - pm.logger.Warnw("failed to update gouging setting, new upload price makes the setting invalid", zap.Error(err)) - gs.MaxUploadPrice = bkp - } else { - pm.logger.Infow("updating max upload price", "old", bkp, "new", gs.MaxUploadPrice, "rate", rate) - updated = true - } + updated = true } } From d615bdd950bbd331d9d81b06d0309b1354206b88 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 23 Aug 2024 15:21:05 +0200 Subject: [PATCH 2/6] e2e: cancel syncer context --- internal/test/e2e/host.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/test/e2e/host.go b/internal/test/e2e/host.go index e6867fa2c..bd10b4af1 100644 --- a/internal/test/e2e/host.go +++ b/internal/test/e2e/host.go @@ -107,9 +107,10 @@ type Host struct { dir string privKey types.PrivateKey - s *syncer.Syncer - cm *chain.Manager - chainDB *coreutils.BoltChainDB + s *syncer.Syncer + syncerCancel context.CancelFunc + cm *chain.Manager + chainDB *coreutils.BoltChainDB store *sqlite.Store wallet *wallet.SingleAddressWallet @@ -157,6 +158,7 @@ func (h *Host) Close() error { h.contracts.Close() h.storage.Close() h.store.Close() + h.syncerCancel() h.s.Close() h.chainDB.Close() return nil @@ -242,7 +244,13 @@ func NewHost(privKey types.PrivateKey, dir string, network *consensus.Network, g NetAddress: l.Addr().String(), }, syncer.WithPeerDiscoveryInterval(100*time.Millisecond), syncer.WithSyncInterval(100*time.Millisecond)) syncErrChan := make(chan error, 1) - go func() { syncErrChan <- s.Run(context.Background()) }() + syncerCtx, syncerCancel := context.WithCancel(context.Background()) + defer func() { + if err != nil { + syncerCancel() + } + }() + go func() { syncErrChan <- s.Run(syncerCtx) }() log := zap.NewNop() db, err := sqlite.OpenDatabase(filepath.Join(dir, "hostd.db"), log.Named("sqlite")) @@ -304,9 +312,10 @@ func NewHost(privKey types.PrivateKey, dir string, network *consensus.Network, g dir: dir, privKey: privKey, - s: s, - cm: cm, - chainDB: chainDB, + s: s, + syncerCancel: syncerCancel, + cm: cm, + chainDB: chainDB, store: db, wallet: wallet, From 028a6fbdd69f20ceede7e51e1cebc70bfae1c875 Mon Sep 17 00:00:00 2001 From: Christopher Schinnerl Date: Mon, 26 Aug 2024 10:36:54 +0200 Subject: [PATCH 3/6] Fix node setup (#1477) Refactoring the node setup it looks like I made `autopilot.Run` blocking. This prevents S3 setup from successfully completing. Fixes #1473 --- cmd/renterd/node.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index b4c330fc4..89dd75ab0 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -232,7 +232,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) } setupFns = append(setupFns, fn{ name: "Autopilot", - fn: func(_ context.Context) error { ap.Run(); return nil }, + fn: func(_ context.Context) error { go ap.Run(); return nil }, }) shutdownFns = append(shutdownFns, fn{ name: "Autopilot", @@ -252,6 +252,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) setupFns: setupFns, shutdownFns: shutdownFns, + bus: bc, cfg: cfg, logger: logger.Sugar(), From 3b7130ac28b14f6b439b29ded01d6456c976a883 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 26 Aug 2024 15:07:02 +0200 Subject: [PATCH 4/6] sql: only raise transaction log level to warn after 1s --- internal/sql/sql.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/sql/sql.go b/internal/sql/sql.go index b677e97fd..2b5920f9d 100644 --- a/internal/sql/sql.go +++ b/internal/sql/sql.go @@ -12,6 +12,7 @@ import ( "time" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "lukechampine.com/frand" ) @@ -172,7 +173,11 @@ LOOP: if sleep > maxBackoff { sleep = maxBackoff } - log.Warn("database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"), zap.Duration("retry", sleep)) + lvl := zapcore.DebugLevel + if time.Since(start) > time.Second { + lvl = zapcore.WarnLevel + } + log.Log(lvl, "database locked", zap.Duration("elapsed", time.Since(attemptStart)), zap.Duration("totalElapsed", time.Since(start)), zap.Stack("stack"), zap.Duration("retry", sleep)) select { case <-ctx.Done(): From cdf3592450938f4e46d3ba5b153a97b102855f73 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Sun, 15 Sep 2024 18:37:10 -0700 Subject: [PATCH 5/6] sqlite: fix Hash256 scanning --- stores/sql/sqlite/chain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/sql/sqlite/chain.go b/stores/sql/sqlite/chain.go index a98b777a4..5ec690994 100644 --- a/stores/sql/sqlite/chain.go +++ b/stores/sql/sqlite/chain.go @@ -132,7 +132,7 @@ func (c chainUpdateTx) WalletRevertIndex(index types.ChainIndex, removed, unspen // delete removed outputs for _, e := range removed { c.l.Debugw(fmt.Sprintf("remove output %v", e.ID), "height", index.Height, "block_id", index.ID) - if res, err := deleteRemovedStmt.Exec(c.ctx, e.ID); err != nil { + if res, err := deleteRemovedStmt.Exec(c.ctx, ssql.Hash256(e.ID)); err != nil { return fmt.Errorf("failed to delete removed output: %w", err) } else if n, err := res.RowsAffected(); err != nil { return fmt.Errorf("failed to get rows affected: %w", err) From 3665e5db4f50a5deea6117260c71c36608572e25 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 19 Sep 2024 15:24:45 +0200 Subject: [PATCH 6/6] stores: fix sqlite revert update failing --- stores/chain_test.go | 46 ++++++++++++++++++++++++++++++++++++++ stores/sql/sqlite/chain.go | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/stores/chain_test.go b/stores/chain_test.go index e0bcc8480..615e27098 100644 --- a/stores/chain_test.go +++ b/stores/chain_test.go @@ -9,6 +9,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/wallet" "go.sia.tech/renterd/api" "go.sia.tech/renterd/stores/sql" ) @@ -208,4 +209,49 @@ func TestProcessChainUpdate(t *testing.T) { if err := ss.ProcessChainUpdate(context.Background(), func(tx sql.ChainUpdateTx) error { return nil }); err != nil { panic("oh no") } + + if err := ss.ProcessChainUpdate(context.Background(), func(tx sql.ChainUpdateTx) error { + index3 := types.ChainIndex{Height: 3} + index4 := types.ChainIndex{Height: 4} + created := []types.SiacoinElement{ + { + StateElement: types.StateElement{}, + SiacoinOutput: types.SiacoinOutput{}, + MaturityHeight: 100, + }, + } + events := []wallet.Event{ + { + Type: wallet.EventTypeV2Transaction, + Data: wallet.EventV2Transaction{}, + }, + } + + // create some elements + err := tx.WalletApplyIndex(index3, created, nil, events, time.Now()) + if err != nil { + return err + } + + // spend them + err = tx.WalletApplyIndex(index4, nil, created, events, time.Now()) + if err != nil { + return err + } + + // revert the spend + err = tx.WalletRevertIndex(index4, nil, created, time.Now()) + if err != nil { + return err + } + + // revert the creation + err = tx.WalletRevertIndex(index3, nil, created, time.Now()) + if err != nil { + return err + } + return nil + }); err != nil { + t.Fatal("unexpected error", err) + } } diff --git a/stores/sql/sqlite/chain.go b/stores/sql/sqlite/chain.go index 5ec690994..74a35d7b8 100644 --- a/stores/sql/sqlite/chain.go +++ b/stores/sql/sqlite/chain.go @@ -155,7 +155,7 @@ func (c chainUpdateTx) WalletRevertIndex(index types.ChainIndex, removed, unspen c.l.Debugw(fmt.Sprintf("recreate unspent output %v", e.ID), "height", index.Height, "block_id", index.ID) if _, err := insertOutputStmt.Exec(c.ctx, time.Now().UTC(), - e.ID, + ssql.Hash256(e.ID), e.StateElement.LeafIndex, ssql.MerkleProof{Hashes: e.StateElement.MerkleProof}, ssql.Currency(e.SiacoinOutput.Value),