From ef1d630e4d58951615400a7887c68ca2f8514c86 Mon Sep 17 00:00:00 2001
From: Chris Hager <chris@linuxuser.at>
Date: Fri, 23 Feb 2024 14:04:10 +0100
Subject: [PATCH] added index, more logging for loading data

---
 README.md                     |  7 +++++++
 database/schema.go            |  1 +
 services/website/html.go      |  9 +++++++++
 services/website/webserver.go | 28 +++++++++++++++++++++-------
 4 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 47cc7ae..27502a1 100644
--- a/README.md
+++ b/README.md
@@ -102,3 +102,10 @@ make test
 make test-race
 make build
 ```
+
+
+### Updating relayscan
+
+Notes for updating relayscan:
+
+- Relay payloads are selected by `inserted_at`. When adding a new relay, you probably want to manually subtract a day from `inserted_at` so they don't show up all for today (`UPDATE mainnet_data_api_payload_delivered SET inserted_at = inserted_at - INTERVAL '1 DAY' WHERE relay='newrelay.xyz';`). See also https://github.com/flashbots/relayscan/issues/28
diff --git a/database/schema.go b/database/schema.go
index 11327d1..5c6046f 100644
--- a/database/schema.go
+++ b/database/schema.go
@@ -97,6 +97,7 @@ CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_value_wei_idx ON
 CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_valuecheck_ok_idx ON ` + TableDataAPIPayloadDelivered + `("value_check_ok");
 CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_slotmissed_idx ON ` + TableDataAPIPayloadDelivered + `("slot_missed");
 CREATE INDEX IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_cb_diff_eth_idx ON ` + TableDataAPIPayloadDelivered + `("coinbase_diff_eth");
+CREATE INDEX CONCURRENTLY IF NOT EXISTS ` + TableDataAPIPayloadDelivered + `_insertedat_relay_idx ON ` + TableDataAPIPayloadDelivered + `("inserted_at", "relay");
 
 
 CREATE TABLE IF NOT EXISTS ` + TableDataAPIBuilderBid + ` (
diff --git a/services/website/html.go b/services/website/html.go
index 7daa2c2..9e7c333 100644
--- a/services/website/html.go
+++ b/services/website/html.go
@@ -18,6 +18,15 @@ type Stats struct {
 	TopBuildersByRelay map[string][]*database.TopBuilderEntry
 }
 
+func NewStats() *Stats {
+	return &Stats{
+		TopRelays:          make([]*database.TopRelayEntry, 0),
+		TopBuilders:        make([]*database.TopBuilderEntry, 0),
+		BuilderProfits:     make([]*database.BuilderProfitEntry, 0),
+		TopBuildersByRelay: make(map[string][]*database.TopBuilderEntry),
+	}
+}
+
 type HTMLData struct {
 	Title string
 
diff --git a/services/website/webserver.go b/services/website/webserver.go
index ff37658..e1aadc2 100644
--- a/services/website/webserver.go
+++ b/services/website/webserver.go
@@ -152,20 +152,26 @@ func (srv *Webserver) getStatsForHours(duration time.Duration) (stats *Stats, er
 	now := time.Now().UTC()
 	since := now.Add(-1 * duration.Abs())
 
+	startTime := time.Now()
 	topRelays, err := srv.db.GetTopRelays(since, now)
 	if err != nil {
 		return nil, err
 	}
+	srv.log.WithField("duration", time.Since(startTime).String).Debug("got top relays")
 
+	startTime = time.Now()
 	topBuilders, err := srv.db.GetTopBuilders(since, now, "")
 	if err != nil {
 		return nil, err
 	}
+	srv.log.WithField("duration", time.Since(startTime).String).Debug("got top builders")
 
+	startTime = time.Now()
 	builderProfits, err := srv.db.GetBuilderProfits(since, now)
 	if err != nil {
 		return nil, err
 	}
+	srv.log.WithField("duration", time.Since(startTime).String).Debug("got builder profits")
 
 	stats = &Stats{
 		Since: since,
@@ -217,34 +223,42 @@ func (srv *Webserver) updateHTML() {
 		htmlData.LastUpdateSlot = entry.Slot
 	}
 
+	startUpdate := time.Now()
 	srv.log.Info("updating 24h stats...")
 	stats["24h"], err = srv.getStatsForHours(24 * time.Hour)
 	if err != nil {
 		srv.log.WithError(err).Error("Failed to get stats for 24h")
 		return
 	}
+	srv.log.WithField("duration", time.Since(startUpdate).String).Info("updated 24h stats")
 
 	if !srv.opts.Only24h {
-		srv.log.Info("updating 7d stats...")
-		stats["7d"], err = srv.getStatsForHours(7 * 24 * time.Hour)
-		if err != nil {
-			srv.log.WithError(err).Error("Failed to get stats for 24h")
-			return
-		}
-
+		startUpdate = time.Now()
 		srv.log.Info("updating 12h stats...")
 		stats["12h"], err = srv.getStatsForHours(12 * time.Hour)
 		if err != nil {
 			srv.log.WithError(err).Error("Failed to get stats for 12h")
 			return
 		}
+		srv.log.WithField("duration", time.Since(startUpdate).String).Info("updated 12h stats")
 
+		startUpdate = time.Now()
 		srv.log.Info("updating 1h stats...")
 		stats["1h"], err = srv.getStatsForHours(1 * time.Hour)
 		if err != nil {
 			srv.log.WithError(err).Error("Failed to get stats for 1h")
 			return
 		}
+		srv.log.WithField("duration", time.Since(startUpdate).String).Info("updated 1h stats")
+
+		startUpdate = time.Now()
+		srv.log.Info("updating 7d stats...")
+		stats["7d"], err = srv.getStatsForHours(7 * 24 * time.Hour)
+		if err != nil {
+			srv.log.WithError(err).Error("Failed to get stats for 24h")
+			return
+		}
+		srv.log.WithField("duration", time.Since(startUpdate).String).Info("updated 7d stats")
 	}
 
 	// Save the html data