Blob Count + DB Migrations + Bunch of fixes (#49)

* geth update and minor cleanup

* use geth in check-payload-value

* cleanup

* use geth instead of FlashbotsRPC

* initial migrations

* add blob count!

* fix stuff

* makefile plus

* remove deprecated collect-live-bids service

* add comments to data-api-backfill

* backfill: logs cleanup

* cleanup

* payload check with min slot

* backfill: query with limit

* check-payload-value finish info

* backfill: log done and time needed

* cleanup

* fix GetLatestDeliveredPayload

* add migration for blob indexes

Makefile

@@ -1,66 +1,90 @@
+# Heavily inspired by Lighthouse: https://github.com/sigp/lighthouse/blob/stable/Makefile
+# and Reth: https://github.com/paradigmxyz/reth/blob/main/Makefile
+.DEFAULT_GOAL := help
+
 VERSION := $(shell git describe --tags --always --dirty="-dev")
 
-all: build-portable
+##@ Help
+
+help: ## Display this help
+	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-v:
+v: ## Show the current version
 	@echo "Version: ${VERSION}"
 
-clean:
+##@ Building
+
+.PHONY: clean
+clean: ## Remove build artifacts
 	rm -rf relayscan build/
 
-build:
+.PHONY: build
+build: ## Build the relayscan binary
 	go build -trimpath -ldflags "-s -X cmd.Version=${VERSION} -X main.Version=${VERSION}" -v -o relayscan .
 
-build-portable:
-	go build -trimpath -ldflags "-s -X cmd.Version=${VERSION} -X main.Version=${VERSION}" -v -o relayscan .
+.PHONY: docker-image
+docker-image: ## Build the relayscan docker image
+	DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --build-arg VERSION=${VERSION} . -t relayscan
 
-test:
-	go test ./...
+.PHONY: generate-ssz
+generate-ssz: ## Generate SSZ serialization code
+	rm -f common/ultrasoundbid_encoding.go
+	sszgen --path common --objs UltrasoundStreamBid
 
-test-race:
-	go test -race ./...
+##@ Production tasks
+
+.PHONY: update-bids-website
+update-bids-website: ## Update the bid archive website
+	go run . service bidcollect --build-website --build-website-upload
+
+##@ Linting and Testing
 
-lint:
+lint: ## Lint the code
 	gofmt -d -s .
 	gofumpt -d -extra .
 	go vet ./...
 	staticcheck ./...
 	golangci-lint run
 
-lt: lint test
+test: ## Run tests
+	go test ./...
+
+test-race: ## Run tests with -race fla
+	go test -race ./...
 
-gofumpt:
+lt: lint test ## Run lint and tests
+
+gofumpt: ## Run gofumpt on the code
 	gofumpt -l -w -extra .
 
-fmt:
+fmt: ## Format the code with gofmt and gofumpt and gc
 	gofmt -s -w .
 	gofumpt -extra -w .
 	gci write .
 	go mod tidy
 
-cover:
+cover: ## Run tests with coverage
 	go test -coverprofile=/tmp/go-sim-lb.cover.tmp ./...
 	go tool cover -func /tmp/go-sim-lb.cover.tmp
 	unlink /tmp/go-sim-lb.cover.tmp
 
-cover-html:
+cover-html: ## Run tests with coverage and output the HTML report
 	go test -coverprofile=/tmp/go-sim-lb.cover.tmp ./...
 	go tool cover -html=/tmp/go-sim-lb.cover.tmp
 	unlink /tmp/go-sim-lb.cover.tmp
 
-docker-image:
-	DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --build-arg VERSION=${VERSION} . -t relayscan
+##@ Development
 
+dev-website: ## Run the relayscan website service in development mode
+	DB_DONT_APPLY_SCHEMA=1 go run . service website --dev
 
-generate-ssz:
-	rm -f common/ultrasoundbid_encoding.go
-	sszgen --path common --objs UltrasoundStreamBid
+dev-bids-website: ## Run the bidcollect website in development mode
+	go run . service bidcollect --devserver
 
-update-bids-website:
-	go run . service bidcollect --build-website --build-website-upload
+dev-postgres-start: ## Start a Postgres container for development
+	docker run -d --name relayscan-postgres -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres postgres
 
-dev-website:
-	DB_DONT_APPLY_SCHEMA=1 go run . service website --dev
+dev-postgres-stop: ## Stop the Postgres container
+	docker rm -f relayscan-postgres
 
-dev-bids-website:
-	go run . service bidcollect --devserver
+dev-postgres-wipe: dev-postgres-stop dev-postgres-start ## Restart the Postgres container (wipes the database)

README.md

@@ -93,14 +93,23 @@ Start by filling the DB with relay data (delivered payloads), and checking it:
 # Copy .env.example to .env.local, update ETH_NODE_URI and source it
 source .env.local
 
-# Start Postgres
-docker run -d --name postgres -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres postgres
+# Start Postgres Docker container
+make dev-postgres-start
 
 # Query only a single relay, and for the shortest time possible
-go run . core data-api-backfill --relay fb --min-slot -1
+go run . core data-api-backfill --relay fb --min-slot -2000
 
-# Now the DB has data, check it (for only a single slot, the latest one, see logs for "latest received payload at slot N" in the backfill command)
+# Now the DB has data, check it (and update in DB)
+go run . core check-payload-value
+
+# Can also check a single slot only:
 go run . core check-payload-value --slot _N_
+
+# Reset DB
+dev-postgres-wipe
+
+# See the Makefile for more commands
+make help
 ```
 
 For linting and testing:
@@ -111,11 +120,13 @@ go install mvdan.cc/gofumpt@latest
 go install honnef.co/go/tools/cmd/[email protected]
 go install github.com/golangci/golangci-lint/cmd/[email protected]
 
-# Lint, test and build
+# Lint and test
 make lint
 make test
 make test-race
-make build
+
+# Format the code
+make fmt
 ```