From d9ec5db340561308b39f731bc4bfb7536b3a6285 Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Wed, 12 Jun 2024 09:39:33 +0200 Subject: [PATCH] website --- .gitignore | 3 +- Makefile | 6 + cmd/service/bidcollect.go | 15 ++ .../bids-combine-and-upload-yesterday.sh | 6 +- .../bids-combine-and-upload.sh | 0 scripts/bidcollect/s3/get-files.sh | 8 + scripts/bidcollect/s3/get-folders.sh | 2 + scripts/bidcollect/s3/upload-file-to-r2.sh | 16 ++ services/bidcollect/website/generator.go | 211 ++++++++++++++++++ .../bidcollect/website/templates/base.html | 26 +-- 10 files changed, 276 insertions(+), 17 deletions(-) rename scripts/{ => bidcollect}/bids-combine-and-upload-yesterday.sh (82%) rename scripts/{ => bidcollect}/bids-combine-and-upload.sh (100%) create mode 100755 scripts/bidcollect/s3/get-files.sh create mode 100755 scripts/bidcollect/s3/get-folders.sh create mode 100755 scripts/bidcollect/s3/upload-file-to-r2.sh create mode 100644 services/bidcollect/website/generator.go diff --git a/.gitignore b/.gitignore index 87cb106..6ee1f64 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ /relayscan /deploy* /test.csv -/csv/ \ No newline at end of file +/csv/ +/build/ \ No newline at end of file diff --git a/Makefile b/Makefile index c5e2809..cf02b7d 100644 --- a/Makefile +++ b/Makefile @@ -55,3 +55,9 @@ docker-image: generate-ssz: rm -f common/ultrasoundbid_encoding.go sszgen --path common --objs UltrasoundStreamBid + +bids-website: + go run . service bidcollect --build-website --build-website-upload + +bids-website-dev: + go run . service bidcollect --devserver diff --git a/cmd/service/bidcollect.go b/cmd/service/bidcollect.go index da29d3c..5fe54e2 100644 --- a/cmd/service/bidcollect.go +++ b/cmd/service/bidcollect.go @@ -23,6 +23,10 @@ var ( runDevServerOnly bool // used to play with file listing website devServerListenAddr = ":8095" + + buildWebsite bool + buildWebsiteUpload bool + buildWebsiteOutDir string ) func init() { @@ -40,6 +44,11 @@ func init() { // for dev purposes bidCollectCmd.Flags().BoolVar(&runDevServerOnly, "devserver", false, "only run devserver to play with file listing website") + + // building the S3 website + bidCollectCmd.Flags().BoolVar(&buildWebsite, "build-website", false, "build file listing website") + bidCollectCmd.Flags().BoolVar(&buildWebsiteUpload, "build-website-upload", false, "upload after building") + bidCollectCmd.Flags().StringVar(&buildWebsiteOutDir, "build-website-out", "build", "output directory for website") } var bidCollectCmd = &cobra.Command{ @@ -52,6 +61,12 @@ var bidCollectCmd = &cobra.Command{ return } + if buildWebsite { + log.Infof("Bidcollect %s building website (output: %s) ...", vars.Version, buildWebsiteOutDir) + website.BuildProdWebsite(log, buildWebsiteOutDir, buildWebsiteUpload) + return + } + log.Infof("Bidcollect starting (%s) ...", vars.Version) // Prepare relays diff --git a/scripts/bids-combine-and-upload-yesterday.sh b/scripts/bidcollect/bids-combine-and-upload-yesterday.sh similarity index 82% rename from scripts/bids-combine-and-upload-yesterday.sh rename to scripts/bidcollect/bids-combine-and-upload-yesterday.sh index 7e48ebc..97c264a 100755 --- a/scripts/bids-combine-and-upload-yesterday.sh +++ b/scripts/bidcollect/bids-combine-and-upload-yesterday.sh @@ -18,13 +18,13 @@ echo "upload for: $d" # change to project root directory cd "$(dirname "$0")" -cd .. +cd ../../ # load environment variables source .env.prod # archive and upload! -./scripts/bids-combine-and-upload.sh "/mnt/data/relayscan-bids/$d/" +./scripts/bidcollect/bids-combine-and-upload.sh "/mnt/data/relayscan-bids/$d/" # update website -# make website +make bids-website diff --git a/scripts/bids-combine-and-upload.sh b/scripts/bidcollect/bids-combine-and-upload.sh similarity index 100% rename from scripts/bids-combine-and-upload.sh rename to scripts/bidcollect/bids-combine-and-upload.sh diff --git a/scripts/bidcollect/s3/get-files.sh b/scripts/bidcollect/s3/get-files.sh new file mode 100755 index 0000000..71405bd --- /dev/null +++ b/scripts/bidcollect/s3/get-files.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# require one argument +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +aws --profile r2 s3 ls s3://relayscan-bidarchive/$1 --endpoint-url "https://${CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \ No newline at end of file diff --git a/scripts/bidcollect/s3/get-folders.sh b/scripts/bidcollect/s3/get-folders.sh new file mode 100755 index 0000000..18c47ef --- /dev/null +++ b/scripts/bidcollect/s3/get-folders.sh @@ -0,0 +1,2 @@ +#!/bin/bash +aws --profile r2 s3 ls s3://relayscan-bidarchive/$1 --endpoint-url "https://${CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com" | awk '{ print $2 }' \ No newline at end of file diff --git a/scripts/bidcollect/s3/upload-file-to-r2.sh b/scripts/bidcollect/s3/upload-file-to-r2.sh new file mode 100755 index 0000000..55d5e8e --- /dev/null +++ b/scripts/bidcollect/s3/upload-file-to-r2.sh @@ -0,0 +1,16 @@ +#!/bin/bash +src=$1 +target=$2 +if [ -z "$src" ]; then + echo "Usage: $0 ["] + exit 1 +fi + +# auto-fill target if not given +if [ -z "$target" ]; then + # remove "/mnt/data/relayscan-bidarchive/" prefix from src and make it the S3 prefix + target="/ethereum/mainnet/${src#"/mnt/data/relayscan-bidarchive/"}" +fi + +echo "uploading $src to S3 $target ..." +aws --profile r2 s3 cp $src s3://relayscan-bidarchive$target --endpoint-url "https://${CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com" diff --git a/services/bidcollect/website/generator.go b/services/bidcollect/website/generator.go new file mode 100644 index 0000000..9418147 --- /dev/null +++ b/services/bidcollect/website/generator.go @@ -0,0 +1,211 @@ +package website + +// +// Quick and dirty website generator +// + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "github.com/tdewolff/minify" + "github.com/tdewolff/minify/css" + "github.com/tdewolff/minify/html" +) + +func BuildProdWebsite(log *logrus.Entry, outDir string, upload bool) { + log.Infof("Creating build server in %s", outDir) + err := os.MkdirAll(outDir, os.ModePerm) + if err != nil { + log.Fatal(err) + } + + dir := "ethereum/mainnet/" + + // Setup minifier + minifier := minify.New() + minifier.AddFunc("text/html", html.Minify) + minifier.AddFunc("text/css", css.Minify) + + // Load month folders from S3 + log.Infof("Getting folders from S3 for %s ...", dir) + months, err := getFoldersFromS3(dir) + if err != nil { + log.Fatal(err) + } + fmt.Println("Months:", months) + + // build root page + log.Infof("Building root page ...") + rootPageData := HTMLData{ //nolint:exhaustruct + Title: "", + Path: "/index.html", + EthMainnetMonths: months, + } + + tpl, err := ParseIndexTemplate() + if err != nil { + log.Fatal(err) + } + + buf := new(bytes.Buffer) + err = tpl.ExecuteTemplate(buf, "base", rootPageData) + if err != nil { + log.Fatal(err) + } + + // minify + mBytes, err := minifier.Bytes("text/html", buf.Bytes()) + if err != nil { + log.Fatal(err) + } + + // write to file + fn := filepath.Join(outDir, "index.html") + log.Infof("Writing to %s ...", fn) + err = os.WriteFile(fn, mBytes, 0o0600) + if err != nil { + log.Fatal(err) + } + + toUpload := []struct{ from, to string }{ + {fn, "/"}, + } + + // build files pages + for _, month := range months { + dir := "ethereum/mainnet/" + month + "/" + log.Infof("Getting files from S3 for %s ...", dir) + files, err := getFilesFromS3(dir) + if err != nil { + log.Fatal(err) + } + + rootPageData := HTMLData{ //nolint:exhaustruct + Title: month, + Path: fmt.Sprintf("ethereum/mainnet/%s/index.html", month), + + CurrentNetwork: "Ethereum Mainnet", + CurrentMonth: month, + Files: files, + } + + tpl, err := ParseFilesTemplate() + if err != nil { + log.Fatal(err) + } + + buf := new(bytes.Buffer) + err = tpl.ExecuteTemplate(buf, "base", rootPageData) + if err != nil { + log.Fatal(err) + } + + // minify + mBytes, err := minifier.Bytes("text/html", buf.Bytes()) + if err != nil { + log.Fatal(err) + } + + // write to file + _outDir := filepath.Join(outDir, dir) + err = os.MkdirAll(_outDir, os.ModePerm) + if err != nil { + log.Fatal(err) + } + + fn := filepath.Join(_outDir, "index.html") + log.Infof("Writing to %s ...", fn) + err = os.WriteFile(fn, mBytes, 0o0600) + if err != nil { + log.Fatal(err) + } + + toUpload = append(toUpload, struct{ from, to string }{fn, "/" + dir}) + } + + if upload { + log.Info("Uploading to S3 ...") + // for _, file := range toUpload { + // fmt.Printf("- %s -> %s\n", file.from, file.to) + // } + + for _, file := range toUpload { + app := "./scripts/bidcollect/s3/upload-file-to-r2.sh" + cmd := exec.Command(app, file.from, file.to) //nolint:gosec + stdout, err := cmd.Output() + if err != nil { + log.Fatal(err) + } + fmt.Println(string(stdout)) + } + } +} + +func getFoldersFromS3(dir string) ([]string, error) { + folders := []string{} + + app := "./scripts/bidcollect/s3/get-folders.sh" + cmd := exec.Command(app, dir) + stdout, err := cmd.Output() + if err != nil { + return folders, err + } + + // Print the output + lines := strings.Split(string(stdout), "\n") + for _, line := range lines { + if line != "" && strings.HasPrefix(line, "20") { + folders = append(folders, strings.TrimSuffix(line, "/")) + } + } + return folders, nil +} + +func getFilesFromS3(month string) ([]FileEntry, error) { + files := []FileEntry{} + + app := "./scripts/bidcollect/s3/get-files.sh" + cmd := exec.Command(app, month) + stdout, err := cmd.Output() + if err != nil { + return files, err + } + + space := regexp.MustCompile(`\s+`) + lines := strings.Split(string(stdout), "\n") + for _, line := range lines { + if line != "" { + line = space.ReplaceAllString(line, " ") + parts := strings.Split(line, " ") + + // parts[2] is the size + size, err := strconv.ParseUint(parts[2], 10, 64) + if err != nil { + return files, err + } + + filename := parts[3] + + if filename == "index.html" { + continue + } else if strings.HasSuffix(filename, ".csv.gz") { + continue + } + + files = append(files, FileEntry{ + Filename: filename, + Size: size, + Modified: parts[1] + " " + parts[0], + }) + } + } + return files, nil +} diff --git a/services/bidcollect/website/templates/base.html b/services/bidcollect/website/templates/base.html index b9f2df9..22959ca 100644 --- a/services/bidcollect/website/templates/base.html +++ b/services/bidcollect/website/templates/base.html @@ -1,6 +1,6 @@ {{ define "base" }} -{{ $title:="Mempool Dumpster ♻️" }} +{{ $title:="Relayscan Bidarchive 📚" }} {{ if ne .Title "" }} {{ $title = (printf "%v | %v" .Title $title) }} @@ -24,22 +24,22 @@ {{ $title }} - + - + - - + + - - + + - - + + @@ -111,7 +111,7 @@ -

Relayscan Bidarchive

+

{{ $title }}

https://github.com/flashbots/relayscan