Run go-ycsb benchmarks (#1016)

.github/workflows/dance.yml

@@ -59,11 +59,9 @@ jobs:
 
           - mongo-tools
 
-          # - restheart
-          # - restheart-auth
-
-          # - ycsb-workloada
-          # - ycsb-workloadc
+          - ycsb-workloada
+          - ycsb-workloadb
+          - ycsb-workloadc
 
     steps:
       - name: Checkout code
@@ -86,11 +84,21 @@ jobs:
           FERRETDB_IMAGE: ${{ inputs.ferretdb_image || 'ghcr.io/ferretdb/ferretdb-dev:main' }}
           POSTGRES_IMAGE: ${{ inputs.postgres_image || 'postgres:16.4' }}
 
+      - name: Connect to Tailscale
+        if: github.event_name != 'pull_request'
+        uses: tailscale/github-action@v2
+        with:
+          oauth-client-id: ${{ secrets.TAILSCALE_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TAILSCALE_SECRET }}
+          tags: tag:ci
+
       - name: Run init
         run: bin/task init
 
       - name: Dance!
         run: bin/task dance CONFIG=${{ matrix.project }}.yml
+        env:
+          DANCE_PUSH: ${{ secrets.DANCE_PUSH }}
 
       - name: Collect logs
         if: failure()

.golangci-new.yml

@@ -12,11 +12,6 @@ linters-settings:
     ignore-test: false
   copyloopvar:
     check-alias: true
-  errcheck:
-    check-type-assertions: false
-    check-blank: false
-    disable-default-exclusions: false
-    exclude-functions: []
   errorlint:
     # see caveats at https://github.com/polyfloyd/go-errorlint#fmterrorf-wrapping-verb
     errorf: false
@@ -66,6 +61,7 @@ linters:
     # checked by the other configuration
     - asciicheck
     - depguard
+    - errcheck
     - exhaustive
     - forbidigo
     - gci
@@ -177,3 +173,6 @@ issues:
   # whole-files: true
 
   exclude-use-default: false
+  exclude-rules:
+    - linters: [govet]
+      text: "composites: go.mongodb.org/mongo-driver/bson/primitive.E struct literal uses unkeyed fields"

.golangci.yml

@@ -18,6 +18,14 @@ linters-settings:
             desc: use `slices` package instead
           - pkg: maps
             desc: use `golang.org/x/exp/maps` package instead
+  errcheck:
+    check-type-assertions: false
+    check-blank: false
+    disable-default-exclusions: false
+    exclude-functions:
+      - (*go.mongodb.org/mongo-driver/mongo.Client).Disconnect
+      - (*go.mongodb.org/mongo-driver/mongo.Cursor).Close
+      - (io.ReadCloser).Close
   exhaustive:
     default-signifies-exhaustive: false
   forbidigo:
@@ -120,6 +128,7 @@ linters:
   enable:
     - asciicheck
     - depguard
+    - errcheck
     - exhaustive
     - forbidigo
     - gci

Taskfile.yaml

@@ -4,7 +4,6 @@ version: "3"
 vars:
   DB: ""
   CONFIG: ""
-  FLAGS: ""
   RACE_FLAG: -race={{and (ne OS "windows") (ne ARCH "arm") (ne ARCH "riscv64")}}
 
 tasks:
@@ -83,7 +82,7 @@ tasks:
     deps: [build]
     dir: projects
     cmds:
-      - ../bin/dance {{.FLAGS}} --database={{.DB}} {{.CONFIG}}
+      - ../bin/dance --database={{.DB}} {{.CONFIG}}
 
   lint:
     desc: "Run linters"

cmd/dance/main.go

@@ -32,6 +32,9 @@ import (
 	"github.com/alecthomas/kong"
 	"github.com/pmezard/go-difflib/difflib"
 	"github.com/sethvargo/go-githubactions"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
 	"golang.org/x/exp/maps"
 	"gopkg.in/yaml.v3"
 
@@ -40,14 +43,14 @@ import (
 	"github.com/FerretDB/dance/internal/runner"
 	"github.com/FerretDB/dance/internal/runner/command"
 	"github.com/FerretDB/dance/internal/runner/gotest"
+	"github.com/FerretDB/dance/internal/runner/ycsb"
 )
 
 func waitForPort(ctx context.Context, port int) error {
 	for ctx.Err() == nil {
 		conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port))
 		if err == nil {
-			conn.Close()
-			return nil
+			return conn.Close()
 		}
 
 		sleepCtx, sleepCancel := context.WithTimeout(ctx, time.Second)
@@ -58,7 +61,7 @@ func waitForPort(ctx context.Context, port int) error {
 	return ctx.Err()
 }
 
-func logResult(label string, res map[string]string) {
+func logResult(label string, res map[string]config.TestResult) {
 	keys := maps.Keys(res)
 	if len(keys) == 0 {
 		return
@@ -67,16 +70,25 @@ func logResult(label string, res map[string]string) {
 	log.Printf("%s tests:", label)
 	sort.Strings(keys)
 	for _, t := range keys {
-		out := res[t]
-		log.Printf("===> %s:\n\t%s\n\n", t, out)
+		log.Printf("===> %s:", t)
+
+		if o := res[t].Output; o != "" {
+			log.Printf("\t%s", o)
+		}
+
+		if m := res[t].Measurements; m != nil {
+			log.Printf("\tMeasurements: %v", m)
+		}
+
+		log.Printf("")
 	}
 }
 
 //nolint:vet // for readability
 var cli struct {
-	// TODO https://github.com/FerretDB/dance/issues/30
 	Database []string `help:"${help_database}" enum:"${enum_database}"               short:"d"`
 	Verbose  bool     `help:"Be more verbose." short:"v"`
+	Push     string   `help:"Push results to the given MongoDB URI."`
 	Config   []string `arg:""                  help:"Project configurations to run." optional:"" type:"existingfile"`
 }
 
@@ -142,6 +154,19 @@ func main() {
 		}
 	}
 
+	var mongoClient *mongo.Client
+
+	if cli.Push != "" {
+		log.Printf("Connecting to %+v to push data...", cli.Push)
+
+		var err error
+		if mongoClient, err = mongo.Connect(ctx, options.Client().ApplyURI(cli.Push)); err != nil {
+			log.Fatalf("Failed to connect to MongoDB URI to push results: %s", err)
+		}
+
+		defer mongoClient.Disconnect(ctx)
+	}
+
 	if len(cli.Config) == 0 {
 		var err error
 		if cli.Config, err = filepath.Glob("*.yml"); err != nil {
@@ -164,7 +189,7 @@ func main() {
 				log.Fatal(err)
 			}
 
-			rl := l.With(slog.String("config", cf), slog.String("db", db))
+			rl := l.With(slog.String("config", cf), slog.String("database", db))
 
 			var runner runner.Runner
 
@@ -173,10 +198,8 @@ func main() {
 				runner, err = command.New(c.Params.(*config.RunnerParamsCommand), rl, cli.Verbose)
 			case config.RunnerTypeGoTest:
 				runner, err = gotest.New(c.Params.(*config.RunnerParamsGoTest), rl, cli.Verbose)
-			case config.RunnerTypeJSTest:
-				fallthrough
 			case config.RunnerTypeYCSB:
-				fallthrough
+				runner, err = ycsb.New(c.Params.(*config.RunnerParamsYCSB), rl)
 			default:
 				log.Fatalf("unknown runner: %q", c.Runner)
 			}
@@ -255,6 +278,30 @@ func main() {
 				action := githubactions.New()
 				action.Noticef("%s", msg)
 			}
+
+			if mongoClient != nil {
+				hostname, _ := os.Hostname()
+
+				var passed bson.D
+				for t, tr := range cmp.Passed {
+					passed = append(passed, bson.E{t, bson.D{{"m", tr.Measurements}}})
+				}
+
+				doc := bson.D{
+					{"config", cf},
+					{"database", db},
+					{"time", time.Now()},
+					{"env", bson.D{
+						{"runner", os.Getenv("RUNNER_NAME")},
+						{"hostname", hostname},
+					}},
+					{"passed", passed},
+				}
+
+				if _, err = mongoClient.Database("dance").Collection("incoming").InsertOne(ctx, doc); err != nil {
+					log.Fatal(err)
+				}
+			}
 		}
 	}
 }

internal/config/results.go

@@ -39,16 +39,16 @@ func (tr *TestResult) IndentedOutput() string {
 // CompareResults represents the comparison between expected and actual test outcomes.
 type CompareResults struct {
 	// expected
-	Failed  map[string]string
-	Skipped map[string]string
-	Passed  map[string]string
+	Failed  map[string]TestResult
+	Skipped map[string]TestResult
+	Passed  map[string]TestResult
 
 	// unexpected
-	XFailed  map[string]string
-	XSkipped map[string]string
-	XPassed  map[string]string
+	XFailed  map[string]TestResult
+	XSkipped map[string]TestResult
+	XPassed  map[string]TestResult
 
-	Unknown map[string]string
+	Unknown map[string]TestResult
 
 	Stats Stats
 }
@@ -88,13 +88,13 @@ func (expected *ExpectedResults) mapStatuses() map[string]Status {
 // Compare compares expected and actual results.
 func (expected *ExpectedResults) Compare(actual map[string]TestResult) (*CompareResults, error) {
 	res := &CompareResults{
-		Failed:   make(map[string]string),
-		Skipped:  make(map[string]string),
-		Passed:   make(map[string]string),
-		XFailed:  make(map[string]string),
-		XSkipped: make(map[string]string),
-		XPassed:  make(map[string]string),
-		Unknown:  make(map[string]string),
+		Failed:   make(map[string]TestResult),
+		Skipped:  make(map[string]TestResult),
+		Passed:   make(map[string]TestResult),
+		XFailed:  make(map[string]TestResult),
+		XSkipped: make(map[string]TestResult),
+		XPassed:  make(map[string]TestResult),
+		Unknown:  make(map[string]TestResult),
 	}
 
 	tests := maps.Keys(actual)
@@ -115,48 +115,53 @@ func (expected *ExpectedResults) Compare(actual map[string]TestResult) (*Compare
 		}
 
 		o := actualResult.IndentedOutput()
+		tr := TestResult{
+			Status:       actualResult.Status,
+			Output:       o,
+			Measurements: actualResult.Measurements,
+		}
 
 		switch expectedStatus {
 		case Fail:
 			switch actualResult.Status {
 			case Fail:
-				res.Failed[test] = o
+				res.Failed[test] = tr
 			case Skip:
-				res.XSkipped[test] = o
+				res.XSkipped[test] = tr
 			case Pass:
-				res.XPassed[test] = o
+				res.XPassed[test] = tr
 			case Ignore, Unknown:
 				fallthrough
 			default:
-				res.Unknown[test] = o
+				res.Unknown[test] = tr
 			}
 
 		case Skip:
 			switch actualResult.Status {
 			case Fail:
-				res.XFailed[test] = o
+				res.XFailed[test] = tr
 			case Skip:
-				res.Skipped[test] = o
+				res.Skipped[test] = tr
 			case Pass:
-				res.XPassed[test] = o
+				res.XPassed[test] = tr
 			case Ignore, Unknown:
 				fallthrough
 			default:
-				res.Unknown[test] = o
+				res.Unknown[test] = tr
 			}
 
 		case Pass:
 			switch actualResult.Status {
 			case Fail:
-				res.XFailed[test] = o
+				res.XFailed[test] = tr
 			case Skip:
-				res.XSkipped[test] = o
+				res.XSkipped[test] = tr
 			case Pass:
-				res.Passed[test] = o
+				res.Passed[test] = tr
 			case Ignore, Unknown:
 				fallthrough
 			default:
-				res.Unknown[test] = o
+				res.Unknown[test] = tr
 			}
 
 		case Ignore:

internal/config/runner.go

@@ -24,9 +24,6 @@ const (
 	// RunnerTypeGoTest indicates a Go test runner.
 	RunnerTypeGoTest RunnerType = "gotest"
 
-	// RunnerTypeJSTest indicates a JavaScript test runner.
-	RunnerTypeJSTest RunnerType = "jstest"
-
 	// RunnerTypeYCSB indicates a YCSB test runner.
 	RunnerTypeYCSB RunnerType = "ycsb"
 )
@@ -63,8 +60,18 @@ type RunnerParamsGoTest struct {
 // runnerParams implements [RunnerParams] interface.
 func (rp *RunnerParamsGoTest) runnerParams() {}
 
+// RunnerParamsYCSB represents `ycsb` runner parameters.
+type RunnerParamsYCSB struct {
+	Dir  string
+	Args []string
+}
+
+// runnerParams implements [RunnerParams] interface.
+func (rp *RunnerParamsYCSB) runnerParams() {}
+
 // check interfaces
 var (
 	_ RunnerParams = (*RunnerParamsCommand)(nil)
 	_ RunnerParams = (*RunnerParamsGoTest)(nil)
+	_ RunnerParams = (*RunnerParamsYCSB)(nil)
 )

internal/configload/configload.go

@@ -163,10 +163,8 @@ func loadContent(content, db string) (*config.Config, error) {
 		p = &runnerParamsCommand{}
 	case config.RunnerTypeGoTest:
 		p = &runnerParamsGoTest{}
-	case config.RunnerTypeJSTest:
-		fallthrough
 	case config.RunnerTypeYCSB:
-		fallthrough
+		p = &runnerParamsYCSB{}
 	default:
 		err = fmt.Errorf("unknown runner type %q", pc.Runner)
 	}