Allow separate configurations (#19)

.github/dependabot.yml

@@ -24,7 +24,7 @@ updates:
       time: "04:42"
   - package-ecosystem: "gitsubmodule"
     directory: "/"
-    labels: [tests]
+    labels: [deps]
     schedule:
       interval: "weekly"
       day: "sunday"

.github/workflows/dance.yml

@@ -55,8 +55,21 @@ jobs:
       - name: Dance!
         run: make dance
         env:
+          DB: ${{ matrix.db }}
           TEST: ${{ matrix.test }}
 
+      - name: Collect logs
+        if: failure()
+        run: |
+          make env-logs > /tmp/compose-logs.txt
+
+      - name: Upload logs
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: compose-logs-${{ matrix.db }}-${{ matrix.test }}.txt
+          path: /tmp/compose-logs.txt
+
       - name: Check dirty
         run: |
           git status

.golangci.yml

@@ -13,7 +13,7 @@ linters-settings:
     exclude:
       - go-sumtype:decl
       - check interfaces
-    capital: true
+    capital: false # for unexported declarations
   goheader:
     template: |-
       Copyright 2021 FerretDB Inc.
@@ -38,7 +38,6 @@ linters-settings:
     line-length: 130
     tab-width: 1
   misspell:
-    locale: US
     ignore-words:
       - guiness # present in the test dataset
   nolintlint:
@@ -69,6 +68,7 @@ linters-settings:
   whitespace:
     multi-if: false
     multi-func: false
+
 linters:
   enable-all: true
   disable:

Makefile

@@ -17,6 +17,9 @@ env-up-detach:
 env-pull:
 	docker-compose pull --include-deps --quiet
 
+env-logs:
+	docker-compose logs --no-color
+
 env-down:                              ## Stop environment
 	docker-compose down --remove-orphans
 
@@ -31,7 +34,7 @@ fmt: bin/gofumpt                       ## Format code
 	bin/gofumpt -w ./cmd/ ./internal/
 
 dance:                                 ## Run all integration tests
-	cd tests && go run ../cmd/dance $(TEST)
+	cd tests && go run ../cmd/dance -db=$(DB) $(TEST)
 
 lint: bin/golangci-lint                ## Run linters
 	bin/golangci-lint run --config=.golangci-required.yml

cmd/dance/main.go

@@ -20,13 +20,15 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"os/signal"
 	"path/filepath"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/pmezard/go-difflib/difflib"
 	"golang.org/x/exp/maps"
+	"golang.org/x/sys/unix"
 	"gopkg.in/yaml.v3"
 
 	"github.com/FerretDB/dance/internal"
@@ -64,11 +66,17 @@ func logResult(label string, res map[string]string) {
 }
 
 func main() {
+	dbF := flag.String("db", "", "database to use: ferretdb, mongodb")
 	vF := flag.Bool("v", false, "be verbose")
 	log.SetFlags(0)
 	flag.Parse()
 
-	ctx := context.Background()
+	ctx, stop := signal.NotifyContext(context.Background(), unix.SIGTERM, unix.SIGINT)
+	go func() {
+		<-ctx.Done()
+		log.Print("Stopping...")
+		stop()
+	}()
 
 	log.Printf("Waiting for port 27017 to be up...")
 	if err := waitForPort(ctx, 27017); err != nil {
@@ -98,12 +106,17 @@ func main() {
 			log.Fatal(err)
 		}
 
-		runRes, err := gotest.Run(dir, config.Args, *vF)
+		expectedConfig, err := config.Results.ForDB(*dbF)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		compareRes, err := config.Tests.Compare(runRes)
+		runRes, err := gotest.Run(ctx, dir, config.Args, *vF)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		compareRes, err := expectedConfig.Compare(runRes)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -114,7 +127,7 @@ func main() {
 			sort.Strings(keys)
 			for _, t := range keys {
 				res := compareRes.UnexpectedRest[t]
-				log.Printf("%s %s:\n\t%s", t, res.Result, res.IndentedOutput())
+				log.Printf("%s %s:\n\t%s", t, res.Status, res.IndentedOutput())
 			}
 		}
 
@@ -136,7 +149,7 @@ func main() {
 		log.Printf("Expectedly skipped: %d.", len(compareRes.ExpectedSkip))
 		log.Printf("Expectedly passed: %d.", len(compareRes.ExpectedPass))
 
-		expectedStats, err := yaml.Marshal(config.Tests.Stats)
+		expectedStats, err := yaml.Marshal(expectedConfig.Stats)
 		if err != nil {
 			log.Fatal(err)
 		}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	go.mongodb.org/mongo-driver v1.8.1
 	golang.org/x/exp v0.0.0-20211221223016-e29036178569
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
 

go.sum

@@ -47,6 +47,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=

internal/config.go

@@ -18,16 +18,27 @@ package internal
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
 
+// Config represents dance configuration.
 type Config struct {
-	Runner string      `yaml:"runner"`
-	Args   []string    `yaml:"args"`
-	Tests  TestsConfig `yaml:"tests"`
+	Runner  string   `yaml:"runner"`
+	Args    []string `yaml:"args"`
+	Results Results  `yaml:"results"`
 }
 
+// Results represents expected dance results.
+type Results struct {
+	// Expected results for both FerretDB and MongoDB.
+	Common   *TestsConfig `yaml:"common"`
+	FerretDB *TestsConfig `yaml:"ferretdb"`
+	MongoDB  *TestsConfig `yaml:"mongodb"`
+}
+
+// Loadconfig loads and validates configuration from file.
 func LoadConfig(path string) (*Config, error) {
 	f, err := os.Open(path)
 	if err != nil {
@@ -43,17 +54,68 @@ func LoadConfig(path string) (*Config, error) {
 		return nil, fmt.Errorf("failed to decode config: %w", err)
 	}
 
-	if err = c.validate(); err != nil {
+	if err = c.fillAndValidate(); err != nil {
 		return nil, err
 	}
 
 	return &c, nil
 }
 
-func (c *Config) validate() error {
-	if _, err := c.Tests.toMap(); err != nil {
-		return err
+func (c *Config) fillAndValidate() error {
+	if c.Results.Common == nil {
+		if c.Results.FerretDB == nil || c.Results.MongoDB == nil {
+			return fmt.Errorf("both FerretDB and MongoDB results must be set (if common results are not set)")
+		}
+	} else {
+		if c.Results.FerretDB != nil || c.Results.MongoDB != nil {
+			return fmt.Errorf("common results are not allowed with FerretDB or MongoDB results")
+		}
+	}
+
+	for _, r := range []*TestsConfig{
+		c.Results.Common,
+		c.Results.FerretDB,
+		c.Results.MongoDB,
+	} {
+		if r == nil {
+			continue
+		}
+
+		if _, err := r.toMap(); err != nil {
+			return err
+		}
+
+		origDefault := r.Default
+		r.Default = status(strings.ToLower(string(origDefault)))
+		if r.Default == "" {
+			r.Default = Pass
+		}
+
+		if _, ok := knownStatuses[r.Default]; !ok {
+			return fmt.Errorf("invalid default result: %q", origDefault)
+		}
 	}
 
 	return nil
 }
+
+func (r *Results) ForDB(db string) (*TestsConfig, error) {
+	switch db {
+	case "ferretdb":
+		if c := r.FerretDB; c != nil {
+			return c, nil
+		}
+	case "mongodb":
+		if c := r.MongoDB; c != nil {
+			return c, nil
+		}
+	default:
+		return nil, fmt.Errorf("unknown database %q", db)
+	}
+
+	if c := r.Common; c != nil {
+		return c, nil
+	}
+
+	return nil, fmt.Errorf("no expected results for %q", db)
+}

internal/gotest/gotest.go

@@ -16,6 +16,7 @@
 package gotest
 
 import (
+	"context"
 	"encoding/json"
 	"io"
 	"os"
@@ -24,7 +25,10 @@ import (
 	"github.com/FerretDB/dance/internal"
 )
 
-func Run(dir string, args []string, verbose bool) (*internal.Results, error) {
+func Run(ctx context.Context, dir string, args []string, verbose bool) (*internal.TestResults, error) {
+	// TODO https://github.com/FerretDB/dance/issues/20
+	_ = ctx
+
 	args = append([]string{"test", "-v", "-json", "-count=1"}, args...)
 	cmd := exec.Command("go", args...)
 	cmd.Dir = dir
@@ -46,33 +50,38 @@ func Run(dir string, args []string, verbose bool) (*internal.Results, error) {
 	d := json.NewDecoder(r)
 	d.DisallowUnknownFields()
 
-	res := &internal.Results{
+	res := &internal.TestResults{
 		TestResults: make(map[string]internal.TestResult),
 	}
 
 	for {
-		var event TestEvent
+		var event testEvent
 		if err = d.Decode(&event); err != nil {
 			if err == io.EOF {
 				break
 			}
 			return nil, err
 		}
 
+		// skip package failures
+		if event.Test == "" {
+			continue
+		}
+
 		testName := event.Package + "/" + event.Test
 		result := res.TestResults[testName]
 		result.Output += event.Output
 		switch event.Action {
-		case ActionPass:
-			result.Result = internal.Pass
-		case ActionFail:
-			result.Result = internal.Fail
-		case ActionSkip:
-			result.Result = internal.Skip
-		case ActionBench, ActionCont, ActionOutput, ActionPause, ActionRun:
+		case actionPass:
+			result.Status = internal.Pass
+		case actionFail:
+			result.Status = internal.Fail
+		case actionSkip:
+			result.Status = internal.Skip
+		case actionBench, actionCont, actionOutput, actionPause, actionRun:
 			fallthrough
 		default:
-			result.Result = internal.Unknown
+			result.Status = internal.Unknown
 		}
 		res.TestResults[testName] = result
 	}

internal/gotest/testjson.go

@@ -16,36 +16,36 @@ package gotest
 
 import "time"
 
-type Action string
+type action string
 
 const (
-	// ActionRun means the test has started running.
-	ActionRun Action = "run"
-	// ActionPause means the test has been paused.
-	ActionPause Action = "pause"
-	// ActionCont means the test has continued running.
-	ActionCont Action = "cont"
-	// ActionPass means the test passed.
-	ActionPass Action = "pass"
-	// ActionBench means the benchmark printed log output but did not fail.
-	ActionBench Action = "bench"
-	// ActionFail means the test or benchmark failed.
-	ActionFail Action = "fail"
-	// ActionOutput means the test printed output.
-	ActionOutput Action = "output"
-	// ActionSkip means the test was skipped or the package contained no tests.
-	ActionSkip Action = "skip"
+	// actionRun means the test has started running.
+	actionRun action = "run"
+	// actionPause means the test has been paused.
+	actionPause action = "pause"
+	// actionCont means the test has continued running.
+	actionCont action = "cont"
+	// actionPass means the test passed.
+	actionPass action = "pass"
+	// actionBench means the benchmark printed log output but did not fail.
+	actionBench action = "bench"
+	// actionFail means the test or benchmark failed.
+	actionFail action = "fail"
+	// actionOutput means the test printed output.
+	actionOutput action = "output"
+	// actionSkip means the test was skipped or the package contained no tests.
+	actionSkip action = "skip"
 )
 
-type TestEvent struct {
+type testEvent struct {
 	Time           time.Time `json:"Time"`
-	Action         Action    `json:"Action"`
+	Action         action    `json:"Action"`
 	Package        string    `json:"Package"`
 	Test           string    `json:"Test"`
 	Output         string    `json:"Output"`
 	ElapsedSeconds float64   `json:"Elapsed"`
 }
 
-func (te TestEvent) Elapsed() time.Duration {
+func (te testEvent) Elapsed() time.Duration {
 	return time.Duration(te.ElapsedSeconds * float64(time.Second))
 }