From 9c5139324ccaafa700544d09a9f47c85359ac44f Mon Sep 17 00:00:00 2001 From: Marcin Owsiany Date: Fri, 7 Apr 2017 17:41:05 +0200 Subject: [PATCH] [WiP] tolerate more than one copy of ginkgo in a binary. This is a proof of concept for https://github.com/kubernetes/ingress/issues/5#issuecomment-290950754 for only one of the flags. --- CHANGELOG.md | 2 +- LICENSE | 1 + config/config.go | 29 ++++++++++++++++++++++--- ginkgo/run_command.go | 4 ++-- ginkgo/watch_command.go | 2 +- ginkgo_dsl.go | 2 +- internal/remote/aggregator.go | 2 +- internal/remote/aggregator_test.go | 2 +- internal/specrunner/spec_runner_test.go | 4 ++-- internal/suite/suite.go | 4 ++-- internal/suite/suite_test.go | 4 ++-- reporters/default_reporter.go | 2 +- reporters/default_reporter_test.go | 4 ++-- 13 files changed, 43 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 154f4cd40..c26bb8cac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Improvements: - Significantly improved parallel test distribution. Now instead of pre-sharding test cases across workers (which can result in idle workers and poor test performance) Ginkgo uses a shared queue to keep all workers busy until all tests are complete. This improves test-time performance and consistency. - `Skip(message)` can be used to skip the current test. - Added `extensions/table` - a Ginkgo DSL for [Table Driven Tests](http://onsi.github.io/ginkgo/#table-driven-tests) -- Add `GinkgoRandomSeed()` - shorthand for `config.GinkgoConfig.RandomSeed` +- Add `GinkgoRandomSeed()` - shorthand for `config.GinkgoConfig.GetRandomSeed()` - Support for retrying flaky tests with `--flakeAttempts` - `ginkgo ./...` now recurses as you'd expect - Added `Specify` a synonym for `It` diff --git a/LICENSE b/LICENSE index 9415ee72c..0194326ff 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ Copyright (c) 2013-2014 Onsi Fakhouri +Copyright 2017 Google Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/config/config.go b/config/config.go index 716484e05..5cba40f2c 100644 --- a/config/config.go +++ b/config/config.go @@ -23,7 +23,9 @@ import ( const VERSION = "1.3.1" type GinkgoConfigType struct { + // Deprecated: use GetRandomSeed() and SetRandomSeed() RandomSeed int64 + randomSeedGetter flag.Getter RandomizeAllSpecs bool RegexScansFilePath bool FocusString string @@ -43,6 +45,22 @@ type GinkgoConfigType struct { var GinkgoConfig = GinkgoConfigType{} +func (config *GinkgoConfigType) GetRandomSeed() int64 { + if config.randomSeedGetter != nil { + return config.randomSeedGetter.Get().(int64) + } else { + return config.RandomSeed + } +} + +func (config *GinkgoConfigType) SetRandomSeed(value int64) { + if config.randomSeedGetter != nil { + config.randomSeedGetter.Set(string(value)) + } else { + config.RandomSeed = value + } +} + type DefaultReporterConfigType struct { NoColor bool SlowSpecThreshold float64 @@ -63,7 +81,12 @@ func processPrefix(prefix string) string { func Flags(flagSet *flag.FlagSet, prefix string, includeParallelFlags bool) { prefix = processPrefix(prefix) - flagSet.Int64Var(&(GinkgoConfig.RandomSeed), prefix+"seed", time.Now().Unix(), "The seed used to randomize the spec suite.") + randomSeedFlag := flag.Lookup(prefix+"seed") + if randomSeedFlag == nil { + flagSet.Int64Var(&(GinkgoConfig.RandomSeed), prefix+"seed", time.Now().Unix(), "The seed used to randomize the spec suite.") + } else { + GinkgoConfig.randomSeedGetter = (*randomSeedFlag).Value.(flag.Getter) + } flagSet.BoolVar(&(GinkgoConfig.RandomizeAllSpecs), prefix+"randomizeAllSpecs", false, "If set, ginkgo will randomize all specs together. By default, ginkgo only randomizes the top level Describe/Context groups.") flagSet.BoolVar(&(GinkgoConfig.SkipMeasurements), prefix+"skipMeasurements", false, "If set, ginkgo will skip any measurement specs.") flagSet.BoolVar(&(GinkgoConfig.FailOnPending), prefix+"failOnPending", false, "If set, ginkgo will mark the test suite as failed if any specs are pending.") @@ -99,8 +122,8 @@ func BuildFlagArgs(prefix string, ginkgo GinkgoConfigType, reporter DefaultRepor prefix = processPrefix(prefix) result := make([]string, 0) - if ginkgo.RandomSeed > 0 { - result = append(result, fmt.Sprintf("--%sseed=%d", prefix, ginkgo.RandomSeed)) + if ginkgo.GetRandomSeed() > 0 { + result = append(result, fmt.Sprintf("--%sseed=%d", prefix, ginkgo.GetRandomSeed())) } if ginkgo.RandomizeAllSpecs { diff --git a/ginkgo/run_command.go b/ginkgo/run_command.go index 9f19192ac..48ec0b849 100644 --- a/ginkgo/run_command.go +++ b/ginkgo/run_command.go @@ -138,7 +138,7 @@ func (r *SpecRunner) ComputeSuccinctMode(numSuites int) { func (r *SpecRunner) UpdateSeed() { if !r.commandFlags.wasSet("seed") { - config.GinkgoConfig.RandomSeed = time.Now().Unix() + config.GinkgoConfig.SetRandomSeed(time.Now().Unix()) } } @@ -152,7 +152,7 @@ func (r *SpecRunner) randomizeOrder(runners []*testrunner.TestRunner) []*testrun } randomizedRunners := make([]*testrunner.TestRunner, len(runners)) - randomizer := rand.New(rand.NewSource(config.GinkgoConfig.RandomSeed)) + randomizer := rand.New(rand.NewSource(config.GinkgoConfig.GetRandomSeed())) permutation := randomizer.Perm(len(runners)) for i, j := range permutation { randomizedRunners[i] = runners[j] diff --git a/ginkgo/watch_command.go b/ginkgo/watch_command.go index 0dc2c5cc9..e1fbec337 100644 --- a/ginkgo/watch_command.go +++ b/ginkgo/watch_command.go @@ -169,6 +169,6 @@ func (w *SpecWatcher) ComputeSuccinctMode(numSuites int) { func (w *SpecWatcher) UpdateSeed() { if !w.commandFlags.wasSet("seed") { - config.GinkgoConfig.RandomSeed = time.Now().Unix() + config.GinkgoConfig.SetRandomSeed(time.Now().Unix()) } } diff --git a/ginkgo_dsl.go b/ginkgo_dsl.go index 8fe0b70a6..dd7871715 100644 --- a/ginkgo_dsl.go +++ b/ginkgo_dsl.go @@ -73,7 +73,7 @@ type GinkgoTestingT interface { //consistent executions from run to run, where your tests contain variability (for //example, when selecting random test data). func GinkgoRandomSeed() int64 { - return config.GinkgoConfig.RandomSeed + return config.GinkgoConfig.GetRandomSeed() } //GinkgoParallelNode returns the parallel node number for the current ginkgo process diff --git a/internal/remote/aggregator.go b/internal/remote/aggregator.go index 522d44e35..73c3dc819 100644 --- a/internal/remote/aggregator.go +++ b/internal/remote/aggregator.go @@ -123,7 +123,7 @@ func (aggregator *Aggregator) registerSuiteBeginning(configAndSuite configAndSui return } - aggregator.stenographer.AnnounceSuite(configAndSuite.summary.SuiteDescription, configAndSuite.config.RandomSeed, configAndSuite.config.RandomizeAllSpecs, aggregator.config.Succinct) + aggregator.stenographer.AnnounceSuite(configAndSuite.summary.SuiteDescription, configAndSuite.config.GetRandomSeed(), configAndSuite.config.RandomizeAllSpecs, aggregator.config.Succinct) totalNumberOfSpecs := 0 if len(aggregator.aggregatedSuiteBeginnings) > 0 { diff --git a/internal/remote/aggregator_test.go b/internal/remote/aggregator_test.go index aedf93927..b45e3465e 100644 --- a/internal/remote/aggregator_test.go +++ b/internal/remote/aggregator_test.go @@ -135,7 +135,7 @@ var _ = Describe("Aggregator", func() { It("should announce the beginning of the suite", func() { Ω(stenographer.Calls()).Should(HaveLen(3)) - Ω(stenographer.Calls()[0]).Should(Equal(call("AnnounceSuite", suiteDescription, ginkgoConfig1.RandomSeed, true, false))) + Ω(stenographer.Calls()[0]).Should(Equal(call("AnnounceSuite", suiteDescription, ginkgoConfig1.GetRandomSeed(), true, false))) Ω(stenographer.Calls()[1]).Should(Equal(call("AnnounceTotalNumberOfSpecs", 30, false))) Ω(stenographer.Calls()[2]).Should(Equal(call("AnnounceAggregatedParallelRun", 2, false))) }) diff --git a/internal/specrunner/spec_runner_test.go b/internal/specrunner/spec_runner_test.go index 77272e8b8..6c8702e87 100644 --- a/internal/specrunner/spec_runner_test.go +++ b/internal/specrunner/spec_runner_test.go @@ -157,7 +157,7 @@ var _ = Describe("Spec Runner", func() { }) It("should report the passed in config", func() { - Ω(reporter1.Config.RandomSeed).Should(BeNumerically("==", 17)) + Ω(reporter1.Config.GetRandomSeed()).Should(BeNumerically("==", 17)) }) It("should report the beginning of the suite", func() { @@ -350,7 +350,7 @@ var _ = Describe("Spec Runner", func() { }) It("should report the passed in config", func() { - Ω(reporter1.Config.RandomSeed).Should(BeNumerically("==", 17)) + Ω(reporter1.Config.GetRandomSeed()).Should(BeNumerically("==", 17)) }) It("should report the beginning of the suite", func() { diff --git a/internal/suite/suite.go b/internal/suite/suite.go index 698a6e568..c56ed02e3 100644 --- a/internal/suite/suite.go +++ b/internal/suite/suite.go @@ -53,7 +53,7 @@ func (suite *Suite) Run(t ginkgoTestingT, description string, reporters []report panic("ginkgo.parallel.node is one-indexed and must be <= ginkgo.parallel.total") } - r := rand.New(rand.NewSource(config.RandomSeed)) + r := rand.New(rand.NewSource(config.GetRandomSeed())) suite.topLevelContainer.Shuffle(r) iterator, hasProgrammaticFocus := suite.generateSpecsIterator(description, config) suite.runner = specrunner.New(description, suite.beforeSuiteNode, iterator, suite.afterSuiteNode, reporters, writer, config) @@ -77,7 +77,7 @@ func (suite *Suite) generateSpecsIterator(description string, config config.Gink specs.RegexScansFilePath = config.RegexScansFilePath if config.RandomizeAllSpecs { - specs.Shuffle(rand.New(rand.NewSource(config.RandomSeed))) + specs.Shuffle(rand.New(rand.NewSource(config.GetRandomSeed()))) } specs.ApplyFocus(description, config.FocusString, config.SkipString) diff --git a/internal/suite/suite_test.go b/internal/suite/suite_test.go index b7bcdbd2e..7a5e983d0 100644 --- a/internal/suite/suite_test.go +++ b/internal/suite/suite_test.go @@ -98,7 +98,7 @@ var _ = Describe("Suite", func() { }) It("provides the config and suite description to the reporter", func() { - Ω(fakeR.Config.RandomSeed).Should(Equal(int64(randomSeed))) + Ω(fakeR.Config.GetRandomSeed()).Should(Equal(int64(randomSeed))) Ω(fakeR.Config.RandomizeAllSpecs).Should(Equal(randomizeAllSpecs)) Ω(fakeR.BeginSummary.SuiteDescription).Should(Equal("suite description")) }) @@ -365,7 +365,7 @@ var _ = Describe("Suite", func() { Describe("GinkgoRandomSeed", func() { It("returns the current config's random seed", func() { - Ω(GinkgoRandomSeed()).Should(Equal(config.GinkgoConfig.RandomSeed)) + Ω(GinkgoRandomSeed()).Should(Equal(config.GinkgoConfig.GetRandomSeed())) }) }) }) diff --git a/reporters/default_reporter.go b/reporters/default_reporter.go index fb82f70a6..4fac25d6a 100644 --- a/reporters/default_reporter.go +++ b/reporters/default_reporter.go @@ -27,7 +27,7 @@ func NewDefaultReporter(config config.DefaultReporterConfigType, stenographer st } func (reporter *DefaultReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { - reporter.stenographer.AnnounceSuite(summary.SuiteDescription, config.RandomSeed, config.RandomizeAllSpecs, reporter.config.Succinct) + reporter.stenographer.AnnounceSuite(summary.SuiteDescription, config.GetRandomSeed(), config.RandomizeAllSpecs, reporter.config.Succinct) if config.ParallelTotal > 1 { reporter.stenographer.AnnounceParallelRun(config.ParallelNode, config.ParallelTotal, reporter.config.Succinct) } else { diff --git a/reporters/default_reporter_test.go b/reporters/default_reporter_test.go index 01528448e..9c291164b 100644 --- a/reporters/default_reporter_test.go +++ b/reporters/default_reporter_test.go @@ -62,7 +62,7 @@ var _ = Describe("DefaultReporter", func() { It("should announce the suite, then announce the number of specs", func() { Ω(stenographer.Calls()).Should(HaveLen(2)) - Ω(stenographer.Calls()[0]).Should(Equal(call("AnnounceSuite", "A Sweet Suite", ginkgoConfig.RandomSeed, true, false))) + Ω(stenographer.Calls()[0]).Should(Equal(call("AnnounceSuite", "A Sweet Suite", ginkgoConfig.GetRandomSeed(), true, false))) Ω(stenographer.Calls()[1]).Should(Equal(call("AnnounceNumberOfSpecs", 8, 10, false))) }) }) @@ -78,7 +78,7 @@ var _ = Describe("DefaultReporter", func() { It("should announce the suite, announce that it's a parallel run, then announce the number of specs", func() { Ω(stenographer.Calls()).Should(HaveLen(2)) - Ω(stenographer.Calls()[0]).Should(Equal(call("AnnounceSuite", "A Sweet Suite", ginkgoConfig.RandomSeed, true, false))) + Ω(stenographer.Calls()[0]).Should(Equal(call("AnnounceSuite", "A Sweet Suite", ginkgoConfig.GetRandomSeed(), true, false))) Ω(stenographer.Calls()[1]).Should(Equal(call("AnnounceParallelRun", 1, 2, false))) }) })