Add flag to enable persistence support

docs/running.md

@@ -16,6 +16,7 @@ Options besides `-url`:
 * `-junit <FILEPATH>` - writes test results in JUnit XML format to the specified file
 * `-debug` - enables verbose logging of test actions for failed tests
 * `-debug-all` - enables verbose logging of test actions for all tests
+* `-enable-persistence-tests` - enables tests that require external persistence (e.g. a database like redis)
 * `-record-failures` - record failed test IDs to the given file. Recorded tests can be skipped by the next run of 
 the harness via `-skip-from`.
 * `-skip-from` - skips any test IDs recorded in the specified file. May be used in conjunction with `-record-failures`

docs/service_spec.md

@@ -110,6 +110,16 @@ This means the SDK is capable of interacting with external persistent data store
 - `dynamodb`
 - `redis`
 
+The persistence store tests are unique in that they rely on external services to be available prior to running the test harness. Because of this, the test harness will not run these tests by default, even if the capabilities exist, without also provided the `-enable-persistence-tests` flag.
+
+These services can be easily provided through the use of docker. Example commands are shown below for your convenience.
+
+```shell
+docker run --network=host amazon/dynamodb-local
+docker run --network=host hashicorp/consul
+docker run --network host redis
+```
+
 #### Capability `"polling-gzip"`
 
 This means the SDK is requesting gzip compression support on polling payloads. The SDK is expected to set the `Accept-Encoding` header to `gzip` in addition to enabling this capability.

framework/harness/harness.go

@@ -108,6 +108,7 @@ func NewTestHarness(
 	testServiceBaseURL string,
 	testHarnessExternalHostname string,
 	testHarnessPort int,
+	testHarnessEnablePersistenceTests bool,
 	statusQueryTimeout time.Duration,
 	debugLogger framework.Logger,
 	startupOutput io.Writer,
@@ -129,6 +130,21 @@ func NewTestHarness(
 	if err != nil {
 		return nil, err
 	}
+
+	// If we aren't running persistence tests, remove the capabilities that would enable it.
+	if !testHarnessEnablePersistenceTests {
+		filteredCapabilities := make([]string, 0, len(testServiceInfo.Capabilities))
+		for _, c := range testServiceInfo.Capabilities {
+			if c == servicedef.CapabilityPersistentDataStoreRedis ||
+				c == servicedef.CapabilityPersistentDataStoreDynamoDB ||
+				c == servicedef.CapabilityPersistentDataStoreConsul {
+				debugLogger.Printf("Disabling capability %q because persistence tests are disabled", c)
+				continue
+			}
+			filteredCapabilities = append(filteredCapabilities, c)
+		}
+		testServiceInfo.Capabilities = filteredCapabilities
+	}
 	h.testServiceInfo = testServiceInfo
 
 	if err := startServer(testHarnessPort, http.HandlerFunc(h.serveHTTP)); err != nil {

main.go

@@ -55,6 +55,7 @@ func run(params commandParams) (*ldtest.Results, error) {
 		params.serviceURL,
 		params.host,
 		params.port,
+		params.enablePersistenceTests,
 		time.Duration(params.queryTimeoutSeconds)*time.Second,
 		mainDebugLogger,
 		os.Stdout,

params.go

@@ -9,17 +9,18 @@ import (
 )
 
 type commandParams struct {
-	serviceURL          string
-	port                int
-	host                string
-	filters             ldtest.RegexFilters
-	stopServiceAtEnd    bool
-	debug               bool
-	debugAll            bool
-	jUnitFile           string
-	recordFailures      string
-	skipFile            string
-	queryTimeoutSeconds int
+	serviceURL             string
+	port                   int
+	host                   string
+	filters                ldtest.RegexFilters
+	stopServiceAtEnd       bool
+	debug                  bool
+	debugAll               bool
+	enablePersistenceTests bool
+	jUnitFile              string
+	recordFailures         string
+	skipFile               string
+	queryTimeoutSeconds    int
 }
 
 func (c *commandParams) Read(args []string) bool {
@@ -33,6 +34,7 @@ func (c *commandParams) Read(args []string) bool {
 	fs.BoolVar(&c.stopServiceAtEnd, "stop-service-at-end", false, "tell test service to exit after the test run")
 	fs.BoolVar(&c.debug, "debug", false, "enable debug logging for failed tests")
 	fs.BoolVar(&c.debugAll, "debug-all", false, "enable debug logging for all tests")
+	fs.BoolVar(&c.enablePersistenceTests, "enable-persistence-tests", false, "enable tests that require external persistence support")
 	fs.StringVar(&c.jUnitFile, "junit", "", "write JUnit XML output to the specified path")
 	fs.StringVar(&c.recordFailures, "record-failures", "", "record failed test IDs to the given file.\n"+
 		"recorded tests can be skipped by the next run of the harness via -skip-from")

sdktests/server_side_persistence_base.go

@@ -41,7 +41,10 @@ const (
 )
 
 func doServerSidePersistentTests(t *ldtest.T) {
+	ranAtLeastOnce := false
+
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreRedis) {
+		ranAtLeastOnce = true
 		rdb := redis.NewClient(&redis.Options{
 			Addr:     "localhost:6379",
 			Password: "", // no password set
@@ -52,6 +55,7 @@ func doServerSidePersistentTests(t *ldtest.T) {
 	}
 
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreConsul) {
+		ranAtLeastOnce = true
 		config := consul.DefaultConfig()
 		config.Address = "localhost:8500"
 
@@ -62,6 +66,7 @@ func doServerSidePersistentTests(t *ldtest.T) {
 	}
 
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreDynamoDB) {
+		ranAtLeastOnce = true
 		mySession := session.Must(session.NewSession(
 			aws.NewConfig().
 				WithRegion("us-east-1").
@@ -81,6 +86,10 @@ func doServerSidePersistentTests(t *ldtest.T) {
 
 		t.Run("dynamodb", newServerSidePersistentTests(t, &store, "").Run)
 	}
+
+	if !ranAtLeastOnce {
+		t.Skip()
+	}
 }
 
 type PersistentStore interface {

sdktests/testsuite_entry_point.go

@@ -92,12 +92,7 @@ func doAllServerSideTests(t *ldtest.T) {
 	t.Run("migrations", doServerSideMigrationTests)
 	t.Run("hooks", doCommonHooksTests)
 	t.Run("wrapper", doServerSideWrapperTests)
-
-	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreRedis) ||
-		t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreConsul) ||
-		t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreDynamoDB) {
-		t.Run("persistent data store", doServerSidePersistentTests)
-	}
+	t.Run("persistent data store", doServerSidePersistentTests)
 }
 
 func doAllClientSideTests(t *ldtest.T) {