diff --git a/build/testdata/schema.json b/build/testdata/schema.json index 14ae7b501..a6c526b79 100644 --- a/build/testdata/schema.json +++ b/build/testdata/schema.json @@ -1 +1,5 @@ -{"claims":"cnab-claim-1.0.0-DRAFT+b5ed2f3","credentials":"cnab-credentials-1.0.0-DRAFT-b6c701f"} \ No newline at end of file +{ + "claims": "cnab-claim-1.0.0-DRAFT+b5ed2f3", + "credentials": "cnab-credentialsets-1.0.0-DRAFT+b6c701f", + "parameters": "cnab-parametersets-1.0.0-DRAFT+TODO" +} \ No newline at end of file diff --git a/go.mod b/go.mod index 0cbbb6eb6..680d083db 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module get.porter.sh/porter go 1.13 replace ( + // https://github.com/cnabio/cnab-go/pull/229 (valueset-schema) + github.com/cnabio/cnab-go => github.com/carolynvs/cnab-go v0.13.4-0.20200820201933-d6bf372247e5 + // See https://github.com/containerd/containerd/issues/3031 // When I try to just use the require, go is shortening it to v2.7.1+incompatible which then fails to build... github.com/docker/distribution => github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible diff --git a/pkg/claims/claimStorage_test.go b/pkg/claims/claimStorage_test.go index 2df3641d2..6996964c9 100644 --- a/pkg/claims/claimStorage_test.go +++ b/pkg/claims/claimStorage_test.go @@ -22,7 +22,7 @@ func TestClaimStorage_HaltOnMigrationRequired(t *testing.T) { // Add an unmigrated claim claimsDir := filepath.Join(home, "claims") config.FileSystem.Mkdir(claimsDir, 0755) - config.TestContext.AddTestFile(filepath.Join("../storage/testdata", "upgraded.json"), filepath.Join(home, "claims", "mybun.json")) + config.TestContext.AddTestFile(filepath.Join("../storage/testdata/claims", "upgraded.json"), filepath.Join(home, "claims", "mybun.json")) dataStore := filesystem.NewStore(*config.Config, hclog.NewNullLogger()) mgr := storage.NewManager(config.Config, dataStore) @@ -31,14 +31,16 @@ func TestClaimStorage_HaltOnMigrationRequired(t *testing.T) { var err error t.Run("list", func(t *testing.T) { _, err = claimStore.ListInstallations() + require.Error(t, err, "Operation should halt because a migration is required") + assert.Contains(t, err.Error(), "The schema of Porter's data is in an older format than supported by this version of Porter") }) t.Run("read", func(t *testing.T) { _, err = claimStore.ReadInstallation("mybun") + require.Error(t, err, "Operation should halt because a migration is required") + assert.Contains(t, err.Error(), "The schema of Porter's data is in an older format than supported by this version of Porter") }) - require.Error(t, err, "Operation should halt because a migration is required") - assert.Contains(t, err.Error(), "The schema of Porter's data is in an older format than supported by this version of Porter") } func TestClaimStorage_OperationAllowedWhenNoMigrationDetected(t *testing.T) { diff --git a/pkg/cnab/provider/credentials_test.go b/pkg/cnab/provider/credentials_test.go index f837c15e9..398b999fe 100644 --- a/pkg/cnab/provider/credentials_test.go +++ b/pkg/cnab/provider/credentials_test.go @@ -2,7 +2,6 @@ package cnabprovider import ( "testing" - "time" "get.porter.sh/porter/pkg/secrets" "github.com/cnabio/cnab-go/bundle" @@ -20,20 +19,15 @@ func TestRuntime_loadCredentials(t *testing.T) { r.TestConfig.TestContext.AddTestFile("testdata/db-creds.json", "/db-creds.json") - cs1 := credentials.CredentialSet{ - Name: "mycreds", - Created: time.Now(), - Modified: time.Now(), - Credentials: []valuesource.Strategy{ - { - Name: "password", - Source: valuesource.Source{ - Key: secrets.SourceSecret, - Value: "password", - }, + cs1 := credentials.NewCredentialSet("mycreds", + valuesource.Strategy{ + Name: "password", + Source: valuesource.Source{ + Key: secrets.SourceSecret, + Value: "password", }, - }, - } + }) + err := r.credentials.Save(cs1) require.NoError(t, err, "Save credential set failed") diff --git a/pkg/credentials/credentialMigration.go b/pkg/credentials/credentialMigration.go deleted file mode 100644 index 86e5d027d..000000000 --- a/pkg/credentials/credentialMigration.go +++ /dev/null @@ -1,108 +0,0 @@ -package credentials - -import ( - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - "get.porter.sh/porter/pkg/context" - "github.com/cnabio/cnab-go/credentials" - "github.com/hashicorp/go-multierror" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" -) - -// CredentialsMigration checks for legacy credentials in yaml format and makes a copy of them -// in json format next to them. This helps with porter switching formats without forcing a hard -// switch and making people do the migration themselves. -type credentialsMigration struct { - *context.Context - migrated []string -} - -func NewCredentialsMigration(c *context.Context) *credentialsMigration { - return &credentialsMigration{Context: c} -} - -// Migrate accepts a path to a credential set directory and migrates every yaml file to json that doesn't already -// have a matching destination file (NAME.json). -func (m *credentialsMigration) Migrate(credsDir string) error { - if hasCreds, _ := m.FileSystem.Exists(credsDir); !hasCreds { - return nil - } - - credFiles, err := m.FileSystem.ReadDir(credsDir) - if err != nil { - return err - } - - lookup := make(map[string]struct{}, len(credFiles)) - for _, f := range credFiles { - lookup[f.Name()] = struct{}{} - } - - var convertErrs error - for name := range lookup { - if filepath.Ext(name) == ".yaml" { - migratedName := strings.TrimSuffix(name, ".yaml") + ".json" - if _, isMigrated := lookup[migratedName]; !isMigrated { - legacyCredFile := filepath.Join(credsDir, name) - err = m.ConvertToJson(legacyCredFile) - if err != nil { - convertErrs = multierror.Append(convertErrs, err) - } - lookup[migratedName] = struct{}{} - } - } - } - - return convertErrs -} - -// ConvertToJson accepts a path to a credential set formatted as yaml, and migrates it to json. -func (m *credentialsMigration) ConvertToJson(path string) error { - if m.Debug { - credName := filepath.Base(path) - fmt.Fprintf(m.Err, "Converting credential %s from yaml to json. The old file is left next to it so you can remove it when you are sure you don't need it anymore.", credName) - } - - b, err := m.FileSystem.ReadFile(path) - if err != nil { - return errors.Wrapf(err, "could not read credentials at %s", path) - } - - var cs credentials.CredentialSet - err = yaml.Unmarshal(b, &cs) - if err != nil { - return errors.Wrapf(err, "could not parse credentials at %s", path) - } - - fi, err := m.FileSystem.Stat(path) - if err != nil { - // Oh well, we tried, stamp them and move on - cs.Modified = time.Now() - cs.Created = cs.Modified - } else { - cs.Modified = fi.ModTime() - cs.Created = cs.Modified // Created isn't available on all OS's so just set them the same - } - - b, err = json.Marshal(cs) - if err != nil { - return errors.Wrapf(err, "could not convert credentials at %s to json", path) - } - - // Change the filename from *.yaml to *.json - destDir, destName := filepath.Split(path) - destName = strings.TrimSuffix(destName, ".yaml") + ".json" - dest := filepath.Join(destDir, destName) - - err = m.FileSystem.WriteFile(dest, b, 0644) - if err != nil { - errors.Wrapf(err, "could not save migrated credentials to %s", dest) - } - m.migrated = append(m.migrated, path) - return nil -} diff --git a/pkg/credentials/credentialMigration_test.go b/pkg/credentials/credentialMigration_test.go deleted file mode 100644 index 2edddc25f..000000000 --- a/pkg/credentials/credentialMigration_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package credentials - -import ( - "io/ioutil" - "testing" - "time" - - "get.porter.sh/porter/pkg/context" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCredentialsMigration_ConvertToJson(t *testing.T) { - c := context.NewTestContext(t) - - oldCS := "/home/.porter/credentials/mybuns.yaml" - c.AddTestFile("testdata/mybuns.yaml", oldCS) - lastMod, _ := time.Parse("2006-Jan-02", "2020-Jan-01") - c.FileSystem.Chtimes(oldCS, lastMod, lastMod) - - m := NewCredentialsMigration(c.Context) - err := m.ConvertToJson(oldCS) - require.NoError(t, err, "ConvertToJson failed") - - newCS := "/home/.porter/credentials/mybuns.json" - exists, _ := c.FileSystem.Exists(newCS) - assert.True(t, exists, "expected migrated credential set to exist at %s", newCS) - exists, _ = c.FileSystem.Exists(oldCS) - assert.True(t, exists, "expected old credential set to still exist at %s", oldCS) - - want, err := ioutil.ReadFile("testdata/mybuns.json") - require.NoError(t, err, "could not read testdata/mybuns.json") - got, err := c.FileSystem.ReadFile(newCS) - require.NoError(t, err, "could not read %s", newCS) - assert.Equal(t, string(want), string(got)) - - require.Len(t, m.migrated, 1, "the wrong number of credential set paths were audited") - assert.Equal(t, oldCS, m.migrated[0], "invalid credential set path was audited") -} - -func TestCredentialsMigration_Migrate_ExistingFileSkipped(t *testing.T) { - c := context.NewTestContext(t) - - oldCS := "/home/.porter/credentials/mybuns.yaml" - existingCS := "/home/.porter/credentials/mybuns.json" - c.AddTestFile("testdata/mybuns.yaml", oldCS) - c.AddTestFile("testdata/mybuns.json", existingCS) - - fi, _ := c.FileSystem.Stat(existingCS) - wantMod := fi.ModTime() - - m := NewCredentialsMigration(c.Context) - err := m.Migrate("/home/.porter/credentials/") - require.NoError(t, err, "ConvertToJson failed") - - assert.Empty(t, m.migrated, "the credential set should not have been audited as migrated") - - fi, _ = c.FileSystem.Stat(existingCS) - gotMod := fi.ModTime() - assert.Equal(t, wantMod, gotMod, "the existing migrated credential set should not have been touched") -} - -func TestCredentialsMigration_Migrate(t *testing.T) { - c := context.NewTestContext(t) - - c.AddTestFile("testdata/mybuns.yaml", "/home/.porter/credentials/bun1.yaml") - c.AddTestFile("testdata/mybuns.yaml", "/home/.porter/credentials/bun2.yaml") - c.AddTestFile("testdata/mybuns.json", "/home/.porter/credentials/bun1.json") - - m := NewCredentialsMigration(c.Context) - err := m.Migrate("/home/.porter/credentials/") - require.NoError(t, err, "ConvertToJson failed") - - require.Len(t, m.migrated, 1, "the migrated credential sets should have been audited") - assert.Equal(t, "/home/.porter/credentials/bun2.yaml", m.migrated[0], "invalid credential set path was audited") -} diff --git a/pkg/credentials/credentialStorage_test.go b/pkg/credentials/credentialStorage_test.go index 937e8105e..40c43afd4 100644 --- a/pkg/credentials/credentialStorage_test.go +++ b/pkg/credentials/credentialStorage_test.go @@ -1,31 +1,34 @@ package credentials import ( + "path/filepath" "testing" + "get.porter.sh/porter/pkg/config" + "get.porter.sh/porter/pkg/storage" + "get.porter.sh/porter/pkg/storage/filesystem" "github.com/cnabio/cnab-go/credentials" "github.com/cnabio/cnab-go/valuesource" + "github.com/hashicorp/go-hclog" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestCredentialStorage_Validate_GoodSources(t *testing.T) { s := CredentialStorage{} - testCreds := credentials.CredentialSet{ - Credentials: []valuesource.Strategy{ - { - Source: valuesource.Source{ - Key: "env", - Value: "SOME_ENV", - }, - }, - { - Source: valuesource.Source{ - Key: "value", - Value: "somevalue", - }, + testCreds := credentials.NewCredentialSet("mycreds", + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "env", + Value: "SOME_ENV", }, }, - } + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "value", + Value: "somevalue", + }, + }) err := s.Validate(testCreds) require.NoError(t, err, "Validate did not return errors") @@ -33,23 +36,83 @@ func TestCredentialStorage_Validate_GoodSources(t *testing.T) { func TestCredentialStorage_Validate_BadSources(t *testing.T) { s := CredentialStorage{} - testCreds := credentials.CredentialSet{ - Credentials: []valuesource.Strategy{ - { - Source: valuesource.Source{ - Key: "wrongthing", - Value: "SOME_ENV", - }, + testCreds := credentials.NewCredentialSet("mycreds", + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "wrongthing", + Value: "SOME_ENV", }, - { - Source: valuesource.Source{ - Key: "anotherwrongthing", - Value: "somevalue", - }, + }, + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "anotherwrongthing", + Value: "somevalue", }, }, - } + ) err := s.Validate(testCreds) require.Error(t, err, "Validate returned errors") } + +func TestCredentialStorage_HaltOnMigrationRequired(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + // Add an unmigrated credential + credDir := filepath.Join(home, "credentials") + config.FileSystem.Mkdir(credDir, 0755) + config.TestContext.AddTestFile(filepath.Join("../storage/testdata/credentials", "mybun.json"), filepath.Join(home, "credentials", "mybun.json")) + + dataStore := filesystem.NewStore(*config.Config, hclog.NewNullLogger()) + mgr := storage.NewManager(config.Config, dataStore) + credStore := credentials.NewCredentialStore(mgr) + + var err error + t.Run("list", func(t *testing.T) { + _, err = credStore.List() + require.Error(t, err, "Operation should halt because a migration is required") + assert.Contains(t, err.Error(), "The schema of Porter's data is in an older format than supported by this version of Porter") + }) + + t.Run("read", func(t *testing.T) { + _, err = credStore.Read("mybun") + require.Error(t, err, "Operation should halt because a migration is required") + assert.Contains(t, err.Error(), "The schema of Porter's data is in an older format than supported by this version of Porter") + }) +} + +func TestCredentialStorage_OperationAllowedWhenNoMigrationDetected(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + // Add migrated credentials data + config.CopyDirectory(filepath.Join("../storage/testdata", "migrated"), home, false) + + dataStore := filesystem.NewStore(*config.Config, hclog.NewNullLogger()) + mgr := storage.NewManager(config.Config, dataStore) + credStore := credentials.NewCredentialStore(mgr) + + names, err := credStore.List() + require.NoError(t, err, "List failed") + assert.NotEmpty(t, names, "Expected credential names to be populated") +} + +func TestCredentialStorage_NoMigrationRequiredForEmptyHome(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + dataStore := filesystem.NewStore(*config.Config, hclog.NewNullLogger()) + mgr := storage.NewManager(config.Config, dataStore) + credStore := credentials.NewCredentialStore(mgr) + + names, err := credStore.List() + require.NoError(t, err, "List failed") + assert.Empty(t, names, "Expected an empty list of credentials since porter home is new") +} diff --git a/pkg/credentials/migrateCredentialsWrapper.go b/pkg/credentials/migrateCredentialsWrapper.go deleted file mode 100644 index f1f35309e..000000000 --- a/pkg/credentials/migrateCredentialsWrapper.go +++ /dev/null @@ -1,49 +0,0 @@ -package credentials - -import ( - "path/filepath" - - "get.porter.sh/porter/pkg/config" - "github.com/cnabio/cnab-go/utils/crud" - "github.com/pkg/errors" -) - -// migrateCredentialsWrapper lets us shim in the migration of the credentials from yaml to json -// transparently for a period of time. -// When we are ready to remove this we can remove the wrapper and go back to directly -// giving the CredentialStore the wrappedStore. -type migrateCredentialsWrapper struct { - *config.Config - *crud.BackingStore - - wrappedStore crud.Store -} - -func newMigrateCredentialsWrapper(c *config.Config, wrappedStore crud.Store) *migrateCredentialsWrapper { - return &migrateCredentialsWrapper{ - Config: c, - wrappedStore: wrappedStore, - } -} - -func (w *migrateCredentialsWrapper) Connect() error { - if w.BackingStore != nil { - return nil - } - - home, err := w.GetHomeDir() - if err != nil { - return errors.Wrap(err, "could not migrate credentials directory") - } - - credsDir := filepath.Join(home, "credentials") - - migration := NewCredentialsMigration(w.Context) - err = migration.Migrate(credsDir) - if err != nil { - errors.Wrap(err, "conversion of the credentials directory failed") - } - - w.BackingStore = crud.NewBackingStore(w.wrappedStore) - return nil -} diff --git a/pkg/credentials/testdata/mybuns.json b/pkg/credentials/testdata/mybuns.json deleted file mode 100644 index c44811d84..000000000 --- a/pkg/credentials/testdata/mybuns.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"azure","created":"2020-01-01T00:00:00Z","modified":"2020-01-01T00:00:00Z","credentials":[{"name":"CLIENT_ID","source":{"env":"OAUTH_CLIENT_ID"}},{"name":"CLIENT_SECRET","source":{"env":"OAUTH_CLIENT_SECRET"}}]} \ No newline at end of file diff --git a/pkg/credentials/testdata/mybuns.yaml b/pkg/credentials/testdata/mybuns.yaml deleted file mode 100644 index f9d83ead2..000000000 --- a/pkg/credentials/testdata/mybuns.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: azure -credentials: - - name: CLIENT_ID - source: - env: OAUTH_CLIENT_ID - - name: CLIENT_SECRET - source: - env: OAUTH_CLIENT_SECRET \ No newline at end of file diff --git a/pkg/generator/credentials.go b/pkg/generator/credentials.go index 9363cd4db..6eff2680d 100644 --- a/pkg/generator/credentials.go +++ b/pkg/generator/credentials.go @@ -20,9 +20,9 @@ type GenerateCredentialsOptions struct { } // GenerateCredentials will generate a credential set based on the given options -func GenerateCredentials(opts GenerateCredentialsOptions) (*credentials.CredentialSet, error) { +func GenerateCredentials(opts GenerateCredentialsOptions) (credentials.CredentialSet, error) { if opts.Name == "" { - return nil, errors.New("credentialset name is required") + return credentials.CredentialSet{}, errors.New("credentialset name is required") } generator := genSurvey if opts.Silent { @@ -30,15 +30,13 @@ func GenerateCredentials(opts GenerateCredentialsOptions) (*credentials.Credenti } credSet, err := genCredentialSet(opts.Name, opts.Credentials, generator) if err != nil { - return nil, err + return credentials.CredentialSet{}, err } - return &credSet, nil + return credSet, nil } func genCredentialSet(name string, creds map[string]bundle.Credential, fn generator) (credentials.CredentialSet, error) { - cs := credentials.CredentialSet{ - Name: name, - } + cs := credentials.NewCredentialSet(name) cs.Credentials = []valuesource.Strategy{} if strings.ContainsAny(name, "./\\") { diff --git a/pkg/generator/credentials_test.go b/pkg/generator/credentials_test.go index a09b7f273..6131b6b48 100644 --- a/pkg/generator/credentials_test.go +++ b/pkg/generator/credentials_test.go @@ -19,7 +19,7 @@ func TestBadCredentialsName(t *testing.T) { cs, err := GenerateCredentials(opts) require.Error(t, err, "bad name should have resulted in an error") - require.Nil(t, cs, "credential set should have been empty") + require.Empty(t, cs, "credential set should have been empty") require.EqualError(t, err, fmt.Sprintf("credentialset name '%s' cannot contain the following characters: './\\'", name)) } @@ -51,7 +51,6 @@ func TestGoodCredentialsName(t *testing.T) { cs, err := GenerateCredentials(opts) require.NoError(t, err, "name should NOT have resulted in an error") - require.NotNil(t, cs, "credential set should have been empty and not nil") require.Equal(t, 3, len(cs.Credentials), "should have had 3 entries") } @@ -79,7 +78,7 @@ func TestEmptyCredentials(t *testing.T) { } cs, err := GenerateCredentials(opts) require.NoError(t, err, "empty credentials should have generated an empty credential set") - require.NotNil(t, cs, "empty credentials should still return an empty credential set") + require.NotEmpty(t, cs, "empty credentials should still return an empty credential set") } func TestNoCredentialsName(t *testing.T) { @@ -90,5 +89,5 @@ func TestNoCredentialsName(t *testing.T) { } cs, err := GenerateCredentials(opts) require.Error(t, err, "expected an error because name is required") - require.Nil(t, cs, "credential set should have been empty") + require.Empty(t, cs, "credential set should have been empty") } diff --git a/pkg/generator/parameters.go b/pkg/generator/parameters.go index 95f662a0b..a3b8847f7 100644 --- a/pkg/generator/parameters.go +++ b/pkg/generator/parameters.go @@ -8,7 +8,6 @@ import ( "get.porter.sh/porter/pkg/parameters" "github.com/cnabio/cnab-go/bundle" - "github.com/cnabio/cnab-go/valuesource" ) // GenerateParametersOptions are the options to generate a Parameter Set @@ -20,9 +19,9 @@ type GenerateParametersOptions struct { } // GenerateParameters will generate a parameter set based on the given options -func (opts *GenerateParametersOptions) GenerateParameters() (*parameters.ParameterSet, error) { +func (opts *GenerateParametersOptions) GenerateParameters() (parameters.ParameterSet, error) { if opts.Name == "" { - return nil, errors.New("parameter set name is required") + return parameters.ParameterSet{}, errors.New("parameter set name is required") } generator := genSurvey if opts.Silent { @@ -30,16 +29,13 @@ func (opts *GenerateParametersOptions) GenerateParameters() (*parameters.Paramet } pset, err := opts.genParameterSet(generator) if err != nil { - return nil, err + return parameters.ParameterSet{}, err } - return &pset, nil + return pset, nil } func (opts *GenerateParametersOptions) genParameterSet(fn generator) (parameters.ParameterSet, error) { - pset := parameters.ParameterSet{ - Name: opts.Name, - } - pset.Parameters = []valuesource.Strategy{} + pset := parameters.NewParameterSet(opts.Name) if strings.ContainsAny(opts.Name, "./\\") { return pset, fmt.Errorf("parameter set name '%s' cannot contain the following characters: './\\'", opts.Name) diff --git a/pkg/generator/parameters_test.go b/pkg/generator/parameters_test.go index cd2a64369..1b8d28394 100644 --- a/pkg/generator/parameters_test.go +++ b/pkg/generator/parameters_test.go @@ -7,7 +7,7 @@ import ( "get.porter.sh/porter/pkg/parameters" "github.com/cnabio/cnab-go/bundle" "github.com/cnabio/cnab-go/bundle/definition" - "github.com/cnabio/cnab-go/valuesource" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,7 +22,7 @@ func TestBadParametersName(t *testing.T) { pset, err := opts.GenerateParameters() require.Error(t, err, "bad name should have resulted in an error") - require.Nil(t, pset, "parameter set should have been empty") + require.Empty(t, pset, "parameter set should have been empty") require.EqualError(t, err, fmt.Sprintf("parameter set name '%s' cannot contain the following characters: './\\'", name)) } @@ -50,7 +50,6 @@ func TestGoodParametersName(t *testing.T) { pset, err := opts.GenerateParameters() require.NoError(t, err, "name should NOT have resulted in an error") - require.NotNil(t, pset, "parameter set should have been empty and not nil") require.Equal(t, 3, len(pset.Parameters), "should have had 3 entries") } @@ -64,7 +63,7 @@ func TestNoParameters(t *testing.T) { } pset, err := opts.GenerateParameters() require.NoError(t, err, "no parameters should have generated an empty parameter set") - require.NotNil(t, pset, "empty parameters should still return an empty parameter set") + require.NotEmpty(t, pset, "empty parameters should still return an empty parameter set") } func TestEmptyParameters(t *testing.T) { @@ -78,7 +77,7 @@ func TestEmptyParameters(t *testing.T) { } pset, err := opts.GenerateParameters() require.NoError(t, err, "empty parameters should have generated an empty parameter set") - require.NotNil(t, pset, "empty parameters should still return an empty parameter set") + require.NotEmpty(t, pset, "empty parameters should still return an empty parameter set") } func TestNoParametersName(t *testing.T) { @@ -89,7 +88,7 @@ func TestNoParametersName(t *testing.T) { } pset, err := opts.GenerateParameters() require.Error(t, err, "expected an error because name is required") - require.Nil(t, pset, "parameter set should have been empty") + require.Empty(t, pset, "parameter set should have been empty") } func TestSkipParameters(t *testing.T) { @@ -113,13 +112,8 @@ func TestSkipParameters(t *testing.T) { }, } - expected := ¶meters.ParameterSet{ - Name: "skip-params", - Parameters: []valuesource.Strategy{}, - } - pset, err := opts.GenerateParameters() require.NoError(t, err, "parameters generation should not have resulted in an error") - require.NotNil(t, pset, "parameter set should not be nil") - require.Equal(t, expected, pset, "parameter set should have empty parameters section") + assert.Equal(t, "skip-params", pset.Name, "Name was not set") + require.Empty(t, pset.Parameters, "parameter set should have empty parameters section") } diff --git a/pkg/parameters/helpers.go b/pkg/parameters/helpers.go index a0e3ebe83..f5bf7784f 100644 --- a/pkg/parameters/helpers.go +++ b/pkg/parameters/helpers.go @@ -24,7 +24,7 @@ type TestParameterProvider struct { func NewTestParameterProvider(t *testing.T, tc *config.TestConfig) TestParameterProvider { backingSecrets := inmemorysecrets.NewStore() - backingParams := crud.NewMockStore() + backingParams := crud.NewBackingStore(crud.NewMockStore()) paramStore := NewParameterStore(backingParams) return TestParameterProvider{ T: t, @@ -43,7 +43,7 @@ func (p *TestParameterProvider) AddTestParameters(path string) { p.T.Fatal(errors.Wrapf(err, "could not read test parameters from %s", path)) } - err = p.ParameterStorage.Save(*cs) + err = p.ParameterStorage.Save(cs) if err != nil { p.T.Fatal(errors.Wrap(err, "could not load test parameters into in memory parameter storage")) } diff --git a/pkg/parameters/parameterStorage_test.go b/pkg/parameters/parameterStorage_test.go index bdb7d8247..7ea8e4ffe 100644 --- a/pkg/parameters/parameterStorage_test.go +++ b/pkg/parameters/parameterStorage_test.go @@ -1,9 +1,14 @@ package parameters import ( + "path/filepath" "testing" + "get.porter.sh/porter/pkg/storage" + "get.porter.sh/porter/pkg/storage/filesystem" "github.com/cnabio/cnab-go/utils/crud" + "github.com/hashicorp/go-hclog" + "github.com/stretchr/testify/assert" "get.porter.sh/porter/pkg/config" "get.porter.sh/porter/pkg/secrets" @@ -15,30 +20,26 @@ import ( func TestParameterStorage_ResolveAll(t *testing.T) { // The inmemory secret store currently only supports secret sources // So all of these have this same source - testParameterSet := ParameterSet{ - Name: "myparamset", - Parameters: []valuesource.Strategy{ - { - Name: "param1", - Source: valuesource.Source{ - Key: "secret", - Value: "param1", - }, - }, - { - Name: "param2", - Source: valuesource.Source{ - Key: "secret", - Value: "param2", - }, + testParameterSet := NewParameterSet("myparamset", + valuesource.Strategy{ + Name: "param1", + Source: valuesource.Source{ + Key: "secret", + Value: "param1", }, }, - } + valuesource.Strategy{ + Name: "param2", + Source: valuesource.Source{ + Key: "secret", + Value: "param2", + }, + }) t.Run("resolve params success", func(t *testing.T) { tc := config.NewTestConfig(t) backingSecrets := inmemorysecrets.NewStore() - backingParams := crud.NewMockStore() + backingParams := crud.NewBackingStore(crud.NewMockStore()) paramStore := NewParameterStore(backingParams) secretStore := secrets.NewSecretStore(backingSecrets) @@ -64,7 +65,7 @@ func TestParameterStorage_ResolveAll(t *testing.T) { t.Run("resolve params failure", func(t *testing.T) { tc := config.NewTestConfig(t) backingSecrets := inmemorysecrets.NewStore() - backingParams := crud.NewMockStore() + backingParams := crud.NewBackingStore(crud.NewMockStore()) paramStore := NewParameterStore(backingParams) secretStore := secrets.NewSecretStore(backingSecrets) @@ -92,40 +93,37 @@ func TestParameterStorage_Validate(t *testing.T) { t.Run("valid sources", func(t *testing.T) { s := ParameterStorage{} - testParameterSet := ParameterSet{ - Parameters: []valuesource.Strategy{ - { - Source: valuesource.Source{ - Key: "env", - Value: "SOME_ENV", - }, - }, - { - Source: valuesource.Source{ - Key: "value", - Value: "somevalue", - }, + testParameterSet := NewParameterSet("myparams", + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "env", + Value: "SOME_ENV", }, - { - Source: valuesource.Source{ - Key: "path", - Value: "/some/path", - }, + }, + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "value", + Value: "somevalue", }, - { - Source: valuesource.Source{ - Key: "command", - Value: "some command", - }, + }, + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "path", + Value: "/some/path", }, - { - Source: valuesource.Source{ - Key: "secret", - Value: "secret", - }, + }, + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "command", + Value: "some command", }, }, - } + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "secret", + Value: "secret", + }, + }) err := s.Validate(testParameterSet) require.NoError(t, err, "Validate did not return errors") @@ -133,24 +131,83 @@ func TestParameterStorage_Validate(t *testing.T) { t.Run("invalid sources", func(t *testing.T) { s := ParameterStorage{} - testParameterSet := ParameterSet{ - Parameters: []valuesource.Strategy{ - { - Source: valuesource.Source{ - Key: "wrongthing", - Value: "SOME_ENV", - }, - }, - { - Source: valuesource.Source{ - Key: "anotherwrongthing", - Value: "somevalue", - }, + testParameterSet := NewParameterSet("myparams", + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "wrongthing", + Value: "SOME_ENV", }, }, - } + valuesource.Strategy{ + Source: valuesource.Source{ + Key: "anotherwrongthing", + Value: "somevalue", + }, + }) err := s.Validate(testParameterSet) require.Error(t, err, "Validate returned errors") }) } + +func TestParameterStorage_HaltOnMigrationRequired(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + // Add an unmigrated parameter + credDir := filepath.Join(home, "parameters") + config.FileSystem.Mkdir(credDir, 0755) + config.TestContext.AddTestFile(filepath.Join("../storage/testdata/parameters", "mybun.json"), filepath.Join(home, "parameters", "mybun.json")) + + dataStore := filesystem.NewStore(*config.Config, hclog.NewNullLogger()) + mgr := storage.NewManager(config.Config, dataStore) + paramStore := NewParameterStore(mgr) + + var err error + t.Run("list", func(t *testing.T) { + _, err = paramStore.List() + require.Error(t, err, "Operation should halt because a migration is required") + assert.Contains(t, err.Error(), "The schema of Porter's data is in an older format than supported by this version of Porter") + }) + + t.Run("read", func(t *testing.T) { + _, err = paramStore.Read("mybun") + require.Error(t, err, "Operation should halt because a migration is required") + assert.Contains(t, err.Error(), "The schema of Porter's data is in an older format than supported by this version of Porter") + }) +} + +func TestParameterStorage_OperationAllowedWhenNoMigrationDetected(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + // Add migrated credentials data + config.CopyDirectory(filepath.Join("../storage/testdata", "migrated"), home, false) + + dataStore := filesystem.NewStore(*config.Config, hclog.NewNullLogger()) + mgr := storage.NewManager(config.Config, dataStore) + paramStore := NewParameterStore(mgr) + + names, err := paramStore.List() + require.NoError(t, err, "List failed") + assert.NotEmpty(t, names, "Expected parameter names to be populated") +} + +func TestParameterStorage_NoMigrationRequiredForEmptyHome(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + dataStore := filesystem.NewStore(*config.Config, hclog.NewNullLogger()) + mgr := storage.NewManager(config.Config, dataStore) + paramStore := NewParameterStore(mgr) + + names, err := paramStore.List() + require.NoError(t, err, "List failed") + assert.Empty(t, names, "Expected an empty list of parameters since porter home is new") +} diff --git a/pkg/parameters/parameters.go b/pkg/parameters/parameters.go index af0fe6666..988c60713 100644 --- a/pkg/parameters/parameters.go +++ b/pkg/parameters/parameters.go @@ -41,13 +41,13 @@ func ParseVariableAssignments(params []string) (map[string]string, error) { // Load a ParameterSet from a file at a given path. // // It does not load the individual parameters. -func Load(path string) (*ParameterSet, error) { - pset := &ParameterSet{} +func Load(path string) (ParameterSet, error) { + var pset ParameterSet data, err := ioutil.ReadFile(path) if err != nil { return pset, err } - return pset, yaml.Unmarshal(data, pset) + return pset, yaml.Unmarshal(data, &pset) } // IsInternal determines if the provided param is an internal parameter diff --git a/pkg/parameters/parameters_test.go b/pkg/parameters/parameters_test.go index 8e432cf8f..2557f135c 100644 --- a/pkg/parameters/parameters_test.go +++ b/pkg/parameters/parameters_test.go @@ -56,57 +56,51 @@ func TestLoad(t *testing.T) { }) t.Run("successful load, unsuccessful unmarshal", func(t *testing.T) { - expected := &ParameterSet{} - pset, err := Load("testdata/paramset_bad.json") require.EqualError(t, err, "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `myparam...` into parameters.ParameterSet") - require.Equal(t, expected, pset) + require.Empty(t, pset) }) t.Run("successful load, successful unmarshal", func(t *testing.T) { - expected := &ParameterSet{ - Name: "mybun", - Created: time.Date(1983, time.April, 18, 1, 2, 3, 4, time.UTC), - Modified: time.Date(1983, time.April, 18, 1, 2, 3, 4, time.UTC), - Parameters: []valuesource.Strategy{ - { - Name: "param_env", - Source: valuesource.Source{ - Key: "env", - Value: "PARAM_ENV", - }, + expected := NewParameterSet("mybun", + valuesource.Strategy{ + Name: "param_env", + Source: valuesource.Source{ + Key: "env", + Value: "PARAM_ENV", }, - { - Name: "param_value", - Source: valuesource.Source{ - Key: "value", - Value: "param_value", - }, - }, - { - Name: "param_command", - Source: valuesource.Source{ - Key: "command", - Value: "echo hello world", - }, + }, + valuesource.Strategy{ + Name: "param_value", + Source: valuesource.Source{ + Key: "value", + Value: "param_value", }, - { - Name: "param_path", - Source: valuesource.Source{ - Key: "path", - Value: "/path/to/param", - }, + }, + valuesource.Strategy{ + Name: "param_command", + Source: valuesource.Source{ + Key: "command", + Value: "echo hello world", }, - { - Name: "param_secret", - Source: valuesource.Source{ - Key: "secret", - Value: "param_secret", - }, + }, + valuesource.Strategy{ + Name: "param_path", + Source: valuesource.Source{ + Key: "path", + Value: "/path/to/param", }, }, - } + valuesource.Strategy{ + Name: "param_secret", + Source: valuesource.Source{ + Key: "secret", + Value: "param_secret", + }, + }) + expected.Created = time.Date(1983, time.April, 18, 1, 2, 3, 4, time.UTC) + expected.Modified = expected.Created pset, err := Load("testdata/paramset.json") require.NoError(t, err) diff --git a/pkg/parameters/parameterset.go b/pkg/parameters/parameterset.go index 9caf1a659..a2a8696f8 100644 --- a/pkg/parameters/parameterset.go +++ b/pkg/parameters/parameterset.go @@ -3,12 +3,27 @@ package parameters import ( "time" + "github.com/cnabio/cnab-go/schema" "github.com/cnabio/cnab-go/valuesource" ) +const ( + // DefaultSchemaVersion is the default SchemaVersion value + // set on new ParameterSet instances, and is the semver portion + // of CNABSpecVersion. + DefaultSchemaVersion = schema.Version("1.0.0-DRAFT+TODO") + + // CNABSpecVersion represents the CNAB Spec version of the Parameters + // that this library implements. + // This value is prefixed with e.g. `cnab-parametersets-` so isn't itself valid semver. + CNABSpecVersion string = "cnab-parametersets-" + string(DefaultSchemaVersion) +) + // ParameterSet represents a collection of parameters and their // sources/strategies for value resolution type ParameterSet struct { + // SchemaVersion is the version of the parameter-set schema. + SchemaVersion schema.Version `json:"schemaVersion" yaml:"schemaVersion"` // Name is the name of the parameter set. Name string `json:"name" yaml:"name"` // Created timestamp of the parameter set. @@ -18,3 +33,17 @@ type ParameterSet struct { // Parameters is a list of parameter specs. Parameters []valuesource.Strategy `json:"parameters" yaml:"parameters"` } + +// NewParameterSet creates a new ParameterSet with the required fields initialized. +func NewParameterSet(name string, params ...valuesource.Strategy) ParameterSet { + now := time.Now() + ps := ParameterSet{ + SchemaVersion: DefaultSchemaVersion, + Name: name, + Created: now, + Modified: now, + Parameters: params, + } + + return ps +} diff --git a/pkg/parameters/parameterset_test.go b/pkg/parameters/parameterset_test.go new file mode 100644 index 000000000..b623a2ea4 --- /dev/null +++ b/pkg/parameters/parameterset_test.go @@ -0,0 +1,68 @@ +package parameters + +import ( + "path/filepath" + "testing" + + "get.porter.sh/porter/pkg/config" + "get.porter.sh/porter/pkg/storage" + "get.porter.sh/porter/pkg/storage/filesystem" + "github.com/cnabio/cnab-go/schema" + "github.com/cnabio/cnab-go/utils/crud" + "github.com/cnabio/cnab-go/valuesource" + "github.com/hashicorp/go-hclog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCNABSpecVersion(t *testing.T) { + version, err := schema.GetSemver(CNABSpecVersion) + require.NoError(t, err) + assert.Equal(t, DefaultSchemaVersion, version) +} + +func TestNewParameterSet(t *testing.T) { + cs := NewParameterSet("myparams", + valuesource.Strategy{ + Name: "password", + Source: valuesource.Source{ + Key: "env", + Value: "DB_PASSWORD", + }, + }) + + assert.Equal(t, "myparams", cs.Name, "Name was not set") + assert.NotEmpty(t, cs.Created, "Created was not set") + assert.NotEmpty(t, cs.Modified, "Modified was not set") + assert.Equal(t, cs.Created, cs.Modified, "Created and Modified should have the same timestamp") + assert.Equal(t, DefaultSchemaVersion, cs.SchemaVersion, "SchemaVersion was not set") + assert.Len(t, cs.Parameters, 1, "Parameters should be initialized with 1 value") +} + +// TODO: (carolynvs) move this into manager_test.go in pkg/storage once parameter set is moved to cnab-go +func TestManager_MigrateParameters(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + credsDir := filepath.Join(home, "parameters") + config.FileSystem.Mkdir(credsDir, 0755) + config.TestContext.AddTestFile(filepath.Join("../storage/testdata/parameters", "mybun.json"), filepath.Join(credsDir, "mybun.json")) + + dataStore := crud.NewBackingStore(filesystem.NewStore(*config.Config, hclog.NewNullLogger())) + mgr := storage.NewManager(config.Config, dataStore) + paramStore := NewParameterStorage(mgr) + + logfilePath, err := mgr.Migrate() + require.NoError(t, err, "Migrate failed") + + c, err := paramStore.Read("mybun") + require.NoError(t, err, "Read parameter failed") + + assert.Equal(t, DefaultSchemaVersion, c.SchemaVersion, "parameter set was not migrated") + + logfile, err := config.FileSystem.ReadFile(logfilePath) + require.NoError(t, err, "error reading logfile") + assert.Equal(t, config.TestContext.GetError(), string(logfile), "the migration should have been copied to both stderr and the logfile") +} diff --git a/pkg/parameters/parameterstore.go b/pkg/parameters/parameterstore.go index 784bbf581..c4775989c 100644 --- a/pkg/parameters/parameterstore.go +++ b/pkg/parameters/parameterstore.go @@ -17,17 +17,22 @@ var ErrNotFound = errors.New("Parameter set does not exist") // Store is a persistent store for parameter sets. type Store struct { - backingStore *crud.BackingStore + backingStore crud.ManagedStore } // NewParameterStore creates a persistent store for parameter sets using the specified // backing key-blob store. -func NewParameterStore(store crud.Store) Store { +func NewParameterStore(store crud.ManagedStore) Store { return Store{ - backingStore: crud.NewBackingStore(store), + backingStore: store, } } +// GetBackingStore returns the data store behind this credentials store. +func (s Store) GetBackingStore() crud.ManagedStore { + return s.backingStore +} + // List the names of the stored parameter sets. func (s Store) List() ([]string, error) { return s.backingStore.List(ItemType, "") diff --git a/pkg/parameters/parameterstore_test.go b/pkg/parameters/parameterstore_test.go index 727f940cd..5828e900d 100644 --- a/pkg/parameters/parameterstore_test.go +++ b/pkg/parameters/parameterstore_test.go @@ -2,6 +2,7 @@ package parameters import ( "testing" + "time" "github.com/cnabio/cnab-go/utils/crud" "github.com/cnabio/cnab-go/valuesource" @@ -9,25 +10,24 @@ import ( ) func TestNewParameterStore(t *testing.T) { - backingParams := crud.NewMockStore() + backingParams := crud.NewBackingStore(crud.NewMockStore()) paramStore := NewParameterStore(backingParams) params, err := paramStore.List() require.NoError(t, err) require.Empty(t, params, "List should return no entries") - myParamSet := ParameterSet{ - Name: "myparams", - Parameters: []valuesource.Strategy{ - { - Name: "myparam", - Source: valuesource.Source{ - Key: "value", - Value: "myparamvalue", - }, + myParamSet := NewParameterSet("myparams", + valuesource.Strategy{ + Name: "myparam", + Source: valuesource.Source{ + Key: "value", + Value: "myparamvalue", }, - }, - } + }) + myParamSet.Created = time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC) + myParamSet.Modified = myParamSet.Created + err = paramStore.Save(myParamSet) require.NoError(t, err, "Save should successfully save") diff --git a/pkg/parameters/testdata/paramset.json b/pkg/parameters/testdata/paramset.json index 5cb28960b..c58d0e093 100644 --- a/pkg/parameters/testdata/paramset.json +++ b/pkg/parameters/testdata/paramset.json @@ -1,4 +1,5 @@ { + "schemaVersion": "1.0.0-DRAFT+TODO", "name": "mybun", "created": "1983-04-18T01:02:03.000000004Z", "modified": "1983-04-18T01:02:03.000000004Z", diff --git a/pkg/porter/credentials.go b/pkg/porter/credentials.go index 730657845..7f7eb9390 100644 --- a/pkg/porter/credentials.go +++ b/pkg/porter/credentials.go @@ -131,7 +131,7 @@ func (p *Porter) GenerateCredentials(opts CredentialOptions) error { cs.Created = time.Now() cs.Modified = cs.Created - err = p.Credentials.Save(*cs) + err = p.Credentials.Save(cs) return errors.Wrapf(err, "unable to save credentials") } diff --git a/pkg/porter/credentials_test.go b/pkg/porter/credentials_test.go index 4aa47ca39..d27013fe5 100644 --- a/pkg/porter/credentials_test.go +++ b/pkg/porter/credentials_test.go @@ -142,7 +142,7 @@ func TestCredentialsList(t *testing.T) { { name: "yaml", format: printer.FormatYaml, - wantContains: []string{`- name: kool-kreds`}, + wantContains: []string{`name: kool-kreds`}, errorMsg: "", }, { @@ -284,6 +284,7 @@ func TestShowCredential_Found(t *testing.T) { name: "json", format: printer.FormatJson, wantOutput: `{ + "schemaVersion": "1.0.0-DRAFT+b6c701f", "name": "kool-kreds", "created": "2019-06-24T16:07:57.415378-05:00", "modified": "2019-06-24T16:07:57.415378-05:00", @@ -319,7 +320,8 @@ func TestShowCredential_Found(t *testing.T) { { name: "yaml", format: printer.FormatYaml, - wantOutput: `name: kool-kreds + wantOutput: `schemaVersion: 1.0.0-DRAFT+b6c701f +name: kool-kreds created: 2019-06-24T16:07:57.415378-05:00 modified: 2019-06-24T16:07:57.415378-05:00 credentials: diff --git a/pkg/porter/helpers.go b/pkg/porter/helpers.go index 90b42158f..324c5ee78 100644 --- a/pkg/porter/helpers.go +++ b/pkg/porter/helpers.go @@ -17,10 +17,8 @@ import ( "get.porter.sh/porter/pkg/mixin" "get.porter.sh/porter/pkg/parameters" "get.porter.sh/porter/pkg/plugins" - "get.porter.sh/porter/pkg/secrets" "github.com/cnabio/cnab-go/bundle" cnabcreds "github.com/cnabio/cnab-go/credentials" - "github.com/cnabio/cnab-go/secrets/host" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" ) @@ -75,8 +73,17 @@ func (p *TestPorter) SetupIntegrationTest() { // Undo changes above to make a unit test friendly Porter, so we hit the host p.Porter = NewWithConfig(p.Config) p.NewCommand = exec.Command - p.TestCredentials.SecretsStore = secrets.NewSecretStore(&host.SecretStore{}) + /* + // Update test providers to use the instances we just reset above + // We mostly don't use test providers for integration tests, but a few provide + // useful helper methods that are still nice to have. + hostSecrets := &host.SecretStore{} + p.TestCredentials.SecretsStore = secrets.NewSecretStore(hostSecrets) + p.TestParameters.SecretsStore = secrets.NewSecretStore(hostSecrets) + */ + + // Run the test in a temp directory homeDir := p.TestConfig.TestContext.UseFilesystem() p.TestConfig.SetupIntegrationTest(homeDir) bundleDir := p.CreateBundleDir() diff --git a/pkg/porter/install_test.go b/pkg/porter/install_test.go index c6bdd8e04..00371882c 100644 --- a/pkg/porter/install_test.go +++ b/pkg/porter/install_test.go @@ -153,18 +153,14 @@ func TestPorter_InstallBundle_WithDepsFromTag(t *testing.T) { p.TestConfig.TestContext.AddTestDirectory("testdata/cache", cacheDir) // Make some fake credentials to give to the install operation, they won't be used because it's a dummy driver - cs := credentials.CredentialSet{ - Name: "wordpress", - Credentials: []valuesource.Strategy{ - { - Name: "kubeconfig", - Source: valuesource.Source{ - Key: secrets.SourceSecret, - Value: "kubeconfig", - }, + cs := credentials.NewCredentialSet("wordpress", + valuesource.Strategy{ + Name: "kubeconfig", + Source: valuesource.Source{ + Key: secrets.SourceSecret, + Value: "kubeconfig", }, - }, - } + }) p.TestCredentials.TestSecrets.AddSecret("kubeconfig", "abc123") err := p.Credentials.Save(cs) require.NoError(t, err, "Credentials.Save failed") diff --git a/pkg/porter/parameters.go b/pkg/porter/parameters.go index 49ed6eeae..76f88f65a 100644 --- a/pkg/porter/parameters.go +++ b/pkg/porter/parameters.go @@ -137,7 +137,7 @@ func (p *Porter) GenerateParameters(opts ParameterOptions) error { pset.Created = time.Now() pset.Modified = pset.Created - err = p.Parameters.Save(*pset) + err = p.Parameters.Save(pset) return errors.Wrapf(err, "unable to save parameter set") } diff --git a/pkg/porter/porter.go b/pkg/porter/porter.go index 2362e1278..73173ba44 100644 --- a/pkg/porter/porter.go +++ b/pkg/porter/porter.go @@ -35,7 +35,7 @@ type Porter struct { Mixins mixin.MixinProvider Plugins plugins.PluginProvider CNAB cnabprovider.CNABProvider - Storage *storage.Manager + Storage storage.StorageProvider } // New porter client, initialized with useful defaults. diff --git a/pkg/porter/storage.go b/pkg/porter/storage.go index eba1f2dfd..d710782f9 100644 --- a/pkg/porter/storage.go +++ b/pkg/porter/storage.go @@ -1,6 +1,20 @@ package porter +import ( + "errors" + "fmt" +) + func (p *Porter) MigrateStorage() error { - _, err := p.Storage.Migrate() - return err + logfilePath, err := p.Storage.Migrate() + + fmt.Fprintf(p.Out, "\nSaved migration logs to %s\n", logfilePath) + + if err != nil { + // The error has already been printed, don't return it otherwise it will be double printed + return errors.New("Migration failed!") + } + + fmt.Fprintln(p.Out, "Migration complete!") + return nil } diff --git a/pkg/porter/testdata/test-creds/kool-kreds.json b/pkg/porter/testdata/test-creds/kool-kreds.json index 92aa10b6d..a7b492959 100644 --- a/pkg/porter/testdata/test-creds/kool-kreds.json +++ b/pkg/porter/testdata/test-creds/kool-kreds.json @@ -1,4 +1,5 @@ { + "schemaVersion": "1.0.0-DRAFT+b6c701f", "name": "kool-kreds", "created": "2019-06-24T16:07:57.415378-05:00", "modified": "2019-06-24T16:07:57.415378-05:00", diff --git a/pkg/storage/filesystem/store.go b/pkg/storage/filesystem/store.go index a20ad5659..af71db99d 100644 --- a/pkg/storage/filesystem/store.go +++ b/pkg/storage/filesystem/store.go @@ -3,6 +3,7 @@ package filesystem import ( "get.porter.sh/porter/pkg/config" "github.com/cnabio/cnab-go/claim" + "github.com/cnabio/cnab-go/credentials" "github.com/cnabio/cnab-go/utils/crud" "github.com/hashicorp/go-hclog" "github.com/pkg/errors" @@ -44,7 +45,14 @@ func (s *Store) Connect() error { func NewFileExtensions() map[string]string { ext := claim.NewClaimStoreFileExtensions() + jsonExt := ".json" + ext[credentials.ItemType] = jsonExt + + // TODO (carolynvs): change to parameters.ItemType once parameters move to cnab-go + ext["parameters"] = jsonExt + // Handle top level files, like schema.json - ext[""] = ".json" + ext[""] = jsonExt + return ext } diff --git a/pkg/storage/manager.go b/pkg/storage/manager.go index 981f61659..d42f9ef29 100644 --- a/pkg/storage/manager.go +++ b/pkg/storage/manager.go @@ -4,14 +4,17 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "path/filepath" "strings" "time" "get.porter.sh/porter/pkg/config" "github.com/cnabio/cnab-go/claim" + "github.com/cnabio/cnab-go/credentials" "github.com/cnabio/cnab-go/schema" "github.com/cnabio/cnab-go/utils/crud" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" ) @@ -84,16 +87,82 @@ func (m *Manager) Close() error { } func (m *Manager) HandleConnect() (func() error, error) { + // Use our own HandleConnect, override other crud.Store methods + // so that we can call it instead of using the underlying datastore's connect. return m.connMgr.HandleConnect() } +func (m *Manager) GetDataStore() crud.Store { + return m.BackingStore +} + +func (m *Manager) Count(itemType string, group string) (int, error) { + handleClose, err := m.HandleConnect() + defer handleClose() + if err != nil { + return 0, err + } + + return m.BackingStore.Count(itemType, group) +} + +func (m *Manager) List(itemType string, group string) ([]string, error) { + handleClose, err := m.HandleConnect() + defer handleClose() + if err != nil { + return nil, err + } + + return m.BackingStore.List(itemType, group) +} + +func (m *Manager) Save(itemType string, group string, name string, data []byte) error { + handleClose, err := m.HandleConnect() + defer handleClose() + if err != nil { + return err + } + + return m.BackingStore.Save(itemType, group, name, data) +} + +func (m *Manager) Read(itemType string, name string) ([]byte, error) { + handleClose, err := m.HandleConnect() + defer handleClose() + if err != nil { + return nil, err + } + + return m.BackingStore.Read(itemType, name) +} + +func (m *Manager) ReadAll(itemType string, group string) ([][]byte, error) { + handleClose, err := m.HandleConnect() + defer handleClose() + if err != nil { + return nil, err + } + + return m.BackingStore.ReadAll(itemType, group) +} + +func (m *Manager) Delete(itemType string, name string) error { + handleClose, err := m.HandleConnect() + defer handleClose() + if err != nil { + return err + } + + return m.BackingStore.Delete(itemType, name) +} + // loadSchema parses the schema file at the root of PORTER_HOME. This file (when present) contains // a list of the current version of each of Porter's storage systems. func (m *Manager) loadSchema() error { b, err := m.BackingStore.Read("", "schema") if err != nil { if strings.Contains(err.Error(), crud.ErrRecordDoesNotExist.Error()) { - emptyHome, err := m.initEmptyPorterHome() + emptyHome, err := m.initEmptyPorterHome(ioutil.Discard) if emptyHome { // Return any errors from creating a schema document in an empty porter home directory return err @@ -112,12 +181,13 @@ func (m *Manager) loadSchema() error { // MigrationRequired determines if a migration of Porter's storage system is necessary. func (m *Manager) MigrationRequired() bool { - // TODO (carolynvs): Include credentials/parameters - return m.ShouldMigrateClaims() + return m.ShouldMigrateClaims() || m.ShouldMigrateCredentials() || m.ShouldMigrateParameters() } // Migrate executes a migration on any/all of Porter's storage sub-systems. func (m *Manager) Migrate() (string, error) { + m.resetSchema() + // Let us call connect and not have it kick us out because the schema is out-of-date m.allowOutOfDateSchema = true defer func() { @@ -144,27 +214,49 @@ func (m *Manager) Migrate() (string, error) { defer logfile.Close() w := io.MultiWriter(m.Err, logfile) + var migrationErr *multierror.Error if m.ShouldMigrateClaims() { fmt.Fprintf(w, "Claims schema is out-of-date (want: %s got: %s)\n", claim.CNABSpecVersion, m.schema.Claims) err = m.migrateClaims(w) - if err != nil { - // Print the error and continue - fmt.Fprintln(w, err) - } + migrationErr = multierror.Append(migrationErr, err) } else { fmt.Fprintln(w, "Claims schema is up-to-date") } - fmt.Fprintf(w, "\nSaved migration logs to %s\n", logfilePath) + if m.ShouldMigrateCredentials() { + fmt.Fprintf(w, "Credentials schema is out-of-date (want: %s got: %s)\n", credentials.CNABSpecVersion, m.schema.Credentials) + err = m.migrateCredentials(w) + migrationErr = multierror.Append(migrationErr, err) + } else { + fmt.Fprintln(w, "Credentials schema is up-to-date") + } - // TODO (carolynvs): migrate credentials - return logfilePath, nil + if m.ShouldMigrateParameters() { + fmt.Fprintf(w, "Parameters schema is out-of-date (want: %s got: %s)\n", ParameterSetCNABSpecVersion, m.schema.Parameters) + err = m.migrateParameters(w) + migrationErr = multierror.Append(migrationErr, err) + } else { + fmt.Fprintln(w, "Parameters schema is up-to-date") + } + + if migrationErr.ErrorOrNil() == nil { + err = m.writeSchema(w) + migrationErr = multierror.Append(migrationErr, err) + } + + return logfilePath, migrationErr.ErrorOrNil() +} + +// resetSchema allows us to relook at our schema.json even after its been read. +func (m *Manager) resetSchema() { + m.schema = Schema{} + m.schemaLoaded = false } // When there is no schema, and no existing storage data, create an initial // schema file and allow the operation to continue. Don't require a // migration. -func (m *Manager) initEmptyPorterHome() (bool, error) { +func (m *Manager) initEmptyPorterHome(w io.Writer) (bool, error) { if m.schema != (Schema{}) { return false, nil } @@ -193,7 +285,7 @@ func (m *Manager) initEmptyPorterHome() (bool, error) { return false, err } - return true, m.writeSchema() + return true, m.writeSchema(w) } // ShouldMigrateClaims determines if the claims storage system requires a migration. @@ -201,41 +293,32 @@ func (m *Manager) ShouldMigrateClaims() bool { return string(m.schema.Claims) != claim.CNABSpecVersion } -// ShouldMigrateCredentials determines if the credentials storage system requires a migration. -func (m *Manager) ShouldMigrateCredentials() bool { - // TODO (carolynvs): This isn't in cnab-go and param sets aren't in the spec... - return false -} - func (m *Manager) migrateClaims(w io.Writer) error { - fmt.Fprint(w, "!!! Migrating claims data to match the CNAB Claim spec https://cdn.cnab.io/schema/cnab-claim-1.0.0-DRAFT+b5ed2f3/claim.schema.json. This is a one-way migration !!!\n") + fmt.Fprintln(w, "Migrating claims data to match the CNAB Claim spec https://cdn.cnab.io/schema/cnab-claim-1.0.0-DRAFT+b5ed2f3/claim.schema.json. This is a one-way migration.") installationNames, err := m.BackingStore.List(claim.ItemTypeClaims, "") if err != nil { - return errors.Wrapf(err, "!!! Migration failed, unable to list installation names") + return errors.Wrapf(err, "Migration failed, unable to list installation names") } + var migrationErr *multierror.Error for _, installationName := range installationNames { err = m.migrateInstallation(w, installationName) if err != nil { fmt.Fprintf(w, errors.Wrapf(err, "Error migrating installation %s. Skipping.\n", installationName).Error()) + migrationErr = multierror.Append(migrationErr, err) } } - err = m.writeSchema() - if err != nil { - return errors.Wrap(err, "!!! Migration failed") - } - - fmt.Fprintf(w, "!!! Migration complete !!!\n") - return nil + return migrationErr.ErrorOrNil() } // writeSchema updates the schema with the most recent version then writes it to disk. -func (m *Manager) writeSchema() error { +func (m *Manager) writeSchema(w io.Writer) error { m.schema = Schema{ Claims: schema.Version(claim.CNABSpecVersion), - Credentials: schema.Version("cnab-credentials-1.0.0-DRAFT-b6c701f"), + Credentials: schema.Version(credentials.CNABSpecVersion), + Parameters: schema.Version(ParameterSetCNABSpecVersion), } schemaB, err := json.Marshal(m.schema) if err != nil { @@ -243,7 +326,12 @@ func (m *Manager) writeSchema() error { } err = m.BackingStore.Save("", "", "schema", schemaB) - return errors.Wrap(err, "Unable to save storage schema file") + if err != nil { + errors.Wrap(err, "Unable to save storage schema file") + } + + fmt.Fprintln(w, "Wrote updated schema.json to storage") + return nil } // migrateInstallation moves the data from the older claim schema into the new format. @@ -266,7 +354,7 @@ func (m *Manager) writeSchema() error { // - RESULTID/ // - RESULTID-OUTPUTNAME func (m *Manager) migrateInstallation(w io.Writer, name string) error { - fmt.Fprintf(w, " - Migrating claim %s to the new claim layout...\n", name) + fmt.Fprintf(w, " - Migrating claim %s to the new schema...\n", name) oldClaimData, err := m.BackingStore.Read(claim.ItemTypeClaims, name) if err != nil { @@ -404,6 +492,103 @@ func (m *Manager) migrateUnversionedClaim(w io.Writer, name string, data []byte) return data, nil } +// ShouldMigrateCredentials determines if the credentials storage system requires a migration. +func (m *Manager) ShouldMigrateCredentials() bool { + return string(m.schema.Credentials) != credentials.CNABSpecVersion +} + +func (m *Manager) migrateCredentials(w io.Writer) error { + // Ensure all credentials have a schemaVersion set + fmt.Fprintln(w, "Migrating credential set data to match the CNAB Credential Set spec https://github.com/cnabio/cnab-spec/blob/cnab-credentialsets-1.0.0-DRAFT+b6c701f/802-credential-sets.md. This is a one-way migration.") + + credStore := credentials.NewCredentialStore(m) + creds, err := credStore.ReadAll() + if err != nil { + return errors.Wrapf(err, "Migration failed, unable to read all credentials") + } + + var migrationErr *multierror.Error + for _, cred := range creds { + // Set a schema version on credentials that don't have it yet + if cred.SchemaVersion != "" { + continue + } + + fmt.Fprintf(w, " - Migrating credential set %s to the new ...\n", cred.Name) + cred.SchemaVersion = credentials.DefaultSchemaVersion + + err = credStore.Save(cred) + if err != nil { + migrationErr = multierror.Append(migrationErr, errors.Wrapf(err, "Cannot save migrated credential set %s. Skipping.", cred.Name)) + } + } + + return migrationErr.ErrorOrNil() +} + +// TODO (carolynvs): Replace with cnab-go's const when this moves to cnab-go +const ( + ParameterSetDefaultSchemaVersion schema.Version = "1.0.0-DRAFT+TODO" + ParameterSetCNABSpecVersion string = "cnab-parametersets-" + string(ParameterSetDefaultSchemaVersion) +) + +// ShouldMigrateParameters determines if the parameters storage system requires a migration. +func (m *Manager) ShouldMigrateParameters() bool { + // Can't reference parameters.CNABSpecVersion because it causes a circular dependency + // It will be resolved when parametersets move to cnab-go + return string(m.schema.Parameters) != ParameterSetCNABSpecVersion +} + +func (m *Manager) migrateParameters(w io.Writer) error { + // Ensure all parameters have a schemaVersion set + // TODO (carolynvs): Update this with a link to the spec once we have it + fmt.Fprintln(w, "Migrating parameter set data to match the CNAB Parameter Set spec TODO. This is a one-way migration.") + + // TODO (carolynvs): Use typed parameter store when it has moved to cnab-go. It causes a circular dependency at the moment. + names, err := m.BackingStore.List("parameters", "") + if err != nil { + return errors.Wrapf(err, "Migration failed, unable to read all credentials") + } + + var migrationErr *multierror.Error + for _, name := range names { + paramB, err := m.BackingStore.Read("parameters", name) + if err != nil { + migrationErr = multierror.Append(migrationErr, errors.Wrapf(err, "Cannot read parameter set %s to migrate it. Skipping.", name)) + continue + } + + param := map[string]interface{}{} + err = json.Unmarshal(paramB, ¶m) + if err != nil { + migrationErr = multierror.Append(migrationErr, errors.Wrapf(err, "Cannot parse parameter set %s to migrate it. Skipping.", name)) + continue + } + + // Set a schema version on credentials that don't have it yet + if _, ok := param["schemaVersion"]; ok { + continue + } + + fmt.Fprintf(w, " - Migrating parameter set %s to the new schema...\n", name) + param["schemaVersion"] = ParameterSetDefaultSchemaVersion + + paramB, err = json.Marshal(param) + if err != nil { + migrationErr = multierror.Append(migrationErr, errors.Wrapf(err, "Cannot marshal parameter set %s to migrate it. Skipping.", name)) + continue + } + + err = m.BackingStore.Save("parameters", "", name, paramB) + if err != nil { + migrationErr = multierror.Append(migrationErr, errors.Wrapf(err, "Cannot save migrated parameter set %s. Skipping.", name)) + continue + } + } + + return migrationErr.ErrorOrNil() +} + // getSchemaVersion attempts to read the schemaVersion stamped on a document. func getSchemaVersion(data []byte) string { var peek struct { diff --git a/pkg/storage/manager_test.go b/pkg/storage/manager_test.go index f9e4f5dbc..1bba5b725 100644 --- a/pkg/storage/manager_test.go +++ b/pkg/storage/manager_test.go @@ -8,6 +8,7 @@ import ( "get.porter.sh/porter/pkg/config" "get.porter.sh/porter/pkg/storage/filesystem" "github.com/cnabio/cnab-go/claim" + "github.com/cnabio/cnab-go/credentials" "github.com/cnabio/cnab-go/schema" "github.com/cnabio/cnab-go/utils/crud" "github.com/hashicorp/go-hclog" @@ -121,11 +122,11 @@ func TestManager_MigrateClaims(t *testing.T) { claimsDir := filepath.Join(home, "claims") config.FileSystem.Mkdir(claimsDir, 0755) - config.TestContext.AddTestFile(filepath.Join("testdata", tc.fileName+".json"), filepath.Join(claimsDir, tc.fileName+".json")) + config.TestContext.AddTestFile(filepath.Join("testdata/claims", tc.fileName+".json"), filepath.Join(claimsDir, tc.fileName+".json")) dataStore := crud.NewBackingStore(filesystem.NewStore(*config.Config, hclog.NewNullLogger())) mgr := NewManager(config.Config, dataStore) - claimStore := claim.NewClaimStore(crud.NewBackingStore(mgr), nil, nil) + claimStore := claim.NewClaimStore(mgr, nil, nil) logfilePath, err := mgr.Migrate() require.NoError(t, err, "Migrate failed") @@ -135,7 +136,7 @@ func TestManager_MigrateClaims(t *testing.T) { require.NotNil(t, c, "claim should be populated") assert.Equal(t, installation, c.Installation, "claim.Installation was not populated") - assert.Contains(t, config.TestContext.GetError(), "!!! Migrating claims data", "the claim should have been migrated") + assert.Contains(t, config.TestContext.GetError(), "Migrating claims data", "the claim should have been migrated") if tc.migrateName { assert.Contains(t, config.TestContext.GetError(), "claim.Name to claim.Installation", "the claim should have been migrated from Name -> Installation") } else { @@ -145,11 +146,6 @@ func TestManager_MigrateClaims(t *testing.T) { logfile, err := config.FileSystem.ReadFile(logfilePath) require.NoError(t, err, "error reading logfile") assert.Equal(t, config.TestContext.GetError(), string(logfile), "the migration should have been copied to both stderr and the logfile") - - // Read a second time, this time there shouldn't be a migration - config.TestContext.ClearOutputs() - _, err = claimStore.ReadLastClaim(installation) - assert.NotContains(t, config.TestContext.GetError(), "!!! Migrating claims data", "the claim should have been migrated a second time") }) } @@ -163,13 +159,13 @@ func TestManager_MigrateClaims(t *testing.T) { dataStore := crud.NewBackingStore(filesystem.NewStore(*config.Config, hclog.NewNullLogger())) mgr := NewManager(config.Config, dataStore) - claimStore := claim.NewClaimStore(crud.NewBackingStore(mgr), nil, nil) + claimStore := claim.NewClaimStore(mgr, nil, nil) c, err := claimStore.ReadLastClaim(installation) require.NoError(t, err, "could not read claim") require.NotNil(t, c, "claim should be populated") assert.Equal(t, installation, c.Installation, "claim.Installation was not populated") - assert.NotContains(t, config.TestContext.GetError(), "!!! Migrating claims data", "the claim should have been migrated") + assert.NotContains(t, config.TestContext.GetError(), "Migrating claims data", "the claim should have been migrated") }) } @@ -181,13 +177,21 @@ func TestManager_NoMigrationEmptyHome(t *testing.T) { dataStore := crud.NewBackingStore(filesystem.NewStore(*config.Config, hclog.NewNullLogger())) mgr := NewManager(config.Config, dataStore) - claimStore := claim.NewClaimStore(crud.NewBackingStore(mgr), nil, nil) + claimStore := claim.NewClaimStore(mgr, nil, nil) _, err := claimStore.ListInstallations() require.NoError(t, err, "ListInstallations failed") + + credStore := credentials.NewCredentialStore(mgr) + _, err = credStore.List() + require.NoError(t, err, "List credentials failed") + + // TODO (carolynvs): use a parameterstore once it's moved to cnab-go + _, err = mgr.List("parameters", "") + require.NoError(t, err, "List parameters failed") } -func TestManager_MigrateInstall(t *testing.T) { +func TestManager_MigrateInstallClaim(t *testing.T) { config := config.NewTestConfig(t) home := config.TestContext.UseFilesystem() config.SetHomeDir(home) @@ -199,7 +203,7 @@ func TestManager_MigrateInstall(t *testing.T) { claimsDir := filepath.Join(home, "claims") config.FileSystem.Mkdir(claimsDir, 0755) - config.TestContext.AddTestFile("testdata/installed.json", filepath.Join(claimsDir, "installed.json")) + config.TestContext.AddTestFile("testdata/claims/installed.json", filepath.Join(claimsDir, "installed.json")) _, err := mgr.Migrate() require.NoError(t, err, "Migrate failed") @@ -218,7 +222,7 @@ func TestManager_MigrateInstall(t *testing.T) { assert.Equal(t, claim.StatusSucceeded, i.GetLastStatus()) } -func TestManager_MigrateUpgrade(t *testing.T) { +func TestManager_MigrateUpgradeClaim(t *testing.T) { config := config.NewTestConfig(t) home := config.TestContext.UseFilesystem() config.SetHomeDir(home) @@ -230,7 +234,7 @@ func TestManager_MigrateUpgrade(t *testing.T) { claimsDir := filepath.Join(home, "claims") config.FileSystem.Mkdir(claimsDir, 0755) - config.TestContext.AddTestFile("testdata/upgraded.json", filepath.Join(claimsDir, "upgraded.json")) + config.TestContext.AddTestFile("testdata/claims/upgraded.json", filepath.Join(claimsDir, "upgraded.json")) _, err := mgr.Migrate() require.NoError(t, err, "Migrate failed") @@ -252,3 +256,84 @@ func TestManager_MigrateUpgrade(t *testing.T) { assert.Equal(t, claim.ActionInstall, installClaim.Action) assert.Equal(t, claim.StatusUnknown, installClaim.GetStatus()) } + +func TestManager_ShouldMigrateCredentials(t *testing.T) { + testcases := []struct { + name string + credentialsVersion string + wantMigrate bool + }{ + {"old schema", "cnab-credentialsets-1.0.0-DRAFT", true}, + {"missing schema", "", true}, + {"current schema", credentials.CNABSpecVersion, false}, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + c := config.NewTestConfig(t) + storage := crud.NewBackingStore(crud.NewMockStore()) + p := NewManager(c.Config, storage) + + p.schema = Schema{ + Credentials: schema.Version(tc.credentialsVersion), + } + + assert.Equal(t, tc.wantMigrate, p.ShouldMigrateCredentials()) + }) + } +} + +func TestManager_MigrateCredentials(t *testing.T) { + config := config.NewTestConfig(t) + home := config.TestContext.UseFilesystem() + config.SetHomeDir(home) + defer config.TestContext.Cleanup() + + credsDir := filepath.Join(home, "credentials") + config.FileSystem.Mkdir(credsDir, 0755) + config.TestContext.AddTestFile(filepath.Join("testdata/credentials", "mybun.json"), filepath.Join(credsDir, "mybun.json")) + + dataStore := crud.NewBackingStore(filesystem.NewStore(*config.Config, hclog.NewNullLogger())) + mgr := NewManager(config.Config, dataStore) + credStore := credentials.NewCredentialStore(mgr) + + logfilePath, err := mgr.Migrate() + require.NoError(t, err, "Migrate failed") + + c, err := credStore.Read("mybun") + require.NoError(t, err, "Read credential failed") + + assert.Equal(t, credentials.DefaultSchemaVersion, c.SchemaVersion, "credential was not migrated") + + logfile, err := config.FileSystem.ReadFile(logfilePath) + require.NoError(t, err, "error reading logfile") + assert.Equal(t, config.TestContext.GetError(), string(logfile), "the migration should have been copied to both stderr and the logfile") +} + +func TestManager_ShouldMigrateParameters(t *testing.T) { + testcases := []struct { + name string + parametersVersion string + wantMigrate bool + }{ + {"old schema", "cnab-parametersets-1.0.0-DRAFT", true}, + {"missing schema", "", true}, + {"current schema", ParameterSetCNABSpecVersion, false}, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + c := config.NewTestConfig(t) + storage := crud.NewBackingStore(crud.NewMockStore()) + p := NewManager(c.Config, storage) + + p.schema = Schema{ + Parameters: schema.Version(tc.parametersVersion), + } + + assert.Equal(t, tc.wantMigrate, p.ShouldMigrateParameters()) + }) + } +} + +// NOTE: TestManager_MigrateParameters is in parameterset_test.go to avoid a circular dependency diff --git a/pkg/storage/provider.go b/pkg/storage/provider.go new file mode 100644 index 000000000..a8533669c --- /dev/null +++ b/pkg/storage/provider.go @@ -0,0 +1,8 @@ +package storage + +// StorageProvider handles high level functions over Porter's storage systems such as +// migrating data formats. +type StorageProvider interface { + // Migrate executes a migration on any/all of Porter's storage sub-systems. + Migrate() (string, error) +} diff --git a/pkg/storage/schema.go b/pkg/storage/schema.go index 742c5157c..53be989a6 100644 --- a/pkg/storage/schema.go +++ b/pkg/storage/schema.go @@ -5,4 +5,5 @@ import "github.com/cnabio/cnab-go/schema" type Schema struct { Claims schema.Version `json:"claims"` Credentials schema.Version `json:"credentials"` + Parameters schema.Version `json:"parameters"` } diff --git a/pkg/storage/testdata/has-installation.json b/pkg/storage/testdata/claims/has-installation.json similarity index 100% rename from pkg/storage/testdata/has-installation.json rename to pkg/storage/testdata/claims/has-installation.json diff --git a/pkg/storage/testdata/has-name.json b/pkg/storage/testdata/claims/has-name.json similarity index 100% rename from pkg/storage/testdata/has-name.json rename to pkg/storage/testdata/claims/has-name.json diff --git a/pkg/storage/testdata/installed.json b/pkg/storage/testdata/claims/installed.json similarity index 100% rename from pkg/storage/testdata/installed.json rename to pkg/storage/testdata/claims/installed.json diff --git a/pkg/storage/testdata/upgraded.json b/pkg/storage/testdata/claims/upgraded.json similarity index 100% rename from pkg/storage/testdata/upgraded.json rename to pkg/storage/testdata/claims/upgraded.json diff --git a/pkg/storage/testdata/credentials/mybun.json b/pkg/storage/testdata/credentials/mybun.json new file mode 100644 index 000000000..b084a08c5 --- /dev/null +++ b/pkg/storage/testdata/credentials/mybun.json @@ -0,0 +1,19 @@ +{ + "name": "mybun", + "created": "2020-01-01T00:00:00Z", + "modified": "2020-01-01T00:00:00Z", + "credentials": [ + { + "name": "CLIENT_ID", + "source": { + "env": "OAUTH_CLIENT_ID" + } + }, + { + "name": "CLIENT_SECRET", + "source": { + "env": "OAUTH_CLIENT_SECRET" + } + } + ] +} \ No newline at end of file diff --git a/pkg/storage/testdata/migrated/credentials/mybun.json b/pkg/storage/testdata/migrated/credentials/mybun.json new file mode 100644 index 000000000..9862300c1 --- /dev/null +++ b/pkg/storage/testdata/migrated/credentials/mybun.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": "1.0.0-DRAFT+b6c701f", + "name": "mybun", + "created": "2020-01-01T00:00:00Z", + "modified": "2020-01-01T00:00:00Z", + "credentials": [ + { + "name": "CLIENT_ID", + "source": { + "env": "OAUTH_CLIENT_ID" + } + }, + { + "name": "CLIENT_SECRET", + "source": { + "env": "OAUTH_CLIENT_SECRET" + } + } + ] +} \ No newline at end of file diff --git a/pkg/storage/testdata/migrated/parameters/mybun.json b/pkg/storage/testdata/migrated/parameters/mybun.json new file mode 100644 index 000000000..c6288df04 --- /dev/null +++ b/pkg/storage/testdata/migrated/parameters/mybun.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": "1.0.0-DRAFT+TODO", + "name": "mybun", + "created": "2020-01-01T00:00:00Z", + "modified": "2020-01-01T00:00:00Z", + "parameters": [ + { + "name": "verbose", + "source": { + "value": "6" + } + }, + { + "name": "db-password", + "source": { + "secret": "mysql-password" + } + } + ] +} \ No newline at end of file diff --git a/pkg/storage/testdata/migrated/schema.json b/pkg/storage/testdata/migrated/schema.json index a705fb888..a6c526b79 100644 --- a/pkg/storage/testdata/migrated/schema.json +++ b/pkg/storage/testdata/migrated/schema.json @@ -1,3 +1,5 @@ { - "claims": "cnab-claim-1.0.0-DRAFT+b5ed2f3" + "claims": "cnab-claim-1.0.0-DRAFT+b5ed2f3", + "credentials": "cnab-credentialsets-1.0.0-DRAFT+b6c701f", + "parameters": "cnab-parametersets-1.0.0-DRAFT+TODO" } \ No newline at end of file diff --git a/pkg/storage/testdata/parameters/mybun.json b/pkg/storage/testdata/parameters/mybun.json new file mode 100644 index 000000000..c18c5fbab --- /dev/null +++ b/pkg/storage/testdata/parameters/mybun.json @@ -0,0 +1,19 @@ +{ + "name": "mybun", + "created": "2020-01-01T00:00:00Z", + "modified": "2020-01-01T00:00:00Z", + "parameters": [ + { + "name": "verbose", + "source": { + "value": "6" + } + }, + { + "name": "db-password", + "source": { + "secret": "mysql-password" + } + } + ] +} \ No newline at end of file diff --git a/tests/claim_migration_test.go b/tests/claim_migration_test.go index e69a26e6e..77a2f8ecb 100644 --- a/tests/claim_migration_test.go +++ b/tests/claim_migration_test.go @@ -30,7 +30,7 @@ func TestClaimMigration_List(t *testing.T) { // Create unmigrated claim data p.FileSystem.Mkdir(claimsDir, 0755) - p.AddTestFile(filepath.Join("../pkg/storage/testdata", "upgraded.json"), filepath.Join(home, "claims", "mybun.json")) + p.AddTestFile(filepath.Join("../pkg/storage/testdata/claims", "upgraded.json"), filepath.Join(home, "claims", "mybun.json")) err = p.MigrateStorage() require.NoError(t, err, "MigrateStorage failed")