From 63a1db3612edb44d7392a46192b4becfff0bbc86 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 12 Oct 2023 04:17:40 +0200 Subject: [PATCH 1/4] tests/upgrade: Add extra tests before runtime upgrade --- tests/upgrade/post/go.mod | 2 +- tests/upgrade/post/go.sum | 6 +- .../scenario/e2e/runtime/secure_upgrade.go | 117 ++++++++++++------ .../scenario/e2e/runtime/secure_upgrade.go | 20 ++- 4 files changed, 93 insertions(+), 52 deletions(-) diff --git a/tests/upgrade/post/go.mod b/tests/upgrade/post/go.mod index 66e8c05ea19..25142f6a08e 100644 --- a/tests/upgrade/post/go.mod +++ b/tests/upgrade/post/go.mod @@ -15,7 +15,7 @@ replace ( golang.org/x/crypto/ed25519 => github.com/oasisprotocol/curve25519-voi/primitives/ed25519 v0.0.0-20210505121811-294cf0fbfb43 ) -require github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231010115211-e67f164f7842 +require github.com/oasisprotocol/oasis-core/go v0.2300.1-0.20231012072316-cf039bf94a0e require ( github.com/a8m/envsubst v1.4.2 // indirect diff --git a/tests/upgrade/post/go.sum b/tests/upgrade/post/go.sum index 13c0f8aa949..92c8c4336e8 100644 --- a/tests/upgrade/post/go.sum +++ b/tests/upgrade/post/go.sum @@ -510,10 +510,8 @@ github.com/oasisprotocol/cometbft v0.37.2-oasis1 h1:7yr/5f0vV1bQLKoZHFhs3mcHay3n github.com/oasisprotocol/cometbft v0.37.2-oasis1/go.mod h1:w7GZmT0jJ/i9Uz9ZWtzE3chuNZ3phUboSrFzPZIoYts= github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce h1:/pEpMk55wH0X+E5zedGEMOdLuWmV8P4+4W3+LZaM6kg= github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= -github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231007081532-553e006f90d6 h1:nI4czq/c7ZapUqSJuE/LflHyWLu/LnJN81nOBRmOgLQ= -github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231007081532-553e006f90d6/go.mod h1:lWZtDYY7ziW20hVzqnSt8UQlUkXNHisG3bS6Q3eTsqQ= -github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231010115211-e67f164f7842 h1:2MXcN7dTHnHKjCDTVjP0011vtkYU1hQxqL4JmdQAU3Q= -github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231010115211-e67f164f7842/go.mod h1:lWZtDYY7ziW20hVzqnSt8UQlUkXNHisG3bS6Q3eTsqQ= +github.com/oasisprotocol/oasis-core/go v0.2300.1-0.20231012072316-cf039bf94a0e h1:M0/NXwj72rsdGnqeY6ZtYBjyFbFRtmR7Ccmmke1CdgI= +github.com/oasisprotocol/oasis-core/go v0.2300.1-0.20231012072316-cf039bf94a0e/go.mod h1:lWZtDYY7ziW20hVzqnSt8UQlUkXNHisG3bS6Q3eTsqQ= github.com/oasisprotocol/safeopen v0.0.0-20200528085122-e01cfdfc7661 h1:MB73kGMtuMGS+6VDoU/mitzff7Cu+aZo9ta5wabuxVA= github.com/oasisprotocol/safeopen v0.0.0-20200528085122-e01cfdfc7661/go.mod h1:SwBxaVibf6Sr2IZ6M3WnUue0yp8dPLAo1riQRNQ60+g= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= diff --git a/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go b/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go index 3229aa0e282..0fb6eb9ac54 100644 --- a/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go +++ b/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go @@ -23,35 +23,58 @@ var ( // consensus verification, and test the functionality of master secret rotations. SecureUpgrade scenario.Scenario = newSecureUpgradeImpl() - secureUpgradePreTestScenario = runtime.NewTestClientScenario([]interface{}{ - // Test that pre-upgrade key/value pairs are still in the database. - runtime.GetKeyValueTx{Key: "pre-key1", Response: "pre-value1", Encrypted: false}, - runtime.GetKeyValueTx{Key: "pre-key2", Response: "pre-value2", Encrypted: true, Generation: 0}, - runtime.GetKeyValueTx{Key: "pre-key1", Response: "", Encrypted: true, Generation: 0}, - runtime.GetKeyValueTx{Key: "pre-key2", Response: "", Encrypted: false}, - // Insert post-upgrade key/value pairs, one in plaintext and one encrypted. - runtime.InsertKeyValueTx{Key: "post-key1", Value: "post-value1", Response: "", Encrypted: false}, - runtime.InsertKeyValueTx{Key: "post-key2", Value: "post-value2", Response: "", Encrypted: true, Generation: 0}, - // Test that post-upgrade key/value pairs were correctly inserted into the database. - runtime.GetKeyValueTx{Key: "post-key1", Response: "post-value1", Encrypted: false}, - runtime.GetKeyValueTx{Key: "post-key2", Response: "post-value2", Encrypted: true, Generation: 0}, - // Test that post-upgrade key/value pairs are either in plaintext or encrypted. - runtime.GetKeyValueTx{Key: "post-key1", Response: "", Encrypted: true, Generation: 0}, - runtime.GetKeyValueTx{Key: "post-key2", Response: "", Encrypted: false}, + // verifyNodeUpgradeTestClientScenario tests that values inserted before node upgrade are still + // in the database. + verifyNodeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ + runtime.GetKeyValueTx{Key: "node-key1", Response: "node-value1", Encrypted: false}, + runtime.GetKeyValueTx{Key: "node-key2", Response: "node-value2", Encrypted: true, Generation: 0}, + runtime.GetKeyValueTx{Key: "node-key1", Response: "", Encrypted: true, Generation: 0}, + runtime.GetKeyValueTx{Key: "node-key2", Response: "", Encrypted: false}, }) - secureUpgradePostTestScenario = runtime.NewTestClientScenario([]interface{}{ - // Test that pre-upgrade key/value pairs are still in the database. - runtime.GetKeyValueTx{Key: "pre-key1", Response: "pre-value1", Encrypted: false}, - runtime.GetKeyValueTx{Key: "pre-key2", Response: "pre-value2", Encrypted: true, Generation: 0}, - runtime.GetKeyValueTx{Key: "pre-key1", Response: "", Encrypted: true, Generation: 0}, - runtime.GetKeyValueTx{Key: "pre-key2", Response: "", Encrypted: false}, - // Test that post-upgrade key/value pairs are still in the database. - runtime.GetKeyValueTx{Key: "post-key1", Response: "post-value1", Encrypted: false}, - runtime.GetKeyValueTx{Key: "post-key2", Response: "post-value2", Encrypted: true, Generation: 0}, - runtime.GetKeyValueTx{Key: "post-key1", Response: "", Encrypted: true, Generation: 0}, - runtime.GetKeyValueTx{Key: "post-key2", Response: "", Encrypted: false}, - // Test master secret generations. + // keyManagerUpgradeTestClientScenario inserts values before key manager upgrade. + keyManagerUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ + runtime.InsertKeyValueTx{Key: "km-key1", Value: "km-value1", Response: "", Encrypted: false}, + runtime.InsertKeyValueTx{Key: "km-key2", Value: "km-value2", Response: "", Encrypted: true, Generation: 0}, + }) + + // verifyKeyManagerUpgradeTestClientScenario tests that values inserted before key manager + // upgrade are still in the database. + verifyKeyManagerUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ + runtime.GetKeyValueTx{Key: "km-key1", Response: "km-value1", Encrypted: false}, + runtime.GetKeyValueTx{Key: "km-key2", Response: "km-value2", Encrypted: true, Generation: 0}, + runtime.GetKeyValueTx{Key: "km-key1", Response: "", Encrypted: true, Generation: 0}, + runtime.GetKeyValueTx{Key: "km-key2", Response: "", Encrypted: false}, + }) + + // runtimeUpgradeTestClientScenario inserts values before key-value runtime upgrade. + runtimeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ + runtime.InsertKeyValueTx{Key: "rt-key1", Value: "rt-value1", Response: "", Encrypted: false}, + runtime.InsertKeyValueTx{Key: "rt-key2", Value: "rt-value2", Response: "", Encrypted: true, Generation: 0}, + }) + + // verifyRuntimeUpgradeTestClientScenario tests that values inserted before key-value runtime + // upgrade are still in the database. + verifyRuntimeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ + runtime.GetKeyValueTx{Key: "rt-key1", Response: "rt-value1", Encrypted: false}, + runtime.GetKeyValueTx{Key: "rt-key2", Response: "rt-value2", Encrypted: true, Generation: 0}, + runtime.GetKeyValueTx{Key: "rt-key1", Response: "", Encrypted: true, Generation: 0}, + runtime.GetKeyValueTx{Key: "rt-key2", Response: "", Encrypted: false}, + }) + + // networkUpgradeTestClientScenario tests that everything works after the upgrade has been + // completed. + networkUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ + runtime.InsertKeyValueTx{Key: "network-key1", Value: "network-value1", Response: "", Encrypted: false}, + runtime.InsertKeyValueTx{Key: "network-key2", Value: "network-value2", Response: "", Encrypted: true}, + runtime.GetKeyValueTx{Key: "network-key1", Response: "network-value1", Encrypted: false}, + runtime.GetKeyValueTx{Key: "network-key2", Response: "network-value2", Encrypted: true}, + runtime.GetKeyValueTx{Key: "network-key1", Response: "", Encrypted: true}, + runtime.GetKeyValueTx{Key: "network-key2", Response: "", Encrypted: false}, + }) + + // masterSecretGenerationsTestClientScenario tests master secret generations. + masterSecretGenerationsTestClientScenario = runtime.NewTestClientScenario([]interface{}{ runtime.InsertKeyValueTx{Key: "msgn-key1", Value: "msgn-value1", Response: "", Encrypted: true, Generation: 0}, runtime.InsertKeyValueTx{Key: "msgn-key2", Value: "msgn-value2", Response: "", Encrypted: true, Generation: 1}, runtime.InsertKeyValueTx{Key: "msgn-key3", Value: "msgn-value3", Response: "", Encrypted: true, Generation: 2}, @@ -60,8 +83,9 @@ var ( runtime.GetKeyValueTx{Key: "msgn-key3", Response: "msgn-value3", Encrypted: true, Generation: 2}, }) - invalidSecureUpgradeTestScenario = runtime.NewTestClientScenario([]interface{}{ - // Test that key/value pair cannot be encrypted with a master secret from the future. + // futureMasterSecretGenerationsTestClientScenario tests that future master secrets are not + // available. + futureMasterSecretGenerationsTestClientScenario = runtime.NewTestClientScenario([]interface{}{ runtime.InsertKeyValueTx{Key: "msgn-key4", Value: "msgn-value4", Response: "", Encrypted: true, Generation: 1000}, }) ) @@ -77,7 +101,7 @@ func newSecureUpgradeImpl() scenario.Scenario { return &secureUpgradeImpl{ Scenario: *runtime.NewScenario( "trust-root/secure-upgrade", - runtime.NewTestClient().WithSeed("post-seed").WithScenario(secureUpgradePreTestScenario), + runtime.NewTestClient().WithSeed("post-seed"), ), } } @@ -175,10 +199,17 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { return fmt.Errorf("failed to refresh runtime bundle: %w", err) } - // Start all workers and run the test client. + // Start all workers. if err = sc.startClientAndComputeWorkers(ctx); err != nil { return nil } + + // Test after node upgrade. + sc.TestClient.WithScenario(runtime.JoinTestClientScenarios( + verifyNodeUpgradeTestClientScenario, + keyManagerUpgradeTestClientScenario, + verifyKeyManagerUpgradeTestClientScenario, + )) if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil { return err } @@ -208,6 +239,16 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { return err } + // Test after key manager upgrade. + sc.TestClient.WithScenario(runtime.JoinTestClientScenarios( + verifyKeyManagerUpgradeTestClientScenario, + runtimeUpgradeTestClientScenario, + verifyRuntimeUpgradeTestClientScenario, + )) + if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil { + return err + } + // Enable master secret rotations. nonce, err = sc.TestEntityNonce(ctx) if err != nil { @@ -231,15 +272,19 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { return err } - // Run the test client again. - sc.TestClient.WithScenario(secureUpgradePostTestScenario) + // Test after key-value runtime upgrade. + sc.TestClient.WithScenario(runtime.JoinTestClientScenarios( + verifyRuntimeUpgradeTestClientScenario, + networkUpgradeTestClientScenario, + masterSecretGenerationsTestClientScenario, + )) if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil { return err } - // Run the test client again, but expect it to fail this time as the requested master secret - // generation is not yet available. - sc.TestClient.WithScenario(invalidSecureUpgradeTestScenario) + // Test future master secrets, but expect the test to fail as the requested master secret + // generation should not be available. + sc.TestClient.WithScenario(futureMasterSecretGenerationsTestClientScenario) err = sc.RunTestClientAndCheckLogs(ctx, childEnv) switch { case err == nil: diff --git a/tests/upgrade/pre/scenario/e2e/runtime/secure_upgrade.go b/tests/upgrade/pre/scenario/e2e/runtime/secure_upgrade.go index 61ee6201c57..37d15b44446 100644 --- a/tests/upgrade/pre/scenario/e2e/runtime/secure_upgrade.go +++ b/tests/upgrade/pre/scenario/e2e/runtime/secure_upgrade.go @@ -21,16 +21,14 @@ var ( // consensus verification, and test the functionality of master secret rotations. SecureUpgrade scenario.Scenario = newSecureUpgradeImpl() - secureUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ - // Insert pre-upgrade key/value pairs, one in plaintext and one encrypted. - runtime.InsertKeyValueTx{Key: "pre-key1", Value: "pre-value1", Response: "", Encrypted: false}, - runtime.InsertKeyValueTx{Key: "pre-key2", Value: "pre-value2", Response: "", Encrypted: true}, - // Test that pre-upgrade key/value pairs were correctly inserted into the database. - runtime.GetKeyValueTx{Key: "pre-key1", Response: "pre-value1", Encrypted: false}, - runtime.GetKeyValueTx{Key: "pre-key2", Response: "pre-value2", Encrypted: true}, - // Test that pre-upgrade key/value pairs are either in plaintext or encrypted. - runtime.GetKeyValueTx{Key: "pre-key1", Response: "", Encrypted: true}, - runtime.GetKeyValueTx{Key: "pre-key2", Response: "", Encrypted: false}, + // nodeUpgradeTestClientScenario tests that everything works before the upgrade starts. + nodeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{ + runtime.InsertKeyValueTx{Key: "node-key1", Value: "node-value1", Response: "", Encrypted: false}, + runtime.InsertKeyValueTx{Key: "node-key2", Value: "node-value2", Response: "", Encrypted: true}, + runtime.GetKeyValueTx{Key: "node-key1", Response: "node-value1", Encrypted: false}, + runtime.GetKeyValueTx{Key: "node-key2", Response: "node-value2", Encrypted: true}, + runtime.GetKeyValueTx{Key: "node-key1", Response: "", Encrypted: true}, + runtime.GetKeyValueTx{Key: "node-key2", Response: "", Encrypted: false}, }) ) @@ -42,7 +40,7 @@ func newSecureUpgradeImpl() scenario.Scenario { return &secureUpgradeImpl{ TrustRootImpl: *runtime.NewTrustRootImpl( "secure-upgrade", - runtime.NewKVTestClient().WithScenario(secureUpgradeTestClientScenario), + runtime.NewKVTestClient().WithScenario(nodeUpgradeTestClientScenario), ), } } From a230f766660c09d90734ff4de45e751037c7aa41 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 12 Oct 2023 03:59:00 +0200 Subject: [PATCH 2/4] tests/upgrade: Test encryption/decryption in upgrade test --- .changelog/5399.feature.md | 1 + .../scenario/e2e/runtime/secure_upgrade.go | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 .changelog/5399.feature.md diff --git a/.changelog/5399.feature.md b/.changelog/5399.feature.md new file mode 100644 index 00000000000..1b45f5eec29 --- /dev/null +++ b/.changelog/5399.feature.md @@ -0,0 +1 @@ +tests/upgrade: Test encryption/decryption in upgrade test diff --git a/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go b/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go index 0fb6eb9ac54..94e5040536f 100644 --- a/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go +++ b/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go @@ -7,7 +7,9 @@ import ( "os" "path/filepath" "strings" + "time" + consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis/cli" @@ -204,11 +206,14 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { return nil } + encryptDecryptTestClientScenario := sc.newEncryptDecryptTestClientScenario(ctx, childEnv) + // Test after node upgrade. sc.TestClient.WithScenario(runtime.JoinTestClientScenarios( verifyNodeUpgradeTestClientScenario, keyManagerUpgradeTestClientScenario, verifyKeyManagerUpgradeTestClientScenario, + encryptDecryptTestClientScenario, )) if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil { return err @@ -244,6 +249,7 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { verifyKeyManagerUpgradeTestClientScenario, runtimeUpgradeTestClientScenario, verifyRuntimeUpgradeTestClientScenario, + encryptDecryptTestClientScenario, )) if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil { return err @@ -277,6 +283,7 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { verifyRuntimeUpgradeTestClientScenario, networkUpgradeTestClientScenario, masterSecretGenerationsTestClientScenario, + encryptDecryptTestClientScenario, )) if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil { return err @@ -328,3 +335,24 @@ func (sc *secureUpgradeImpl) startClientAndComputeWorkers(ctx context.Context) e return nil } + +func (sc *secureUpgradeImpl) newEncryptDecryptTestClientScenario(ctx context.Context, childEnv *env.Env) runtime.TestClientScenario { + return func(submit func(req interface{}) error) error { + // Fetch current epoch. + epoch, err := sc.Net.Controller().Beacon.GetEpoch(ctx, consensus.HeightLatest) + if err != nil { + return fmt.Errorf("failed to get current epoch: %w", err) + } + + // Use unique values. + now := time.Now().UnixNano() + keyPairID := fmt.Sprintf("key-pair-id-%d", now) + message := fmt.Sprintf("message-%d", now) + + return submit(runtime.EncryptDecryptTx{ + Epoch: epoch, + KeyPairID: keyPairID, + Message: []byte(message), + }) + } +} From 3d61ef6c1d46117dbdacb1f97dbeb7fa42da6bd0 Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 12 Oct 2023 11:44:30 +0200 Subject: [PATCH 3/4] tests/upgrade: Upload artifacts --- .buildkite/code.pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/code.pipeline.yml b/.buildkite/code.pipeline.yml index 3b6bcee6f15..a585a30dc4b 100644 --- a/.buildkite/code.pipeline.yml +++ b/.buildkite/code.pipeline.yml @@ -382,6 +382,7 @@ steps: - label: E2E upgrade tests - sgx1 timeout_in_minutes: 60 command: + - trap 'buildkite-agent artifact upload "/tmp/oasis-{post,pre}-upgrade/e2e/**/*.log;/tmp/oasis-{post,pre}-upgrade/e2e/**/genesis.json;/tmp/oasis-{post,pre}-upgrade/e2e/**/runtime_genesis.json"' EXIT - .buildkite/scripts/download_e2e_test_artifacts.sh - .buildkite/scripts/test_upgrade.sh env: From c8f3c569727e3f1172ee1c55b65c99c2e508517c Mon Sep 17 00:00:00 2001 From: Peter Nose Date: Thu, 12 Oct 2023 13:24:05 +0200 Subject: [PATCH 4/4] tests/upgrade: Enable master secret rotations after runtime upgrade --- .../scenario/e2e/runtime/secure_upgrade.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go b/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go index 94e5040536f..4a7c82fb706 100644 --- a/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go +++ b/tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go @@ -255,26 +255,26 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error { return err } - // Enable master secret rotations. + // Upgrade the compute runtime. nonce, err = sc.TestEntityNonce(ctx) if err != nil { return err } - if err = sc.UpdateRotationInterval(ctx, childEnv, cli, 1, nonce); err != nil { - return err - } - - // Wait until at least 3 secrets are generated. - if _, err = sc.WaitMasterSecret(ctx, 3); err != nil { + if err := sc.UpgradeComputeRuntime(ctx, childEnv, cli, sc.upgradedRuntimeIndex, nonce); err != nil { return err } - // Upgrade the compute runtime. + // Enable master secret rotations. nonce, err = sc.TestEntityNonce(ctx) if err != nil { return err } - if err := sc.UpgradeComputeRuntime(ctx, childEnv, cli, sc.upgradedRuntimeIndex, nonce); err != nil { + if err = sc.UpdateRotationInterval(ctx, childEnv, cli, 1, nonce); err != nil { + return err + } + + // Wait until at least 3 secrets are generated. + if _, err = sc.WaitMasterSecret(ctx, 3); err != nil { return err }