diff --git a/test/e2e/common/common.go b/test/e2e/common/common.go index 2aa29f05dd..473dfc83f8 100644 --- a/test/e2e/common/common.go +++ b/test/e2e/common/common.go @@ -8,6 +8,7 @@ import ( "flag" "os" "os/user" + "path/filepath" "strconv" "testing" "time" @@ -30,6 +31,22 @@ var ( Architectures = []string{"amd64", "arm64"} CreateInfra = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing") DeleteInfra = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing") + + // kubeconfig: path to kubeconfig file, in not provided, + // a new k8s cluster will be created + KubeConfig = flag.String("kubeConfig", "", "Path to kubeconfig file") +) + +var ( + RetinaChartPath = func(rootDir string) string { + return filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") + } + RetinaAdvancedProfilePath = func(rootDir string) string { + return filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml") + } + KubeConfigFilePath = func(rootDir string) string { + return filepath.Join(rootDir, "test", "e2e", "test.pem") + } ) func ClusterNameForE2ETest(t *testing.T) string { diff --git a/test/e2e/framework/kubernetes/validate-service.go b/test/e2e/framework/kubernetes/validate-service.go index 4ba7463bc2..9b2a6dabe4 100644 --- a/test/e2e/framework/kubernetes/validate-service.go +++ b/test/e2e/framework/kubernetes/validate-service.go @@ -62,7 +62,7 @@ func (v *ValidateResource) Run() error { return nil } -func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceName, labels string) (bool, error) { +func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, _, labels string) (bool, error) { var serviceList *corev1.ServiceList serviceList, err := clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labels}) if err != nil { diff --git a/test/e2e/infra/azure_temp_infra_setup.go b/test/e2e/infra/azure_temp_infra_setup.go new file mode 100644 index 0000000000..52bf32b8a6 --- /dev/null +++ b/test/e2e/infra/azure_temp_infra_setup.go @@ -0,0 +1,49 @@ +package infra + +import ( + "context" + "crypto/rand" + "math/big" + "os" + "testing" + + "github.com/microsoft/retina/test/e2e/common" + "github.com/microsoft/retina/test/e2e/framework/types" + jobs "github.com/microsoft/retina/test/e2e/jobs" + "github.com/stretchr/testify/require" +) + +func CreateAzureTempK8sInfra(ctx context.Context, t *testing.T, rootDir string) string { + kubeConfigFilePath := common.KubeConfigFilePath(rootDir) + clusterName := common.ClusterNameForE2ETest(t) + + subID := os.Getenv("AZURE_SUBSCRIPTION_ID") + require.NotEmpty(t, subID, "AZURE_SUBSCRIPTION_ID environment variable must be set") + + location := os.Getenv("AZURE_LOCATION") + if location == "" { + nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations)))) + if err != nil { + t.Fatal("Failed to generate a secure random index", err) + } + location = common.AzureLocations[nBig.Int64()] + } + + rg := os.Getenv("AZURE_RESOURCE_GROUP") + if rg == "" { + // Use the cluster name as the resource group name by default. + rg = clusterName + } + + // CreateTestInfra + createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra)) + createTestInfra.Run(ctx) + + t.Cleanup(func() { + err := jobs.DeleteTestInfra(subID, rg, location, *common.DeleteInfra).Run() + if err != nil { + t.Logf("Failed to delete test infrastructure: %v", err) + } + }) + return kubeConfigFilePath +} diff --git a/test/e2e/jobs/jobs.go b/test/e2e/jobs/jobs.go index 375f69aa4e..81b64118a0 100644 --- a/test/e2e/jobs/jobs.go +++ b/test/e2e/jobs/jobs.go @@ -1,9 +1,6 @@ package retina import ( - "fmt" - "time" - "github.com/microsoft/retina/test/e2e/common" "github.com/microsoft/retina/test/e2e/framework/azure" "github.com/microsoft/retina/test/e2e/framework/generic" @@ -13,7 +10,6 @@ import ( "github.com/microsoft/retina/test/e2e/scenarios/dns" "github.com/microsoft/retina/test/e2e/scenarios/drop" "github.com/microsoft/retina/test/e2e/scenarios/latency" - "github.com/microsoft/retina/test/e2e/scenarios/perf" tcp "github.com/microsoft/retina/test/e2e/scenarios/tcp" "github.com/microsoft/retina/test/e2e/scenarios/windows" ) @@ -59,23 +55,19 @@ func CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath string }, nil) } - job.AddStep(&generic.LoadFlags{ - TagEnv: generic.DefaultTagEnv, - ImageNamespaceEnv: generic.DefaultImageNamespace, - ImageRegistryEnv: generic.DefaultImageRegistry, - }, nil) - return job } -func DeleteTestInfra(subID, rg, clusterName, location string) *types.Job { +func DeleteTestInfra(subID, rg, location string, deleteInfra bool) *types.Job { job := types.NewJob("Delete e2e test infrastructure") - job.AddStep(&azure.DeleteResourceGroup{ - SubscriptionID: subID, - ResourceGroupName: rg, - Location: location, - }, nil) + if deleteInfra { + job.AddStep(&azure.DeleteResourceGroup{ + SubscriptionID: subID, + ResourceGroupName: rg, + Location: location, + }, nil) + } return job } @@ -274,50 +266,14 @@ func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace strin return job } -func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job { - job := types.NewJob("Run performance tests") - - benchmarkFile := fmt.Sprintf("netperf-benchmark-%s.json", time.Now().Format("20060102150405")) - resultFile := fmt.Sprintf("netperf-result-%s.json", time.Now().Format("20060102150405")) - regressionFile := fmt.Sprintf("netperf-regression-%s.json", time.Now().Format("20060102150405")) +func LoadGenericFlags() *types.Job { + job := types.NewJob("Loading Generic Flags to env") - job.AddStep(&perf.GetNetworkPerformanceMeasures{ - KubeConfigFilePath: kubeConfigFilePath, - ResultTag: "no-retina", - JsonOutputFile: benchmarkFile, - }, &types.StepOptions{ - SkipSavingParametersToJob: true, - }) - - job.AddStep(&kubernetes.InstallHelmChart{ - Namespace: "kube-system", - ReleaseName: "retina", - KubeConfigFilePath: kubeConfigFilePath, - ChartPath: chartPath, - TagEnv: generic.DefaultTagEnv, + job.AddStep(&generic.LoadFlags{ + TagEnv: generic.DefaultTagEnv, + ImageNamespaceEnv: generic.DefaultImageNamespace, + ImageRegistryEnv: generic.DefaultImageRegistry, }, nil) - job.AddStep(&perf.GetNetworkPerformanceMeasures{ - KubeConfigFilePath: kubeConfigFilePath, - ResultTag: "retina", - JsonOutputFile: resultFile, - }, &types.StepOptions{ - SkipSavingParametersToJob: true, - }) - - job.AddStep(&perf.GetNetworkRegressionResults{ - BaseResultsFile: benchmarkFile, - NewResultsFile: resultFile, - RegressionResultsFile: regressionFile, - }, &types.StepOptions{ - SkipSavingParametersToJob: true, - }) - - job.AddStep(&perf.PublishPerfResults{ - ResultsFile: regressionFile, - }, &types.StepOptions{ - SkipSavingParametersToJob: true, - }) - return job } diff --git a/test/e2e/jobs/perf.go b/test/e2e/jobs/perf.go new file mode 100644 index 0000000000..e3620d8441 --- /dev/null +++ b/test/e2e/jobs/perf.go @@ -0,0 +1,73 @@ +package retina + +import ( + "fmt" + "time" + + "github.com/microsoft/retina/test/e2e/framework/generic" + "github.com/microsoft/retina/test/e2e/framework/kubernetes" + "github.com/microsoft/retina/test/e2e/framework/types" + "github.com/microsoft/retina/test/e2e/scenarios/perf" +) + +func RunPerfTest(kubeConfigFilePath, chartPath, advancedValuePath, retinaMode string) *types.Job { + job := types.NewJob("Run performance tests") + + benchmarkFile := fmt.Sprintf("netperf-benchmark-%s.json", time.Now().Format("20060102150405")) + resultFile := fmt.Sprintf("netperf-result-%s.json", time.Now().Format("20060102150405")) + regressionFile := fmt.Sprintf("netperf-regression-%s.json", time.Now().Format("20060102150405")) + + job.AddStep(&perf.GetNetworkPerformanceMeasures{ + KubeConfigFilePath: kubeConfigFilePath, + ResultTag: "no-retina", + JsonOutputFile: benchmarkFile, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + + job.AddStep(&kubernetes.InstallHelmChart{ + Namespace: "kube-system", + ReleaseName: "retina", + KubeConfigFilePath: kubeConfigFilePath, + ChartPath: chartPath, + TagEnv: generic.DefaultTagEnv, + }, nil) + + if retinaMode == "advanced" { + job.AddStep(&kubernetes.UpgradeRetinaHelmChart{ + Namespace: "kube-system", + ReleaseName: "retina", + KubeConfigFilePath: kubeConfigFilePath, + ChartPath: chartPath, + ValuesFile: advancedValuePath, + TagEnv: generic.DefaultTagEnv, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + } + + job.AddStep(&perf.GetNetworkPerformanceMeasures{ + KubeConfigFilePath: kubeConfigFilePath, + ResultTag: "retina", + JsonOutputFile: resultFile, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + + job.AddStep(&perf.GetNetworkRegressionResults{ + BaseResultsFile: benchmarkFile, + NewResultsFile: resultFile, + RegressionResultsFile: regressionFile, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + + job.AddStep(&perf.PublishPerfResults{ + ResultsFile: regressionFile, + RetinaMode: retinaMode, + }, &types.StepOptions{ + SkipSavingParametersToJob: true, + }) + + return job +} diff --git a/test/e2e/retina_e2e_test.go b/test/e2e/retina_e2e_test.go index 546f596bb5..9cf7c58315 100644 --- a/test/e2e/retina_e2e_test.go +++ b/test/e2e/retina_e2e_test.go @@ -3,8 +3,6 @@ package retina import ( - "crypto/rand" - "math/big" "os" "path/filepath" "testing" @@ -12,6 +10,7 @@ import ( "github.com/microsoft/retina/test/e2e/common" "github.com/microsoft/retina/test/e2e/framework/helpers" "github.com/microsoft/retina/test/e2e/framework/types" + "github.com/microsoft/retina/test/e2e/infra" jobs "github.com/microsoft/retina/test/e2e/jobs" "github.com/stretchr/testify/require" ) @@ -21,57 +20,46 @@ func TestE2ERetina(t *testing.T) { ctx, cancel := helpers.Context(t) defer cancel() - // Truncate the username to 8 characters - clusterName := common.ClusterNameForE2ETest(t) - - subID := os.Getenv("AZURE_SUBSCRIPTION_ID") - require.NotEmpty(t, subID) - - location := os.Getenv("AZURE_LOCATION") - if location == "" { - nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations)))) - if err != nil { - t.Fatalf("Failed to generate a secure random index: %v", err) - } - location = common.AzureLocations[nBig.Int64()] - } - - rg := os.Getenv("AZURE_RESOURCE_GROUP") - if rg == "" { - // Use the cluster name as the resource group name by default. - rg = clusterName - } - cwd, err := os.Getwd() require.NoError(t, err) // Get to root of the repo by going up two directories rootDir := filepath.Dir(filepath.Dir(cwd)) - chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") hubblechartPath := filepath.Join(rootDir, "deploy", "hubble", "manifests", "controller", "helm", "retina") - profilePath := filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml") - kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") - // CreateTestInfra - createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra)) - createTestInfra.Run(ctx) + err = jobs.LoadGenericFlags().Run() + require.NoError(t, err, "failed to load generic flags") - t.Cleanup(func() { - if *common.DeleteInfra { - _ = jobs.DeleteTestInfra(subID, rg, clusterName, location).Run() - } - }) + if *common.KubeConfig == "" { + *common.KubeConfig = infra.CreateAzureTempK8sInfra(ctx, t, rootDir) + } // Install and test Retina basic metrics - basicMetricsE2E := types.NewRunner(t, jobs.InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath, common.TestPodNamespace)) + basicMetricsE2E := types.NewRunner(t, + jobs.InstallAndTestRetinaBasicMetrics( + common.KubeConfigFilePath(rootDir), + common.RetinaChartPath(rootDir), + common.TestPodNamespace), + ) basicMetricsE2E.Run(ctx) // Upgrade and test Retina with advanced metrics - advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace)) + advanceMetricsE2E := types.NewRunner(t, + jobs.UpgradeAndTestRetinaAdvancedMetrics( + common.KubeConfigFilePath(rootDir), + common.RetinaChartPath(rootDir), + common.RetinaAdvancedProfilePath(rootDir), + common.TestPodNamespace), + ) advanceMetricsE2E.Run(ctx) // Install and test Hubble basic metrics - validatehubble := types.NewRunner(t, jobs.ValidateHubble(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace)) + validatehubble := types.NewRunner(t, + jobs.ValidateHubble( + common.KubeConfigFilePath(rootDir), + hubblechartPath, + common.TestPodNamespace), + ) validatehubble.Run(ctx) } diff --git a/test/e2e/retina_perf_test.go b/test/e2e/retina_perf_test.go index 38651bc0ea..7346c64285 100644 --- a/test/e2e/retina_perf_test.go +++ b/test/e2e/retina_perf_test.go @@ -3,8 +3,8 @@ package retina import ( - "crypto/rand" - "math/big" + "flag" + "fmt" "os" "path/filepath" "testing" @@ -12,63 +12,54 @@ import ( "github.com/microsoft/retina/test/e2e/common" "github.com/microsoft/retina/test/e2e/framework/helpers" "github.com/microsoft/retina/test/e2e/framework/types" + "github.com/microsoft/retina/test/e2e/infra" jobs "github.com/microsoft/retina/test/e2e/jobs" "github.com/stretchr/testify/require" ) +var ( + // Add flags for the test + // retina-mode: basic, advanced, hubble + retinaMode = flag.String("retina-mode", "basic", "One of basic or advanced") +) + +func validateRetinaMode(t *testing.T) { + switch *retinaMode { + case "basic", "advanced": + fmt.Printf("Running retina in %s mode\n", *retinaMode) + default: + require.Fail(t, "invalid retina-mode", "must be one of: basic, advanced") + } +} + // This test creates a new k8s cluster runs some network performance tests // saves the data as benchmark information and then installs retina and runs the performance tests // to compare the results and publishes a json with regression information. func TestE2EPerfRetina(t *testing.T) { + validateRetinaMode(t) + ctx, cancel := helpers.Context(t) defer cancel() - clusterName := common.ClusterNameForE2ETest(t) + cwd, err := os.Getwd() + require.NoError(t, err) - subID := os.Getenv("AZURE_SUBSCRIPTION_ID") - require.NotEmpty(t, subID, "AZURE_SUBSCRIPTION_ID environment variable must be set") + // Get to root of the repo by going up two directories + rootDir := filepath.Dir(filepath.Dir(cwd)) - location := os.Getenv("AZURE_LOCATION") - if location == "" { - nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations)))) - if err != nil { - t.Fatal("Failed to generate a secure random index", err) - } - location = common.AzureLocations[nBig.Int64()] - } + err = jobs.LoadGenericFlags().Run() + require.NoError(t, err, "failed to load generic flags") - rg := os.Getenv("AZURE_RESOURCE_GROUP") - if rg == "" { - // Use the cluster name as the resource group name by default. - rg = clusterName + if *common.KubeConfig == "" { + *common.KubeConfig = infra.CreateAzureTempK8sInfra(ctx, t, rootDir) } - cwd, err := os.Getwd() - require.NoError(t, err) - appInsightsKey := os.Getenv(common.AzureAppInsightsKeyEnv) if appInsightsKey == "" { t.Log("No app insights key provided, results will be saved locally at ./ as `netperf-benchmark-*`, `netperf-result-*`, and `netperf-regression-*`") } - // Get to root of the repo by going up two directories - rootDir := filepath.Dir(filepath.Dir(cwd)) - - chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") - kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") - - // CreateTestInfra - createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, true)) - createTestInfra.Run(ctx) - - t.Cleanup(func() { - err := jobs.DeleteTestInfra(subID, rg, clusterName, location).Run() - if err != nil { - t.Logf("Failed to delete test infrastructure: %v", err) - } - }) - // Gather benchmark results then install retina and run the performance tests - runner := types.NewRunner(t, jobs.RunPerfTest(kubeConfigFilePath, chartPath)) + runner := types.NewRunner(t, jobs.RunPerfTest(*common.KubeConfig, common.RetinaChartPath(rootDir), common.RetinaAdvancedProfilePath(rootDir), *retinaMode)) runner.Run(ctx) } diff --git a/test/e2e/scale_test.go b/test/e2e/scale_test.go index 687d32ceb5..c5af899b1a 100644 --- a/test/e2e/scale_test.go +++ b/test/e2e/scale_test.go @@ -49,12 +49,12 @@ func TestE2ERetina_Scale(t *testing.T) { // Get to root of the repo by going up two directories rootDir := filepath.Dir(filepath.Dir(cwd)) - chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina") - kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem") + err = jobs.LoadGenericFlags().Run() + require.NoError(t, err, "failed to load generic flags") // Scale test parameters opt := jobs.DefaultScaleTestOptions() - opt.KubeconfigPath = kubeConfigFilePath + opt.KubeconfigPath = common.KubeConfigFilePath(rootDir) NumDeployments := os.Getenv("NUM_DEPLOYMENTS") NumReplicas := os.Getenv("NUM_REPLICAS") @@ -90,13 +90,11 @@ func TestE2ERetina_Scale(t *testing.T) { opt.LabelsToGetMetrics = map[string]string{"k8s-app": "retina"} // CreateTestInfra - createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra)) + createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, common.KubeConfigFilePath(rootDir), *common.CreateInfra)) createTestInfra.Run(ctx) t.Cleanup(func() { - if *common.DeleteInfra { - _ = jobs.DeleteTestInfra(subID, rg, clusterName, location).Run() - } + _ = jobs.DeleteTestInfra(subID, rg, location, *common.DeleteInfra).Run() }) fqdn, err := azure.GetFqdnFn(subID, rg, clusterName) @@ -104,11 +102,11 @@ func TestE2ERetina_Scale(t *testing.T) { opt.AdditionalTelemetryProperty["clusterFqdn"] = fqdn // Install Retina - installRetina := types.NewRunner(t, jobs.InstallRetina(kubeConfigFilePath, chartPath)) + installRetina := types.NewRunner(t, jobs.InstallRetina(common.KubeConfigFilePath(rootDir), common.RetinaChartPath(rootDir))) installRetina.Run(ctx) t.Cleanup(func() { - _ = jobs.UninstallRetina(kubeConfigFilePath, chartPath).Run() + _ = jobs.UninstallRetina(common.KubeConfigFilePath(rootDir), common.RetinaChartPath(rootDir)).Run() }) scale := types.NewRunner(t, jobs.ScaleTest(&opt)) diff --git a/test/e2e/scenarios/perf/publish-perf-results.go b/test/e2e/scenarios/perf/publish-perf-results.go index 4f1dd05344..36f56dfdf5 100644 --- a/test/e2e/scenarios/perf/publish-perf-results.go +++ b/test/e2e/scenarios/perf/publish-perf-results.go @@ -14,6 +14,7 @@ import ( ) type PublishPerfResults struct { + RetinaMode string ResultsFile string } @@ -59,6 +60,7 @@ func (v *PublishPerfResults) Run() error { telemetryClient, err := telemetry.NewAppInsightsTelemetryClient("retina-perf-test", map[string]string{ "retinaVersion": retinaVersion, + "retinaMode": v.RetinaMode, }) if err != nil { return errors.Wrap(err, "failed to create telemetry client")