diff --git a/cmd/container.go b/cmd/container.go new file mode 100644 index 00000000..bf977b0f --- /dev/null +++ b/cmd/container.go @@ -0,0 +1,70 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + + "github.com/zapier/kubechecks/pkg/app_watcher" + "github.com/zapier/kubechecks/pkg/appdir" + "github.com/zapier/kubechecks/pkg/argo_client" + "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/container" + "github.com/zapier/kubechecks/pkg/vcs/github_client" + "github.com/zapier/kubechecks/pkg/vcs/gitlab_client" +) + +func newContainer(ctx context.Context, cfg config.ServerConfig) (container.Container, error) { + var err error + + var ctr = container.Container{ + Config: cfg, + } + + switch cfg.VcsType { + case "gitlab": + ctr.VcsClient, err = gitlab_client.CreateGitlabClient(cfg) + case "github": + ctr.VcsClient, err = github_client.CreateGithubClient(cfg) + default: + err = fmt.Errorf("unknown vcs-type: %q", cfg.VcsType) + } + if err != nil { + return ctr, errors.Wrap(err, "failed to create vcs client") + } + + if ctr.ArgoClient, err = argo_client.NewArgoClient(cfg); err != nil { + return ctr, errors.Wrap(err, "failed to create argo client") + } + + vcsToArgoMap := appdir.NewVcsToArgoMap() + ctr.VcsToArgoMap = vcsToArgoMap + + if cfg.MonitorAllApplications { + if err = buildAppsMap(ctx, ctr.ArgoClient, ctr.VcsToArgoMap); err != nil { + return ctr, errors.Wrap(err, "failed to build apps map") + } + + ctr.ApplicationWatcher, err = app_watcher.NewApplicationWatcher(vcsToArgoMap) + if err != nil { + return ctr, errors.Wrap(err, "failed to create watch applications") + } + + go ctr.ApplicationWatcher.Run(ctx, 1) + } + + return ctr, nil +} + +func buildAppsMap(ctx context.Context, argoClient *argo_client.ArgoClient, result container.VcsToArgoMap) error { + apps, err := argoClient.GetApplications(ctx) + if err != nil { + return errors.Wrap(err, "failed to list applications") + } + for _, app := range apps.Items { + result.AddApp(&app) + } + + return nil +} diff --git a/cmd/controller_cmd.go b/cmd/controller_cmd.go index 03f3899f..35e46582 100644 --- a/cmd/controller_cmd.go +++ b/cmd/controller_cmd.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "fmt" "os" "os/signal" "syscall" @@ -15,9 +14,11 @@ import ( "github.com/zapier/kubechecks/pkg" "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/container" "github.com/zapier/kubechecks/pkg/events" - "github.com/zapier/kubechecks/pkg/repo" "github.com/zapier/kubechecks/pkg/server" + "github.com/zapier/kubechecks/pkg/vcs" + "github.com/zapier/kubechecks/telemetry" ) // ControllerCmd represents the run command @@ -26,58 +27,89 @@ var ControllerCmd = &cobra.Command{ Short: "Start the VCS Webhook handler.", Long: ``, Run: func(cmd *cobra.Command, args []string) { - clientType := viper.GetString("vcs-type") - client, err := createVCSClient(clientType) + ctx := context.Background() + + log.Info(). + Str("git-tag", pkg.GitTag). + Str("git-commit", pkg.GitCommit). + Msg("Starting KubeChecks") + + log.Info().Msg("parsing configuration") + cfg, err := config.New() if err != nil { - log.Fatal().Err(err).Msg("failed to create vcs client") + log.Fatal().Err(err).Msg("failed to parse configuration") } - cfg := config.ServerConfig{ - UrlPrefix: viper.GetString("webhook-url-prefix"), - WebhookSecret: viper.GetString("webhook-secret"), - VcsClient: client, + ctr, err := newContainer(ctx, cfg) + if err != nil { + log.Fatal().Err(err).Msg("failed to create container") } - log.Info().Msg("Initializing git settings") - if err := repo.InitializeGitSettings(cfg.VcsClient.Username(), cfg.VcsClient.Email()); err != nil { - log.Fatal().Err(err).Msg("failed to initialize git settings") + t, err := initTelemetry(ctx, cfg) + if err != nil { + log.Panic().Err(err).Msg("Failed to initialize telemetry") } + defer t.Shutdown() - fmt.Println("Starting KubeChecks:", pkg.GitTag, pkg.GitCommit) - ctx := context.Background() - server := server.NewServer(ctx, &cfg) - - go server.Start(ctx) - - // graceful termination handler. - // when we receive a SIGTERM from kubernetes, check for in-flight requests before exiting. - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM) - done := make(chan bool, 1) - - go func() { - sig := <-sigs - log.Debug().Str("signal", sig.String()).Msg("received signal") - done <- true - }() - - <-done - log.Info().Msg("shutting down...") - for events.GetInFlight() > 0 { - log.Info().Int("count", events.GetInFlight()).Msg("waiting for in-flight requests to complete") - time.Sleep(time.Second * 3) + log.Info().Msg("initializing git settings") + if err = initializeGit(ctr); err != nil { + log.Fatal().Err(err).Msg("failed to initialize git settings") } - log.Info().Msg("good bye.") - }, - PreRunE: func(cmd *cobra.Command, args []string) error { - log.Info().Msg("Server Configuration: ") - log.Info().Msgf("Webhook URL Base: %s", viper.GetString("webhook-url-base")) - log.Info().Msgf("Webhook URL Prefix: %s", viper.GetString("webhook-url-prefix")) - log.Info().Msgf("VCS Type: %s", viper.GetString("vcs-type")) - return nil + + log.Info().Msgf("starting web server") + startWebserver(ctx, ctr) + + log.Info().Msgf("listening for requests") + waitForShutdown() + + log.Info().Msg("shutting down gracefully") + waitForPendingRequest() }, } +func initTelemetry(ctx context.Context, cfg config.ServerConfig) (*telemetry.OperatorTelemetry, error) { + return telemetry.Init( + ctx, "kubechecks", pkg.GitTag, pkg.GitCommit, + cfg.EnableOtel, cfg.OtelCollectorHost, cfg.OtelCollectorPort, + ) +} + +func startWebserver(ctx context.Context, ctr container.Container) { + srv := server.NewServer(ctr) + go srv.Start(ctx) +} + +func initializeGit(ctr container.Container) error { + if err := vcs.InitializeGitSettings(ctr.Config, ctr.VcsClient); err != nil { + return err + } + + return nil +} + +func waitForPendingRequest() { + for events.GetInFlight() > 0 { + log.Info().Int("count", events.GetInFlight()).Msg("waiting for in-flight requests to complete") + time.Sleep(time.Second * 3) + } +} + +func waitForShutdown() { + // graceful termination handler. + // when we receive a SIGTERM from kubernetes, check for in-flight requests before exiting. + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + done := make(chan bool, 1) + + go func() { + sig := <-sigs + log.Debug().Str("signal", sig.String()).Msg("received signal") + done <- true + }() + + <-done +} + func panicIfError(err error) { if err != nil { panic(err) diff --git a/cmd/process.go b/cmd/process.go index bb9336c6..ad7d6511 100644 --- a/cmd/process.go +++ b/cmd/process.go @@ -1,11 +1,8 @@ package cmd import ( - "context" - "github.com/rs/zerolog/log" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/zapier/kubechecks/pkg/config" "github.com/zapier/kubechecks/pkg/server" @@ -16,34 +13,25 @@ var processCmd = &cobra.Command{ Short: "Process a pull request", Long: "", Run: func(cmd *cobra.Command, args []string) { - ctx := context.TODO() - - log.Info().Msg("building apps map from argocd") - result, err := config.BuildAppsMap(ctx) - if err != nil { - log.Fatal().Err(err).Msg("failed to build apps map") - } - - clientType := viper.GetString("vcs-type") - client, err := createVCSClient(clientType) - if err != nil { - log.Fatal().Err(err).Msg("failed to create vcs client") - } + ctx := cmd.Context() cfg := config.ServerConfig{ UrlPrefix: "--unused--", WebhookSecret: "--unused--", - VcsToArgoMap: result, - VcsClient: client, } - repo, err := client.LoadHook(ctx, args[0]) + ctr, err := newContainer(ctx, cfg) + if err != nil { + log.Fatal().Err(err).Msg("failed to create container") + } + + repo, err := ctr.VcsClient.LoadHook(ctx, args[0]) if err != nil { log.Fatal().Err(err).Msg("failed to load hook") return } - server.ProcessCheckEvent(ctx, repo, &cfg) + server.ProcessCheckEvent(ctx, repo, cfg, ctr) }, } diff --git a/cmd/root.go b/cmd/root.go index eca9bb6f..5ed55abf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "os" "strings" @@ -10,9 +9,6 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - - "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/telemetry" ) // RootCmd represents the base command when called without any subcommands @@ -26,14 +22,6 @@ var RootCmd = &cobra.Command{ // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { - ctx := context.Background() - t, err := initTelemetry(ctx) - if err != nil { - log.Panic().Err(err).Msg("Failed to initialize telemetry") - } - - defer t.Shutdown() - cobra.CheckErr(RootCmd.Execute()) } @@ -42,7 +30,6 @@ const envPrefix = "kubechecks" var envKeyReplacer = strings.NewReplacer("-", "_") func init() { - // allows environment variables to use _ instead of - viper.SetEnvKeyReplacer(envKeyReplacer) // sync-provider becomes SYNC_PROVIDER viper.SetEnvPrefix(envPrefix) // port becomes KUBECHECKS_PORT @@ -51,13 +38,13 @@ func init() { flags := RootCmd.PersistentFlags() stringFlag(flags, "log-level", "Set the log output level.", newStringOpts(). - withChoices( - zerolog.LevelErrorValue, - zerolog.LevelWarnValue, - zerolog.LevelInfoValue, - zerolog.LevelDebugValue, - zerolog.LevelTraceValue, - ). + withChoices( + zerolog.LevelErrorValue, + zerolog.LevelWarnValue, + zerolog.LevelInfoValue, + zerolog.LevelDebugValue, + zerolog.LevelTraceValue, + ). withDefault("info"). withShortHand("l"), ) @@ -94,13 +81,6 @@ func init() { setupLogOutput() } -func initTelemetry(ctx context.Context) (*telemetry.OperatorTelemetry, error) { - enableOtel := viper.GetBool("otel-enabled") - otelHost := viper.GetString("otel-collector-host") - otelPort := viper.GetString("otel-collector-port") - return telemetry.Init(ctx, "kubechecks", pkg.GitTag, pkg.GitCommit, enableOtel, otelHost, otelPort) -} - func setupLogOutput() { output := zerolog.ConsoleWriter{Out: os.Stdout} log.Logger = log.Output(output) diff --git a/cmd/vcs.go b/cmd/vcs.go deleted file mode 100644 index 5cad3db7..00000000 --- a/cmd/vcs.go +++ /dev/null @@ -1,20 +0,0 @@ -package cmd - -import ( - "fmt" - - "github.com/zapier/kubechecks/pkg/vcs" - "github.com/zapier/kubechecks/pkg/vcs/github_client" - "github.com/zapier/kubechecks/pkg/vcs/gitlab_client" -) - -func createVCSClient(clientType string) (vcs.Client, error) { - switch clientType { - case "gitlab": - return gitlab_client.CreateGitlabClient() - case "github": - return github_client.CreateGithubClient() - default: - return nil, fmt.Errorf("unknown vcs type: %s", clientType) - } -} diff --git a/cmd/version.go b/cmd/version.go index c84c98ea..448e8fb2 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -13,7 +13,7 @@ var versionCmd = &cobra.Command{ Short: "List version information", Long: ``, Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Arrgh\nVersion:%s\nSHA%s", pkg.GitTag, pkg.GitCommit) + fmt.Printf("kubechecks\nVersion:%s\nSHA%s\n", pkg.GitTag, pkg.GitCommit) }, } diff --git a/go.mod b/go.mod index b682492a..b4aa6357 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/rs/zerolog v1.31.0 github.com/sashabaranov/go-openai v1.19.3 github.com/shurcooL/githubv4 v0.0.0-20231126234147-1cffa1f02456 - github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 @@ -210,6 +209,7 @@ require ( github.com/sergi/go-diff v1.3.1 // indirect github.com/shteou/go-ignore v0.3.1 // indirect github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.1 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spdx/tools-golang v0.5.3 // indirect diff --git a/pkg/affected_apps/argocd_matcher.go b/pkg/affected_apps/argocd_matcher.go index 688b584c..3a765002 100644 --- a/pkg/affected_apps/argocd_matcher.go +++ b/pkg/affected_apps/argocd_matcher.go @@ -6,19 +6,20 @@ import ( "github.com/rs/zerolog/log" - "github.com/zapier/kubechecks/pkg/config" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/appdir" + "github.com/zapier/kubechecks/pkg/container" + "github.com/zapier/kubechecks/pkg/vcs" ) type ArgocdMatcher struct { - appsDirectory *config.AppDirectory + appsDirectory *appdir.AppDirectory } -func NewArgocdMatcher(vcsToArgoMap config.VcsToArgoMap, repo *repo.Repo, repoPath string) (*ArgocdMatcher, error) { +func NewArgocdMatcher(vcsToArgoMap container.VcsToArgoMap, repo *vcs.Repo, repoPath string) (*ArgocdMatcher, error) { repoApps := getArgocdApps(vcsToArgoMap, repo) kustomizeAppFiles := getKustomizeApps(vcsToArgoMap, repo, repoPath) - appDirectory := config.NewAppDirectory(). + appDirectory := appdir.NewAppDirectory(). Union(repoApps). Union(kustomizeAppFiles) @@ -27,7 +28,7 @@ func NewArgocdMatcher(vcsToArgoMap config.VcsToArgoMap, repo *repo.Repo, repoPat }, nil } -func logCounts(repoApps *config.AppDirectory) { +func logCounts(repoApps *appdir.AppDirectory) { if repoApps == nil { log.Debug().Msg("found no apps") } else { @@ -35,17 +36,17 @@ func logCounts(repoApps *config.AppDirectory) { } } -func getKustomizeApps(vcsToArgoMap config.VcsToArgoMap, repo *repo.Repo, repoPath string) *config.AppDirectory { +func getKustomizeApps(vcsToArgoMap container.VcsToArgoMap, repo *vcs.Repo, repoPath string) *appdir.AppDirectory { log.Debug().Msgf("creating fs for %s", repoPath) fs := os.DirFS(repoPath) log.Debug().Msg("following kustomize apps") - kustomizeAppFiles := vcsToArgoMap.WalkKustomizeApps(repo, fs) + kustomizeAppFiles := vcsToArgoMap.WalkKustomizeApps(repo.CloneURL, fs) logCounts(kustomizeAppFiles) return kustomizeAppFiles } -func getArgocdApps(vcsToArgoMap config.VcsToArgoMap, repo *repo.Repo) *config.AppDirectory { +func getArgocdApps(vcsToArgoMap container.VcsToArgoMap, repo *vcs.Repo) *appdir.AppDirectory { log.Debug().Msgf("looking for %s repos", repo.CloneURL) repoApps := vcsToArgoMap.GetAppsInRepo(repo.CloneURL) diff --git a/pkg/affected_apps/argocd_matcher_test.go b/pkg/affected_apps/argocd_matcher_test.go index 759ff0ef..97a64d5b 100644 --- a/pkg/affected_apps/argocd_matcher_test.go +++ b/pkg/affected_apps/argocd_matcher_test.go @@ -6,17 +6,17 @@ import ( "github.com/stretchr/testify/require" - "github.com/zapier/kubechecks/pkg/config" - repo2 "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/appdir" + "github.com/zapier/kubechecks/pkg/vcs" ) func TestCreateNewMatcherWithNilVcsMap(t *testing.T) { // setup var ( - repo repo2.Repo + repo vcs.Repo path string - vcsMap = config.NewVcsToArgoMap() + vcsMap = appdir.NewVcsToArgoMap() ) // run test diff --git a/pkg/affected_apps/config_matcher.go b/pkg/affected_apps/config_matcher.go index b0f0091e..4ad331e3 100644 --- a/pkg/affected_apps/config_matcher.go +++ b/pkg/affected_apps/config_matcher.go @@ -10,18 +10,22 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/zapier/kubechecks/pkg/argo_client" + "github.com/zapier/kubechecks/pkg/container" "github.com/zapier/kubechecks/pkg/repo_config" ) type ConfigMatcher struct { cfg *repo_config.Config - argoClient *argo_client.ArgoClient + argoClient argoClient } -func NewConfigMatcher(cfg *repo_config.Config) *ConfigMatcher { - argoClient := argo_client.GetArgoClient() - return &ConfigMatcher{cfg: cfg, argoClient: argoClient} +type argoClient interface { + GetApplications(ctx context.Context) (*v1alpha1.ApplicationList, error) + GetApplicationsByAppset(ctx context.Context, appsetName string) (*v1alpha1.ApplicationList, error) +} + +func NewConfigMatcher(cfg *repo_config.Config, ctr container.Container) *ConfigMatcher { + return &ConfigMatcher{cfg: cfg, argoClient: ctr.ArgoClient} } func (b *ConfigMatcher) AffectedApps(ctx context.Context, changeList []string, targetBranch string) (AffectedItems, error) { @@ -131,7 +135,8 @@ func (b *ConfigMatcher) appsFromApplicationSetForDir(ctx context.Context, dir st } } - apps := []*repo_config.ArgoCdApplicationConfig{} + var apps []*repo_config.ArgoCdApplicationConfig + for _, appset := range appsets { appList, err := b.argoClient.GetApplicationsByAppset(ctx, appset.Name) if err != nil { @@ -149,6 +154,7 @@ func (b *ConfigMatcher) appsFromApplicationSetForDir(ctx context.Context, dir st }) } } + return appsets, apps, nil } diff --git a/pkg/affected_apps/config_matcher_test.go b/pkg/affected_apps/config_matcher_test.go index a6f7480b..101c531c 100644 --- a/pkg/affected_apps/config_matcher_test.go +++ b/pkg/affected_apps/config_matcher_test.go @@ -2,52 +2,14 @@ package affected_apps import ( "context" - "io" "testing" - "github.com/argoproj/argo-cd/v2/pkg/apiclient" - "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/stretchr/testify/assert" - "google.golang.org/grpc" - "github.com/zapier/kubechecks/pkg/argo_client" "github.com/zapier/kubechecks/pkg/repo_config" ) -type MockArgoApplicationServiceClient struct { - application.ApplicationServiceClient -} - -type MockCloser struct { - CloseFunc func() error -} - -func (m MockCloser) Close() error { - if m.CloseFunc != nil { - return m.CloseFunc() - } - return nil -} - -type MockArgoClient struct { - apiclient.Client -} - -func (m MockArgoApplicationServiceClient) List(_ context.Context, _ *application.ApplicationQuery, _ ...grpc.CallOption) (*v1alpha1.ApplicationList, error) { - return &v1alpha1.ApplicationList{}, nil -} - -func (m MockArgoClient) NewApplicationClient() (io.Closer, application.ApplicationServiceClient, error) { - return MockCloser{}, MockArgoApplicationServiceClient{}, nil - -} - -func NewMockArgoClient() *argo_client.ArgoClient { - apiClient := MockArgoClient{} - return argo_client.NewArgoClient(apiClient) -} - func Test_dirMatchForApp(t *testing.T) { type args struct { changeDir string @@ -132,9 +94,12 @@ func TestConfigMatcher_triggeredApps(t *testing.T) { }, } - mockArgoClient := NewMockArgoClient() for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + mockArgoClient := newMockArgoClient() + c := testLoadConfig(t, tt.configDir) b := &ConfigMatcher{ cfg: c, @@ -147,6 +112,23 @@ func TestConfigMatcher_triggeredApps(t *testing.T) { } } +func newMockArgoClient() argoClient { + return new(mockArgoClient) +} + +type mockArgoClient struct { +} + +func (m mockArgoClient) GetApplications(ctx context.Context) (*v1alpha1.ApplicationList, error) { + return new(v1alpha1.ApplicationList), nil +} + +func (m mockArgoClient) GetApplicationsByAppset(ctx context.Context, appsetName string) (*v1alpha1.ApplicationList, error) { + return new(v1alpha1.ApplicationList), nil +} + +var _ argoClient = new(mockArgoClient) + func testLoadConfig(t *testing.T, configDir string) *repo_config.Config { cfg, err := repo_config.LoadRepoConfig(configDir) if err != nil { diff --git a/pkg/aisummary/openai_client.go b/pkg/aisummary/openai_client.go index 74c883c2..b4f5d64a 100644 --- a/pkg/aisummary/openai_client.go +++ b/pkg/aisummary/openai_client.go @@ -10,7 +10,6 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/rs/zerolog/log" "github.com/sashabaranov/go-openai" - "github.com/spf13/viper" "go.opentelemetry.io/otel" ) @@ -22,9 +21,8 @@ type OpenAiClient struct { var openAiClient *OpenAiClient var once sync.Once -func GetOpenAiClient() *OpenAiClient { +func GetOpenAiClient(apiToken string) *OpenAiClient { once.Do(func() { - apiToken := viper.GetString("openai-api-token") if apiToken != "" { log.Info().Msg("enabling OpenAI client") client := openai.NewClient(apiToken) diff --git a/pkg/app_watcher/app_watcher.go b/pkg/app_watcher/app_watcher.go index c2afaf15..997a55d8 100644 --- a/pkg/app_watcher/app_watcher.go +++ b/pkg/app_watcher/app_watcher.go @@ -3,32 +3,33 @@ package app_watcher import ( "context" "reflect" + "strings" + "time" appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned" "github.com/rs/zerolog/log" - "github.com/zapier/kubechecks/pkg/config" "k8s.io/client-go/tools/clientcmd" - "strings" - "time" - appv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" informers "github.com/argoproj/argo-cd/v2/pkg/client/informers/externalversions/application/v1alpha1" applisters "github.com/argoproj/argo-cd/v2/pkg/client/listers/application/v1alpha1" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" + + "github.com/zapier/kubechecks/pkg/appdir" ) // ApplicationWatcher is the controller that watches ArgoCD Application resources via the Kubernetes API type ApplicationWatcher struct { - cfg *config.ServerConfig applicationClientset appclientset.Interface appInformer cache.SharedIndexInformer appLister applisters.ApplicationLister + + vcsToArgoMap appdir.VcsToArgoMap } // NewApplicationWatcher creates new instance of ApplicationWatcher. -func NewApplicationWatcher(cfg *config.ServerConfig) (*ApplicationWatcher, error) { +func NewApplicationWatcher(vcsToArgoMap appdir.VcsToArgoMap) (*ApplicationWatcher, error) { // this assumes kubechecks is running inside the cluster kubeCfg, err := clientcmd.BuildConfigFromFlags("", "") if err != nil { @@ -38,8 +39,8 @@ func NewApplicationWatcher(cfg *config.ServerConfig) (*ApplicationWatcher, error appClient := appclientset.NewForConfigOrDie(kubeCfg) ctrl := ApplicationWatcher{ - cfg: cfg, applicationClientset: appClient, + vcsToArgoMap: vcsToArgoMap, } appInformer, appLister := ctrl.newApplicationInformerAndLister(time.Second * 30) @@ -78,7 +79,7 @@ func (ctrl *ApplicationWatcher) onApplicationAdded(obj interface{}) { log.Error().Err(err).Msg("appwatcher: could not get key for added application") } log.Info().Str("key", key).Msg("appwatcher: onApplicationAdded") - ctrl.cfg.VcsToArgoMap.AddApp(app) + ctrl.vcsToArgoMap.AddApp(app) } func (ctrl *ApplicationWatcher) onApplicationUpdated(old, new interface{}) { @@ -96,7 +97,7 @@ func (ctrl *ApplicationWatcher) onApplicationUpdated(old, new interface{}) { // We want to update when any of Source or Sources parameters has changed if !reflect.DeepEqual(oldApp.Spec.Source, newApp.Spec.Source) || !reflect.DeepEqual(oldApp.Spec.Sources, newApp.Spec.Sources) { log.Info().Str("key", key).Msg("appwatcher: onApplicationUpdated") - ctrl.cfg.VcsToArgoMap.UpdateApp(old.(*appv1alpha1.Application), new.(*appv1alpha1.Application)) + ctrl.vcsToArgoMap.UpdateApp(old.(*appv1alpha1.Application), new.(*appv1alpha1.Application)) } } @@ -112,7 +113,7 @@ func (ctrl *ApplicationWatcher) onApplicationDeleted(obj interface{}) { } log.Info().Str("key", key).Msg("appwatcher: onApplicationDeleted") - ctrl.cfg.VcsToArgoMap.DeleteApp(app) + ctrl.vcsToArgoMap.DeleteApp(app) } /* @@ -126,13 +127,15 @@ func (ctrl *ApplicationWatcher) newApplicationInformerAndLister(refreshTimeout t ) lister := applisters.NewApplicationLister(informer.GetIndexer()) - informer.AddEventHandler( + if _, err := informer.AddEventHandler( cache.ResourceEventHandlerFuncs{ AddFunc: ctrl.onApplicationAdded, UpdateFunc: ctrl.onApplicationUpdated, DeleteFunc: ctrl.onApplicationDeleted, }, - ) + ); err != nil { + log.Error().Err(err).Msg("failed to add event handlers") + } return informer, lister } diff --git a/pkg/app_watcher/app_watcher_test.go b/pkg/app_watcher/app_watcher_test.go index e47dc85c..b46c8025 100644 --- a/pkg/app_watcher/app_watcher_test.go +++ b/pkg/app_watcher/app_watcher_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/appdir" ) func initTestObjects() *ApplicationWatcher { - // Setup the fake Application client set and informer. + // set up the fake Application client set and informer. testApp1 := &v1alpha1.Application{ ObjectMeta: metav1.ObjectMeta{Name: "test-app-1", Namespace: "default"}, Spec: v1alpha1.ApplicationSpec{ @@ -31,9 +31,7 @@ func initTestObjects() *ApplicationWatcher { clientset := appclientsetfake.NewSimpleClientset(testApp1, testApp2) ctrl := &ApplicationWatcher{ applicationClientset: clientset, - cfg: &config.ServerConfig{ - VcsToArgoMap: config.NewVcsToArgoMap(), - }, + vcsToArgoMap: appdir.NewVcsToArgoMap(), } appInformer, appLister := ctrl.newApplicationInformerAndLister(time.Second * 1) @@ -44,18 +42,18 @@ func initTestObjects() *ApplicationWatcher { } func TestApplicationAdded(t *testing.T) { - ctrl := initTestObjects() + appWatcher := initTestObjects() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go ctrl.Run(ctx, 1) + go appWatcher.Run(ctx, 1) time.Sleep(time.Second * 1) - assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 2) + assert.Equal(t, len(appWatcher.vcsToArgoMap.GetMap()), 2) - _, err := ctrl.applicationClientset.ArgoprojV1alpha1().Applications("default").Create(ctx, &v1alpha1.Application{ + _, err := appWatcher.applicationClientset.ArgoprojV1alpha1().Applications("default").Create(ctx, &v1alpha1.Application{ ObjectMeta: metav1.ObjectMeta{Name: "test-app-3", Namespace: "default"}, Spec: v1alpha1.ApplicationSpec{ Source: &v1alpha1.ApplicationSource{RepoURL: "https://gitlab.com/test/repo-3.git"}, @@ -66,7 +64,7 @@ func TestApplicationAdded(t *testing.T) { } time.Sleep(time.Second * 1) - assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 3) + assert.Equal(t, len(appWatcher.vcsToArgoMap.GetMap()), 3) } func TestApplicationUpdated(t *testing.T) { @@ -79,10 +77,10 @@ func TestApplicationUpdated(t *testing.T) { time.Sleep(time.Second * 1) - assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 2) + assert.Equal(t, len(ctrl.vcsToArgoMap.GetMap()), 2) - oldAppDirectory := ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") - newAppDirectory := ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo-3.git") + oldAppDirectory := ctrl.vcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + newAppDirectory := ctrl.vcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo-3.git") assert.Equal(t, oldAppDirectory.Count(), 1) assert.Equal(t, newAppDirectory.Count(), 0) // @@ -96,8 +94,8 @@ func TestApplicationUpdated(t *testing.T) { t.Error(err) } time.Sleep(time.Second * 1) - oldAppDirectory = ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") - newAppDirectory = ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo-3.git") + oldAppDirectory = ctrl.vcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + newAppDirectory = ctrl.vcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo-3.git") assert.Equal(t, oldAppDirectory.Count(), 0) assert.Equal(t, newAppDirectory.Count(), 1) } @@ -112,9 +110,9 @@ func TestApplicationDeleted(t *testing.T) { time.Sleep(time.Second * 1) - assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 2) + assert.Equal(t, len(ctrl.vcsToArgoMap.GetMap()), 2) - appDirectory := ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + appDirectory := ctrl.vcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") assert.Equal(t, appDirectory.Count(), 1) // err := ctrl.applicationClientset.ArgoprojV1alpha1().Applications("default").Delete(ctx, "test-app-1", metav1.DeleteOptions{}) @@ -123,7 +121,7 @@ func TestApplicationDeleted(t *testing.T) { } time.Sleep(time.Second * 1) - appDirectory = ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + appDirectory = ctrl.vcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") assert.Equal(t, appDirectory.Count(), 0) } diff --git a/pkg/config/app_directory.go b/pkg/appdir/app_directory.go similarity index 99% rename from pkg/config/app_directory.go rename to pkg/appdir/app_directory.go index 2d123d34..cc5f6512 100644 --- a/pkg/config/app_directory.go +++ b/pkg/appdir/app_directory.go @@ -1,4 +1,4 @@ -package config +package appdir import ( "path/filepath" diff --git a/pkg/config/app_directory_test.go b/pkg/appdir/app_directory_test.go similarity index 99% rename from pkg/config/app_directory_test.go rename to pkg/appdir/app_directory_test.go index a7243f6c..b67f2aa8 100644 --- a/pkg/config/app_directory_test.go +++ b/pkg/appdir/app_directory_test.go @@ -1,4 +1,4 @@ -package config +package appdir import ( "fmt" diff --git a/pkg/appdir/repoUrl.go b/pkg/appdir/repoUrl.go new file mode 100644 index 00000000..68b8182c --- /dev/null +++ b/pkg/appdir/repoUrl.go @@ -0,0 +1,40 @@ +package appdir + +import ( + "fmt" + "net/url" + "strings" + + giturls "github.com/whilp/git-urls" +) + +type RepoURL struct { + Host, Path string +} + +func (r RepoURL) CloneURL() string { + return fmt.Sprintf("git@%s:%s", r.Host, r.Path) +} + +func buildNormalizedRepoUrl(host, path string) RepoURL { + path = strings.TrimPrefix(path, "/") + path = strings.TrimSuffix(path, ".git") + return RepoURL{host, path} +} + +func NormalizeRepoUrl(s string) (RepoURL, error) { + var parser func(string) (*url.URL, error) + + if strings.HasPrefix(s, "http") { + parser = url.Parse + } else { + parser = giturls.Parse + } + + r, err := parser(s) + if err != nil { + return RepoURL{}, err + } + + return buildNormalizedRepoUrl(r.Host, r.Path), nil +} diff --git a/pkg/appdir/repoUrl_test.go b/pkg/appdir/repoUrl_test.go new file mode 100644 index 00000000..ece78042 --- /dev/null +++ b/pkg/appdir/repoUrl_test.go @@ -0,0 +1,66 @@ +package appdir + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" +) + +func TestNormalizeStrings(t *testing.T) { + testCases := []struct { + input string + expected RepoURL + }{ + { + input: "git@github.com:one/two", + expected: RepoURL{"github.com", "one/two"}, + }, + { + input: "https://github.com/one/two", + expected: RepoURL{"github.com", "one/two"}, + }, + { + input: "git@gitlab.com:djeebus/helm-test.git", + expected: RepoURL{"gitlab.com", "djeebus/helm-test"}, + }, + { + input: "https://gitlab.com/djeebus/helm-test.git", + expected: RepoURL{"gitlab.com", "djeebus/helm-test"}, + }, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("case %s", tc.input), func(t *testing.T) { + actual, err := NormalizeRepoUrl(tc.input) + require.NoError(t, err) + assert.Equal(t, tc.expected, actual) + }) + } +} + +// TestBuildNormalizedRepoURL tests the buildNormalizedRepoUrl function. +func TestBuildNormalizedRepoURL(t *testing.T) { + tests := []struct { + host string + path string + expected RepoURL + }{ + { + host: "example.com", + path: "/repository.git", + expected: RepoURL{ + Host: "example.com", + Path: "repository", + }, + }, + // ... additional test cases + } + + for _, tc := range tests { + result := buildNormalizedRepoUrl(tc.host, tc.path) + assert.Equal(t, tc.expected, result) + } +} diff --git a/pkg/appdir/vcstoargomap.go b/pkg/appdir/vcstoargomap.go new file mode 100644 index 00000000..4fd02046 --- /dev/null +++ b/pkg/appdir/vcstoargomap.go @@ -0,0 +1,99 @@ +package appdir + +import ( + "io/fs" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/rs/zerolog/log" +) + +type VcsToArgoMap struct { + appDirByRepo map[RepoURL]*AppDirectory +} + +func NewVcsToArgoMap() VcsToArgoMap { + return VcsToArgoMap{ + appDirByRepo: make(map[RepoURL]*AppDirectory), + } +} + +func (v2a VcsToArgoMap) GetMap() map[RepoURL]*AppDirectory { + return v2a.appDirByRepo +} + +func (v2a VcsToArgoMap) GetAppsInRepo(repoCloneUrl string) *AppDirectory { + repoUrl, err := NormalizeRepoUrl(repoCloneUrl) + if err != nil { + log.Warn().Err(err).Msgf("failed to parse %s", repoCloneUrl) + } + + appdir := v2a.appDirByRepo[repoUrl] + if appdir == nil { + appdir = NewAppDirectory() + v2a.appDirByRepo[repoUrl] = appdir + } + + return appdir +} + +func (v2a VcsToArgoMap) WalkKustomizeApps(cloneURL string, fs fs.FS) *AppDirectory { + var ( + err error + + result = NewAppDirectory() + appdir = v2a.GetAppsInRepo(cloneURL) + apps = appdir.GetApps(nil) + ) + + for _, app := range apps { + appPath := app.Spec.GetSource().Path + if err = walkKustomizeFiles(result, fs, app.Name, appPath); err != nil { + log.Error().Err(err).Msgf("failed to parse kustomize.yaml in %s", appPath) + } + } + + return result +} + +func (v2a VcsToArgoMap) AddApp(app *v1alpha1.Application) { + if app.Spec.Source == nil { + log.Warn().Msgf("%s/%s: no source, skipping", app.Namespace, app.Name) + return + } + + appDirectory := v2a.GetAppsInRepo(app.Spec.Source.RepoURL) + appDirectory.ProcessApp(*app) +} + +func (v2a VcsToArgoMap) UpdateApp(old *v1alpha1.Application, new *v1alpha1.Application) { + if new.Spec.Source == nil { + log.Warn().Msgf("%s/%s: no source, skipping", new.Namespace, new.Name) + return + } + + oldAppDirectory := v2a.GetAppsInRepo(old.Spec.Source.RepoURL) + oldAppDirectory.RemoveApp(*old) + + newAppDirectory := v2a.GetAppsInRepo(new.Spec.Source.RepoURL) + newAppDirectory.ProcessApp(*new) +} + +func (v2a VcsToArgoMap) DeleteApp(app *v1alpha1.Application) { + if app.Spec.Source == nil { + log.Warn().Msgf("%s/%s: no source, skipping", app.Namespace, app.Name) + return + } + + oldAppDirectory := v2a.GetAppsInRepo(app.Spec.Source.RepoURL) + oldAppDirectory.RemoveApp(*app) +} + +func (v2a VcsToArgoMap) GetVcsRepos() []string { + var repos []string + + for key := range v2a.appDirByRepo { + repos = append(repos, key.CloneURL()) + } + + return repos +} diff --git a/pkg/config/config_test.go b/pkg/appdir/vcstoargumap_test.go similarity index 64% rename from pkg/config/config_test.go rename to pkg/appdir/vcstoargumap_test.go index 3502a5db..aacb21b0 100644 --- a/pkg/config/config_test.go +++ b/pkg/appdir/vcstoargumap_test.go @@ -1,71 +1,13 @@ -package config +package appdir import ( - "fmt" "testing" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestNormalizeStrings(t *testing.T) { - testCases := []struct { - input string - expected RepoURL - }{ - { - input: "git@github.com:one/two", - expected: RepoURL{"github.com", "one/two"}, - }, - { - input: "https://github.com/one/two", - expected: RepoURL{"github.com", "one/two"}, - }, - { - input: "git@gitlab.com:djeebus/helm-test.git", - expected: RepoURL{"gitlab.com", "djeebus/helm-test"}, - }, - { - input: "https://gitlab.com/djeebus/helm-test.git", - expected: RepoURL{"gitlab.com", "djeebus/helm-test"}, - }, - } - - for _, tc := range testCases { - t.Run(fmt.Sprintf("case %s", tc.input), func(t *testing.T) { - actual, err := NormalizeRepoUrl(tc.input) - require.NoError(t, err) - assert.Equal(t, tc.expected, actual) - }) - } -} - -// TestBuildNormalizedRepoURL tests the buildNormalizedRepoUrl function. -func TestBuildNormalizedRepoURL(t *testing.T) { - tests := []struct { - host string - path string - expected RepoURL - }{ - { - host: "example.com", - path: "/repository.git", - expected: RepoURL{ - Host: "example.com", - Path: "repository", - }, - }, - // ... additional test cases - } - - for _, tc := range tests { - result := buildNormalizedRepoUrl(tc.host, tc.path) - assert.Equal(t, tc.expected, result) - } -} - // TestAddApp tests the AddApp method from the VcsToArgoMap type. func TestAddApp(t *testing.T) { // Setup your mocks and expected calls here. diff --git a/pkg/config/walk_kustomize_files.go b/pkg/appdir/walk_kustomize_files.go similarity index 99% rename from pkg/config/walk_kustomize_files.go rename to pkg/appdir/walk_kustomize_files.go index 7c18257b..0ab819a8 100644 --- a/pkg/config/walk_kustomize_files.go +++ b/pkg/appdir/walk_kustomize_files.go @@ -1,4 +1,4 @@ -package config +package appdir import ( "io" diff --git a/pkg/config/walk_kustomize_files_test.go b/pkg/appdir/walk_kustomize_files_test.go similarity index 99% rename from pkg/config/walk_kustomize_files_test.go rename to pkg/appdir/walk_kustomize_files_test.go index 8c5bb8bb..c5b77fa3 100644 --- a/pkg/config/walk_kustomize_files_test.go +++ b/pkg/appdir/walk_kustomize_files_test.go @@ -1,4 +1,4 @@ -package config +package appdir import ( "testing" diff --git a/pkg/argo_client/client.go b/pkg/argo_client/client.go index 5e69e4c8..fda0df09 100644 --- a/pkg/argo_client/client.go +++ b/pkg/argo_client/client.go @@ -2,54 +2,45 @@ package argo_client import ( "io" - "sync" - - "github.com/argoproj/argo-cd/v2/pkg/apiclient/applicationset" - "github.com/rs/zerolog/log" - "github.com/spf13/viper" "github.com/argoproj/argo-cd/v2/pkg/apiclient" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + "github.com/argoproj/argo-cd/v2/pkg/apiclient/applicationset" "github.com/argoproj/argo-cd/v2/pkg/apiclient/settings" + "github.com/rs/zerolog/log" "github.com/argoproj/argo-cd/v2/pkg/apiclient/cluster" + + "github.com/zapier/kubechecks/pkg/config" ) type ArgoClient struct { client apiclient.Client } -var argoClient *ArgoClient -var once sync.Once +func NewArgoClient(cfg config.ServerConfig) (*ArgoClient, error) { + opts := &apiclient.ClientOptions{ + ServerAddr: cfg.ArgoCDServerAddr, + AuthToken: cfg.ArgoCDToken, + GRPCWebRootPath: cfg.ArgoCDPathPrefix, + Insecure: cfg.ArgoCDInsecure, + } -func GetArgoClient() *ArgoClient { - once.Do(func() { - argoClient = createArgoClient() - }) - return argoClient -} + log.Info(). + Str("server-addr", opts.ServerAddr). + Int("auth-token", len(opts.AuthToken)). + Str("grpc-web-root-path", opts.GRPCWebRootPath). + Bool("insecure", cfg.ArgoCDInsecure). + Msg("ArgoCD client configuration") -func createArgoClient() *ArgoClient { - clientOptions := &apiclient.ClientOptions{ - ServerAddr: viper.GetString("argocd-api-server-addr"), - AuthToken: viper.GetString("argocd-api-token"), - GRPCWebRootPath: viper.GetString("argocd-api-path-prefix"), - Insecure: viper.GetBool("argocd-api-insecure"), - } - argo, err := apiclient.NewClient(clientOptions) + argo, err := apiclient.NewClient(opts) if err != nil { - log.Fatal().Err(err).Msg("could not create ArgoCD API client") + return nil, err } return &ArgoClient{ client: argo, - } -} - -func NewArgoClient(client apiclient.Client) *ArgoClient { - return &ArgoClient{ - client: client, - } + }, nil } // GetApplicationClient has related argocd diff code https://github.com/argoproj/argo-cd/blob/d3ff9757c460ae1a6a11e1231251b5d27aadcdd1/cmd/argocd/commands/app.go#L899 diff --git a/pkg/argo_client/manifests.go b/pkg/argo_client/manifests.go index dbfebccc..be0e9320 100644 --- a/pkg/argo_client/manifests.go +++ b/pkg/argo_client/manifests.go @@ -20,7 +20,7 @@ import ( "github.com/zapier/kubechecks/telemetry" ) -func GetManifestsLocal(ctx context.Context, name string, tempRepoDir string, changedAppFilePath string, app argoappv1.Application) ([]string, error) { +func GetManifestsLocal(ctx context.Context, argoClient *ArgoClient, name string, tempRepoDir string, changedAppFilePath string, app argoappv1.Application) ([]string, error) { var err error ctx, span := otel.Tracer("Kubechecks").Start(ctx, "GetManifestsLocal") @@ -33,7 +33,6 @@ func GetManifestsLocal(ctx context.Context, name string, tempRepoDir string, cha duration := time.Since(start) getManifestsDuration.WithLabelValues(name).Observe(duration.Seconds()) }() - argoClient := GetArgoClient() clusterCloser, clusterClient := argoClient.GetClusterClient() defer clusterCloser.Close() @@ -97,9 +96,9 @@ func GetManifestsLocal(ctx context.Context, name string, tempRepoDir string, cha return res.Manifests, nil } -func FormatManifestsYAML(manifestBytes []string) []string { +func ConvertJsonToYamlManifests(jsonManifests []string) []string { var manifests []string - for _, manifest := range manifestBytes { + for _, manifest := range jsonManifests { ret, err := yaml.JSONToYAML([]byte(manifest)) if err != nil { log.Warn().Err(err).Msg("Failed to format manifest") diff --git a/pkg/config/config.go b/pkg/config/config.go index 9f4d8f4d..d0449773 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,92 +1,88 @@ package config import ( - "fmt" - "net/url" - "strings" - - "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/pkg/errors" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" - giturls "github.com/whilp/git-urls" - - "github.com/zapier/kubechecks/pkg/vcs" + "github.com/spf13/viper" ) -type RepoURL struct { - Host, Path string -} - -func (r RepoURL) CloneURL() string { - return fmt.Sprintf("git@%s:%s", r.Host, r.Path) -} - -func buildNormalizedRepoUrl(host, path string) RepoURL { - path = strings.TrimPrefix(path, "/") - path = strings.TrimSuffix(path, ".git") - return RepoURL{host, path} +type ServerConfig struct { + // argocd + ArgoCDServerAddr string + ArgoCDToken string + ArgoCDPathPrefix string + ArgoCDInsecure bool + + // otel + EnableOtel bool + OtelCollectorHost string + OtelCollectorPort string + + // vcs + VcsBaseUrl string + VcsToken string + VcsType string + + // webhooks + EnsureWebhooks bool + WebhookSecret string + WebhookUrlBase string + + // misc + EnableConfTest bool + FallbackK8sVersion string + LabelFilter string + LogLevel zerolog.Level + MonitorAllApplications bool + OpenAIAPIToken string + PoliciesLocation []string + SchemasLocations []string + ShowDebugInfo bool + TidyOutdatedCommentsMode string + UrlPrefix string } -func NormalizeRepoUrl(s string) (RepoURL, error) { - var parser func(string) (*url.URL, error) - - if strings.HasPrefix(s, "http") { - parser = url.Parse - } else { - parser = giturls.Parse - } - - r, err := parser(s) +func New() (ServerConfig, error) { + logLevelString := viper.GetString("log-level") + logLevel, err := zerolog.ParseLevel(logLevelString) if err != nil { - return RepoURL{}, err - } - - return buildNormalizedRepoUrl(r.Host, r.Path), nil -} - -func (v2a *VcsToArgoMap) AddApp(app *v1alpha1.Application) { - if app.Spec.Source == nil { - log.Warn().Msgf("%s/%s: no source, skipping", app.Namespace, app.Name) - return + return ServerConfig{}, errors.Wrap(err, "failed to parse log level") } - appDirectory := v2a.GetAppsInRepo(app.Spec.Source.RepoURL) - appDirectory.ProcessApp(*app) -} - -func (v2a *VcsToArgoMap) UpdateApp(old *v1alpha1.Application, new *v1alpha1.Application) { - if new.Spec.Source == nil { - log.Warn().Msgf("%s/%s: no source, skipping", new.Namespace, new.Name) - return + cfg := ServerConfig{ + ArgoCDInsecure: viper.GetBool("argocd-api-insecure"), + ArgoCDToken: viper.GetString("argocd-api-token"), + ArgoCDPathPrefix: viper.GetString("argocd-api-path-prefix"), + ArgoCDServerAddr: viper.GetString("argocd-api-server-addr"), + + EnableConfTest: viper.GetBool("enable-conftest"), + EnableOtel: viper.GetBool("otel-enabled"), + EnsureWebhooks: viper.GetBool("ensure-webhooks"), + FallbackK8sVersion: viper.GetString("fallback-k8s-version"), + LabelFilter: viper.GetString("label-filter"), + LogLevel: logLevel, + MonitorAllApplications: viper.GetBool("monitor-all-applications"), + OpenAIAPIToken: viper.GetString("openai-api-token"), + OtelCollectorHost: viper.GetString("otel-collector-host"), + OtelCollectorPort: viper.GetString("otel-collector-port"), + PoliciesLocation: viper.GetStringSlice("policies-location"), + SchemasLocations: viper.GetStringSlice("schemas-location"), + ShowDebugInfo: viper.GetBool("show-debug-info"), + TidyOutdatedCommentsMode: viper.GetString("tidy-outdated-comments-mode"), + UrlPrefix: viper.GetString("webhook-url-prefix"), + WebhookSecret: viper.GetString("webhook-secret"), + WebhookUrlBase: viper.GetString("webhook-url-base"), + + VcsBaseUrl: viper.GetString("vcs-base-url"), + VcsToken: viper.GetString("vcs-token"), + VcsType: viper.GetString("vcs-type"), } - oldAppDirectory := v2a.GetAppsInRepo(old.Spec.Source.RepoURL) - oldAppDirectory.RemoveApp(*old) - - newAppDirectory := v2a.GetAppsInRepo(new.Spec.Source.RepoURL) - newAppDirectory.ProcessApp(*new) -} - -func (v2a *VcsToArgoMap) DeleteApp(app *v1alpha1.Application) { - if app.Spec.Source == nil { - log.Warn().Msgf("%s/%s: no source, skipping", app.Namespace, app.Name) - return - } + log.Info().Msg("Server Configuration: ") + log.Info().Msgf("Webhook URL Base: %s", cfg.WebhookUrlBase) + log.Info().Msgf("Webhook URL Prefix: %s", cfg.UrlPrefix) + log.Info().Msgf("VCS Type: %s", cfg.VcsType) - oldAppDirectory := v2a.GetAppsInRepo(app.Spec.Source.RepoURL) - oldAppDirectory.RemoveApp(*app) -} - -type ServerConfig struct { - UrlPrefix string - WebhookSecret string - VcsToArgoMap VcsToArgoMap - VcsClient vcs.Client -} - -func (cfg *ServerConfig) GetVcsRepos() []string { - var repos []string - for key := range cfg.VcsToArgoMap.appDirByRepo { - repos = append(repos, key.CloneURL()) - } - return repos + return cfg, nil } diff --git a/pkg/config/vcstoargomap.go b/pkg/config/vcstoargomap.go deleted file mode 100644 index 31f61390..00000000 --- a/pkg/config/vcstoargomap.go +++ /dev/null @@ -1,75 +0,0 @@ -package config - -import ( - "context" - "io/fs" - - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - - "github.com/zapier/kubechecks/pkg/argo_client" - "github.com/zapier/kubechecks/pkg/repo" -) - -type VcsToArgoMap struct { - appDirByRepo map[RepoURL]*AppDirectory -} - -func NewVcsToArgoMap() VcsToArgoMap { - return VcsToArgoMap{ - appDirByRepo: make(map[RepoURL]*AppDirectory), - } -} - -func (v2a *VcsToArgoMap) GetMap() map[RepoURL]*AppDirectory { - return v2a.appDirByRepo -} - -func BuildAppsMap(ctx context.Context) (VcsToArgoMap, error) { - result := NewVcsToArgoMap() - argoClient := argo_client.GetArgoClient() - - apps, err := argoClient.GetApplications(ctx) - if err != nil { - return result, errors.Wrap(err, "failed to list applications") - } - for _, app := range apps.Items { - result.AddApp(&app) - } - - return result, nil -} - -func (v2a *VcsToArgoMap) GetAppsInRepo(repoCloneUrl string) *AppDirectory { - repoUrl, err := NormalizeRepoUrl(repoCloneUrl) - if err != nil { - log.Warn().Err(err).Msgf("failed to parse %s", repoCloneUrl) - } - - appdir := v2a.appDirByRepo[repoUrl] - if appdir == nil { - appdir = NewAppDirectory() - v2a.appDirByRepo[repoUrl] = appdir - } - - return appdir -} - -func (v2a *VcsToArgoMap) WalkKustomizeApps(repo *repo.Repo, fs fs.FS) *AppDirectory { - var ( - err error - - result = NewAppDirectory() - appdir = v2a.GetAppsInRepo(repo.CloneURL) - apps = appdir.GetApps(nil) - ) - - for _, app := range apps { - appPath := app.Spec.GetSource().Path - if err = walkKustomizeFiles(result, fs, app.Name, appPath); err != nil { - log.Error().Err(err).Msgf("failed to parse kustomize.yaml in %s", appPath) - } - } - - return result -} diff --git a/pkg/conftest/conftest.go b/pkg/conftest/conftest.go index ab72876d..88312e8e 100644 --- a/pkg/conftest/conftest.go +++ b/pkg/conftest/conftest.go @@ -13,11 +13,11 @@ import ( "github.com/open-policy-agent/conftest/output" "github.com/open-policy-agent/conftest/runner" "github.com/rs/zerolog/log" - "github.com/spf13/viper" "go.opentelemetry.io/otel" "github.com/zapier/kubechecks/pkg" "github.com/zapier/kubechecks/pkg/local" + "github.com/zapier/kubechecks/pkg/msg" "github.com/zapier/kubechecks/telemetry" ) @@ -34,7 +34,9 @@ type emojiable interface { // as a GitLab comment. The validation checks resources against Zapier policies and // provides feedback for warnings or errors as informational messages. Failure to // pass a policy check currently does not block deploy. -func Conftest(ctx context.Context, app *v1alpha1.Application, repoPath string, vcs emojiable) (pkg.CheckResult, error) { +func Conftest( + ctx context.Context, app *v1alpha1.Application, repoPath string, policiesLocations []string, vcs emojiable, +) (msg.CheckResult, error) { _, span := otel.Tracer("Kubechecks").Start(ctx, "Conftest") defer span.End() @@ -42,7 +44,6 @@ func Conftest(ctx context.Context, app *v1alpha1.Application, repoPath string, v log.Debug().Str("dir", confTestDir).Str("app", app.Name).Msg("running conftest in dir for application") - policiesLocations := viper.GetStringSlice("policies-location") var locations []string for _, policiesLocation := range policiesLocations { log.Debug().Str("policies-location", policiesLocation).Msg("viper") @@ -53,7 +54,7 @@ func Conftest(ctx context.Context, app *v1alpha1.Application, repoPath string, v } if len(locations) == 0 { - return pkg.CheckResult{ + return msg.CheckResult{ State: pkg.StateWarning, Summary: "no policies locations configured", }, nil @@ -76,7 +77,7 @@ func Conftest(ctx context.Context, app *v1alpha1.Application, repoPath string, v results, err := r.Run(ctx, []string{confTestDir}) if err != nil { telemetry.SetError(span, err, "ConfTest Run") - return pkg.CheckResult{}, err + return msg.CheckResult{}, err } var b bytes.Buffer @@ -100,7 +101,7 @@ func Conftest(ctx context.Context, app *v1alpha1.Application, repoPath string, v innerStrings = append(innerStrings, passedMessage) } - var cr pkg.CheckResult + var cr msg.CheckResult if failures { cr.State = pkg.StateFailure } else if warnings { diff --git a/pkg/container/main.go b/pkg/container/main.go new file mode 100644 index 00000000..70c99b7c --- /dev/null +++ b/pkg/container/main.go @@ -0,0 +1,33 @@ +package container + +import ( + "io/fs" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + + "github.com/zapier/kubechecks/pkg/app_watcher" + "github.com/zapier/kubechecks/pkg/appdir" + "github.com/zapier/kubechecks/pkg/argo_client" + "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/vcs" +) + +type Container struct { + ApplicationWatcher *app_watcher.ApplicationWatcher + ArgoClient *argo_client.ArgoClient + + Config config.ServerConfig + + VcsClient vcs.VcsClient + VcsToArgoMap VcsToArgoMap +} + +type VcsToArgoMap interface { + AddApp(*v1alpha1.Application) + UpdateApp(old, new *v1alpha1.Application) + DeleteApp(*v1alpha1.Application) + GetVcsRepos() []string + GetAppsInRepo(string) *appdir.AppDirectory + GetMap() map[appdir.RepoURL]*appdir.AppDirectory + WalkKustomizeApps(cloneURL string, fs fs.FS) *appdir.AppDirectory +} diff --git a/pkg/diff/ai_summary.go b/pkg/diff/ai_summary.go index 760e150d..b0a123bb 100644 --- a/pkg/diff/ai_summary.go +++ b/pkg/diff/ai_summary.go @@ -7,10 +7,12 @@ import ( "github.com/zapier/kubechecks/pkg" "github.com/zapier/kubechecks/pkg/aisummary" + "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/msg" "github.com/zapier/kubechecks/telemetry" ) -func AIDiffSummary(ctx context.Context, mrNote *pkg.Message, name string, manifests []string, diff string) { +func AIDiffSummary(ctx context.Context, mrNote *msg.Message, cfg config.ServerConfig, name string, manifests []string, diff string) { ctx, span := otel.Tracer("Kubechecks").Start(ctx, "AIDiffSummary") defer span.End() @@ -19,17 +21,17 @@ func AIDiffSummary(ctx context.Context, mrNote *pkg.Message, name string, manife return } - aiSummary, err := aisummary.GetOpenAiClient().SummarizeDiff(ctx, name, manifests, diff) + aiSummary, err := aisummary.GetOpenAiClient(cfg.OpenAIAPIToken).SummarizeDiff(ctx, name, manifests, diff) if err != nil { telemetry.SetError(span, err, "OpenAI SummarizeDiff") log.Error().Err(err).Msg("failed to summarize diff") - cr := pkg.CheckResult{State: pkg.StateNone, Summary: "failed to summarize diff", Details: err.Error()} + cr := msg.CheckResult{State: pkg.StateNone, Summary: "failed to summarize diff", Details: err.Error()} mrNote.AddToAppMessage(ctx, name, cr) return } if aiSummary != "" { - cr := pkg.CheckResult{State: pkg.StateNone, Summary: "Show AI Summary Diff", Details: aiSummary} + cr := msg.CheckResult{State: pkg.StateNone, Summary: "Show AI Summary Diff", Details: aiSummary} mrNote.AddToAppMessage(ctx, name, cr) } } diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index 50592afb..af77e6d2 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -25,8 +25,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/pkg/argo_client" + "github.com/zapier/kubechecks/pkg/container" + "github.com/zapier/kubechecks/pkg/msg" "github.com/zapier/kubechecks/telemetry" ) @@ -42,17 +42,21 @@ func isAppMissingErr(err error) bool { } /* -Take cli output and return as a string or an array of strings instead of printing +GetDiff takes cli output and return as a string or an array of strings instead of printing changedFilePath should be the root of the changed folder from https://github.com/argoproj/argo-cd/blob/d3ff9757c460ae1a6a11e1231251b5d27aadcdd1/cmd/argocd/commands/app.go#L879 */ -func GetDiff(ctx context.Context, manifests []string, app argoappv1.Application, addApp, removeApp func(application2 argoappv1.Application)) (pkg.CheckResult, string, error) { +func GetDiff( + ctx context.Context, manifests []string, app argoappv1.Application, + ctr container.Container, + addApp, removeApp func(application2 argoappv1.Application), +) (msg.CheckResult, string, error) { ctx, span := otel.Tracer("Kubechecks").Start(ctx, "GetDiff") defer span.End() - argoClient := argo_client.GetArgoClient() + argoClient := ctr.ArgoClient log.Debug().Str("name", app.Name).Msg("generating diff for application...") @@ -68,7 +72,7 @@ func GetDiff(ctx context.Context, manifests []string, app argoappv1.Application, if err != nil { if !isAppMissingErr(err) { telemetry.SetError(span, err, "Get Argo Managed Resources") - return pkg.CheckResult{}, "", err + return msg.CheckResult{}, "", err } resources = new(application.ManagedResourcesResponse) @@ -88,22 +92,22 @@ func GetDiff(ctx context.Context, manifests []string, app argoappv1.Application, argoSettings, err := settingsClient.Get(ctx, &settings.SettingsQuery{}) if err != nil { telemetry.SetError(span, err, "Get Argo Cluster Settings") - return pkg.CheckResult{}, "", err + return msg.CheckResult{}, "", err } liveObjs, err := cmdutil.LiveObjects(resources.Items) if err != nil { telemetry.SetError(span, err, "Get Argo Live Objects") - return pkg.CheckResult{}, "", err + return msg.CheckResult{}, "", err } groupedObjs, err := groupObjsByKey(unstructureds, liveObjs, app.Spec.Destination.Namespace) if err != nil { - return pkg.CheckResult{}, "", err + return msg.CheckResult{}, "", err } if items, err = groupObjsForDiff(resources, groupedObjs, items, argoSettings, app.Name); err != nil { - return pkg.CheckResult{}, "", err + return msg.CheckResult{}, "", err } diffBuffer := &strings.Builder{} @@ -131,13 +135,13 @@ func GetDiff(ctx context.Context, manifests []string, app argoappv1.Application, Build() if err != nil { telemetry.SetError(span, err, "Build Diff") - return pkg.CheckResult{}, "failed to build diff", err + return msg.CheckResult{}, "failed to build diff", err } diffRes, err := argodiff.StateDiff(item.live, item.target, diffConfig) if err != nil { telemetry.SetError(span, err, "State Diff") - return pkg.CheckResult{}, "failed to state diff", err + return msg.CheckResult{}, "failed to state diff", err } if diffRes.Modified || item.target == nil || item.live == nil { @@ -159,7 +163,7 @@ func GetDiff(ctx context.Context, manifests []string, app argoappv1.Application, err := PrintDiff(diffBuffer, live, target) if err != nil { telemetry.SetError(span, err, "Print Diff") - return pkg.CheckResult{}, "", err + return msg.CheckResult{}, "", err } switch { case item.target == nil: @@ -189,7 +193,7 @@ func GetDiff(ctx context.Context, manifests []string, app argoappv1.Application, diff := diffBuffer.String() - var cr pkg.CheckResult + var cr msg.CheckResult cr.Summary = summary cr.Details = fmt.Sprintf("```diff\n%s\n```", diff) diff --git a/pkg/events/check.go b/pkg/events/check.go index d829e7e3..db7da13e 100644 --- a/pkg/events/check.go +++ b/pkg/events/check.go @@ -6,7 +6,6 @@ import ( "os" "reflect" "strings" - "sync" "sync/atomic" "time" @@ -14,7 +13,6 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/spf13/viper" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -22,11 +20,11 @@ import ( "github.com/zapier/kubechecks/pkg" "github.com/zapier/kubechecks/pkg/affected_apps" "github.com/zapier/kubechecks/pkg/argo_client" - "github.com/zapier/kubechecks/pkg/config" "github.com/zapier/kubechecks/pkg/conftest" + "github.com/zapier/kubechecks/pkg/container" "github.com/zapier/kubechecks/pkg/diff" "github.com/zapier/kubechecks/pkg/kubepug" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/msg" "github.com/zapier/kubechecks/pkg/repo_config" "github.com/zapier/kubechecks/pkg/validate" "github.com/zapier/kubechecks/pkg/vcs" @@ -34,17 +32,16 @@ import ( ) type CheckEvent struct { - client vcs.Client // Client exposing methods to communicate with platform of user choice - fileList []string // What files have changed in this PR/MR - TempWorkingDir string // Location of the local repo - repo *repo.Repo + fileList []string // What files have changed in this PR/MR + TempWorkingDir string // Location of the local repo + repo *vcs.Repo logger zerolog.Logger workerLimits int - vcsNote *pkg.Message + vcsNote *msg.Message affectedItems affected_apps.AffectedItems - cfg *config.ServerConfig + ctr container.Container addedAppsSet map[string]v1alpha1.Application appsSent int32 @@ -54,11 +51,10 @@ type CheckEvent struct { var inFlight int32 -func NewCheckEvent(repo *repo.Repo, cfg *config.ServerConfig) *CheckEvent { +func NewCheckEvent(repo *vcs.Repo, ctr container.Container) *CheckEvent { ce := &CheckEvent{ - cfg: cfg, - client: cfg.VcsClient, - repo: repo, + ctr: ctr, + repo: repo, } ce.logger = log.Logger.With().Str("repo", repo.Name).Int("event_id", repo.CheckID).Logger() @@ -67,7 +63,7 @@ func NewCheckEvent(repo *repo.Repo, cfg *config.ServerConfig) *CheckEvent { // getRepo gets the repo from a CheckEvent. In normal operations a CheckEvent can only be made by the VCSHookHandler // As the Repo is built from a webhook payload via the VCSClient, it should always be present. If not, error -func (ce *CheckEvent) getRepo(ctx context.Context) (*repo.Repo, error) { +func (ce *CheckEvent) getRepo(ctx context.Context) (*vcs.Repo, error) { _, span := otel.Tracer("Kubechecks").Start(ctx, "CheckEventGetRepo") defer span.End() var err error @@ -140,7 +136,7 @@ func (ce *CheckEvent) GetListOfChangedFiles(ctx context.Context) ([]string, erro return ce.fileList, err } -// Walks the repo to find any apps or appsets impacted by the changes in the MR/PR. +// GenerateListOfAffectedApps walks the repo to find any apps or appsets impacted by the changes in the MR/PR. func (ce *CheckEvent) GenerateListOfAffectedApps(ctx context.Context, targetBranch string) error { _, span := otel.Tracer("Kubechecks").Start(ctx, "GenerateListOfAffectedApps") defer span.End() @@ -150,10 +146,10 @@ func (ce *CheckEvent) GenerateListOfAffectedApps(ctx context.Context, targetBran cfg, _ := repo_config.LoadRepoConfig(ce.TempWorkingDir) if cfg != nil { log.Debug().Msg("using the config matcher") - matcher = affected_apps.NewConfigMatcher(cfg) + matcher = affected_apps.NewConfigMatcher(cfg, ce.ctr) } else { log.Debug().Msg("using an argocd matcher") - matcher, err = affected_apps.NewArgocdMatcher(ce.cfg.VcsToArgoMap, ce.repo, ce.TempWorkingDir) + matcher, err = affected_apps.NewArgocdMatcher(ce.ctr.VcsToArgoMap, ce.repo, ce.TempWorkingDir) if err != nil { return errors.Wrap(err, "failed to create argocd matcher") } @@ -184,14 +180,14 @@ func (ce *CheckEvent) ProcessApps(ctx context.Context) { )) defer span.End() - err := ce.client.TidyOutdatedComments(ctx, ce.repo) + err := ce.ctr.VcsClient.TidyOutdatedComments(ctx, ce.repo) if err != nil { ce.logger.Error().Err(err).Msg("Failed to tidy outdated comments") } if len(ce.affectedItems.Applications) <= 0 && len(ce.affectedItems.ApplicationSets) <= 0 { ce.logger.Info().Msg("No affected apps or appsets, skipping") - ce.client.PostMessage(ctx, ce.repo, ce.repo.CheckID, "No changes") + ce.ctr.VcsClient.PostMessage(ctx, ce.repo, ce.repo.CheckID, "No changes") return } @@ -237,7 +233,7 @@ func (ce *CheckEvent) ProcessApps(ctx context.Context) { ce.logger.Info().Msg("Finished") comment := ce.vcsNote.BuildComment(ctx) - if err = ce.client.UpdateMessage(ctx, ce.vcsNote, comment); err != nil { + if err = ce.ctr.VcsClient.UpdateMessage(ctx, ce.vcsNote, comment); err != nil { ce.logger.Error().Err(err).Msg("failed to push comment") } @@ -280,7 +276,7 @@ func (ce *CheckEvent) CommitStatus(ctx context.Context, status pkg.CommitState) _, span := otel.Tracer("Kubechecks").Start(ctx, "CommitStatus") defer span.End() - if err := ce.client.CommitStatus(ctx, ce.repo, status); err != nil { + if err := ce.ctr.VcsClient.CommitStatus(ctx, ce.repo, status); err != nil { log.Warn().Err(err).Msg("failed to update commit status") } } @@ -323,47 +319,43 @@ func (ce *CheckEvent) processApp(ctx context.Context, app v1alpha1.Application) ce.vcsNote.AddNewApp(ctx, appName) ce.logger.Debug().Msgf("Getting manifests for app: %s with code at %s/%s", appName, ce.TempWorkingDir, dir) - manifests, err := argo_client.GetManifestsLocal(ctx, appName, ce.TempWorkingDir, dir, app) + jsonManifests, err := argo_client.GetManifestsLocal(ctx, ce.ctr.ArgoClient, appName, ce.TempWorkingDir, dir, app) if err != nil { ce.logger.Error().Err(err).Msgf("Unable to get manifests for %s in %s", appName, dir) - cr := pkg.CheckResult{State: pkg.StateError, Summary: "Unable to get manifests", Details: fmt.Sprintf("```\n%s\n```", ce.cleanupGetManifestsError(err))} + cr := msg.CheckResult{State: pkg.StateError, Summary: "Unable to get manifests", Details: fmt.Sprintf("```\n%s\n```", ce.cleanupGetManifestsError(err))} ce.vcsNote.AddToAppMessage(ctx, appName, cr) return } // Argo diff logic wants unformatted manifests but everything else wants them as YAML, so we prepare both - formattedManifests := argo_client.FormatManifestsYAML(manifests) - ce.logger.Trace().Msgf("Manifests:\n%+v\n", formattedManifests) + yamlManifests := argo_client.ConvertJsonToYamlManifests(jsonManifests) + ce.logger.Trace().Msgf("Manifests:\n%+v\n", yamlManifests) - k8sVersion, err := argo_client.GetArgoClient().GetKubernetesVersionByApplication(ctx, app) + k8sVersion, err := ce.ctr.ArgoClient.GetKubernetesVersionByApplication(ctx, app) if err != nil { ce.logger.Error().Err(err).Msg("Error retrieving the Kubernetes version") - k8sVersion = viper.GetString("fallback-k8s-version") + k8sVersion = ce.ctr.Config.FallbackK8sVersion } else { k8sVersion = fmt.Sprintf("%s.0", k8sVersion) ce.logger.Info().Msgf("Kubernetes version: %s", k8sVersion) } - var wg sync.WaitGroup + runner := newRunner(span, ctx, app, appName, k8sVersion, ce.TempWorkingDir, jsonManifests, yamlManifests, ce.logger, ce.vcsNote) - run := ce.createRunner(span, ctx, appName, &wg) + runner.Run("validating app against schema", ce.validateSchemas) + runner.Run("generating diff for app", ce.generateDiff) - run("validating app against schema", ce.validateSchemas(ctx, appName, k8sVersion, ce.TempWorkingDir, formattedManifests)) - run("generating diff for app", ce.generateDiff(ctx, app, manifests, ce.queueApp, ce.removeApp)) - - if viper.GetBool("enable-conftest") { - run("validation policy", ce.validatePolicy(ctx, appName)) + if ce.ctr.Config.EnableConfTest { + runner.Run("validation policy", ce.validatePolicy) } - run("running pre-upgrade check", ce.runPreupgradeCheck(ctx, appName, k8sVersion, formattedManifests)) + runner.Run("running pre-upgrade check", ce.runPreupgradeCheck) - wg.Wait() + runner.Wait() - ce.vcsNote.SetFooter(start, ce.repo.SHA) + ce.vcsNote.SetFooter(start, ce.repo.SHA, ce.ctr.Config.LabelFilter, ce.ctr.Config.ShowDebugInfo) } -type checkFunction func() (pkg.CheckResult, error) - const ( errorCommentFormat = ` :warning: **Error while %s** :warning: @@ -375,105 +367,59 @@ Check kubechecks application logs for more information. ` ) -func (ce *CheckEvent) createRunner(span trace.Span, grpCtx context.Context, app string, wg *sync.WaitGroup) func(string, checkFunction) { - return func(desc string, fn checkFunction) { - wg.Add(1) - - go func() { - defer func() { - wg.Done() - - if r := recover(); r != nil { - ce.logger.Error().Str("app", app).Str("check", desc).Msgf("panic while running check") - - telemetry.SetError(span, fmt.Errorf("%v", r), desc) - result := pkg.CheckResult{State: pkg.StatePanic, Summary: desc, Details: fmt.Sprintf(errorCommentFormat, desc, r)} - ce.vcsNote.AddToAppMessage(grpCtx, app, result) - } - }() - - ce.logger.Info().Str("app", app).Str("check", desc).Msgf("running check") - cr, err := fn() - ce.logger.Info(). - Err(err). - Str("app", app). - Str("check", desc). - Uint8("result", uint8(cr.State)). - Msg("check result") - - if err != nil { - telemetry.SetError(span, err, desc) - result := pkg.CheckResult{State: pkg.StateError, Summary: desc, Details: fmt.Sprintf(errorCommentFormat, desc, err)} - ce.vcsNote.AddToAppMessage(grpCtx, app, result) - return - } - - ce.vcsNote.AddToAppMessage(grpCtx, app, cr) - - ce.logger.Info().Str("app", app).Str("check", desc).Str("result", cr.State.BareString()).Msgf("check done") - }() - } -} - -func (ce *CheckEvent) runPreupgradeCheck(grpCtx context.Context, app string, k8sVersion string, formattedManifests []string) func() (pkg.CheckResult, error) { - return func() (pkg.CheckResult, error) { - s, err := kubepug.CheckApp(grpCtx, app, k8sVersion, formattedManifests) - if err != nil { - return pkg.CheckResult{}, err - } +var EmptyCheckResult msg.CheckResult - return s, nil +func (ce *CheckEvent) runPreupgradeCheck(data CheckData) (msg.CheckResult, error) { + s, err := kubepug.CheckApp(data.ctx, data.appName, data.k8sVersion, data.yamlManifests) + if err != nil { + return EmptyCheckResult, err } -} -func (ce *CheckEvent) validatePolicy(ctx context.Context, app string) func() (pkg.CheckResult, error) { - return func() (pkg.CheckResult, error) { - argoApp, err := argo_client.GetArgoClient().GetApplicationByName(ctx, app) - if err != nil { - return pkg.CheckResult{}, errors.Wrapf(err, "could not retrieve ArgoCD App data: %q", app) - } + return s, nil +} - cr, err := conftest.Conftest(ctx, argoApp, ce.TempWorkingDir, ce.client) - if err != nil { - return pkg.CheckResult{}, err - } +func (ce *CheckEvent) validatePolicy(data CheckData) (msg.CheckResult, error) { + argoApp, err := ce.ctr.ArgoClient.GetApplicationByName(data.ctx, data.appName) + if err != nil { + return EmptyCheckResult, errors.Wrapf(err, "could not retrieve ArgoCD App data: %q", data.appName) + } - return cr, nil + cr, err := conftest.Conftest(data.ctx, argoApp, ce.TempWorkingDir, ce.ctr.Config.PoliciesLocation, ce.ctr.VcsClient) + if err != nil { + return EmptyCheckResult, err } + + return cr, nil } -func (ce *CheckEvent) generateDiff(ctx context.Context, app v1alpha1.Application, manifests []string, addApp, removeApp func(app v1alpha1.Application)) func() (pkg.CheckResult, error) { - return func() (pkg.CheckResult, error) { - cr, rawDiff, err := diff.GetDiff(ctx, manifests, app, addApp, removeApp) - if err != nil { - return pkg.CheckResult{}, err - } +func (ce *CheckEvent) generateDiff(data CheckData) (msg.CheckResult, error) { + cr, rawDiff, err := diff.GetDiff(data.ctx, data.jsonManifests, data.app, ce.ctr, ce.queueApp, ce.removeApp) + if err != nil { + return EmptyCheckResult, err + } - diff.AIDiffSummary(ctx, ce.vcsNote, app.Name, manifests, rawDiff) + diff.AIDiffSummary(data.ctx, ce.vcsNote, ce.ctr.Config, data.appName, data.jsonManifests, rawDiff) - return cr, nil - } + return cr, nil } -func (ce *CheckEvent) validateSchemas(ctx context.Context, app, k8sVersion, tempRepoPath string, formattedManifests []string) func() (pkg.CheckResult, error) { - return func() (pkg.CheckResult, error) { - cr, err := validate.ArgoCdAppValidate(ctx, app, k8sVersion, tempRepoPath, formattedManifests) - if err != nil { - return pkg.CheckResult{}, err - } - - return cr, nil +func (ce *CheckEvent) validateSchemas(data CheckData) (msg.CheckResult, error) { + cr, err := validate.ArgoCdAppValidate(data.ctx, ce.ctr.Config, data.appName, data.k8sVersion, data.repoPath, data.yamlManifests) + if err != nil { + return EmptyCheckResult, err } + + return cr, nil } // Creates a generic Note struct that we can write into across all worker threads -func (ce *CheckEvent) createNote(ctx context.Context) *pkg.Message { +func (ce *CheckEvent) createNote(ctx context.Context) *msg.Message { ctx, span := otel.Tracer("check").Start(ctx, "createNote") defer span.End() ce.logger.Info().Msgf("Creating note") - return ce.client.PostMessage(ctx, ce.repo, ce.repo.CheckID, ":hourglass: kubechecks running ... ") + return ce.ctr.VcsClient.PostMessage(ctx, ce.repo, ce.repo.CheckID, ":hourglass: kubechecks running ... ") } // cleanupGetManifestsError takes an error as input and returns a simplified and more user-friendly error message. diff --git a/pkg/events/runner.go b/pkg/events/runner.go new file mode 100644 index 00000000..66c3cfb3 --- /dev/null +++ b/pkg/events/runner.go @@ -0,0 +1,116 @@ +package events + +import ( + "context" + "fmt" + "sync" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/rs/zerolog" + "go.opentelemetry.io/otel/trace" + + "github.com/zapier/kubechecks/pkg" + "github.com/zapier/kubechecks/pkg/msg" + "github.com/zapier/kubechecks/telemetry" +) + +type CheckData struct { + span trace.Span + ctx context.Context + logger zerolog.Logger + note *msg.Message + app v1alpha1.Application + + appName string + k8sVersion string + repoPath string + jsonManifests []string + yamlManifests []string +} + +type Runner struct { + CheckData + + wg sync.WaitGroup +} + +func newRunner( + span trace.Span, ctx context.Context, app v1alpha1.Application, + appName, k8sVersion, repoPath string, jsonManifests, yamlManifests []string, + logger zerolog.Logger, note *msg.Message, +) *Runner { + logger = logger. + With(). + Str("app", appName). + Logger() + + return &Runner{ + CheckData: CheckData{ + app: app, + appName: appName, + k8sVersion: k8sVersion, + repoPath: repoPath, + jsonManifests: jsonManifests, + yamlManifests: yamlManifests, + + ctx: ctx, + logger: logger, + note: note, + span: span, + }, + } +} + +type checkFunction func(data CheckData) (msg.CheckResult, error) + +func (r *Runner) Run(desc string, fn checkFunction) { + r.wg.Add(1) + + go func() { + logger := r.logger + + defer func() { + r.wg.Done() + + if err := recover(); err != nil { + logger.Error().Str("check", desc).Msgf("panic while running check") + + telemetry.SetError(r.span, fmt.Errorf("%v", err), desc) + result := msg.CheckResult{ + State: pkg.StatePanic, + Summary: desc, + Details: fmt.Sprintf(errorCommentFormat, desc, err), + } + r.note.AddToAppMessage(r.ctx, r.appName, result) + } + }() + + logger = logger.With(). + Str("check", desc). + Logger() + + logger.Info().Msgf("running check") + cr, err := fn(r.CheckData) + logger.Info(). + Err(err). + Uint8("result", uint8(cr.State)). + Msg("check result") + + if err != nil { + telemetry.SetError(r.span, err, desc) + result := msg.CheckResult{State: pkg.StateError, Summary: desc, Details: fmt.Sprintf(errorCommentFormat, desc, err)} + r.note.AddToAppMessage(r.ctx, r.appName, result) + return + } + + r.note.AddToAppMessage(r.ctx, r.appName, cr) + + logger.Info(). + Str("result", cr.State.BareString()). + Msgf("check done") + }() +} + +func (r *Runner) Wait() { + r.wg.Wait() +} diff --git a/pkg/kubepug/kubepug.go b/pkg/kubepug/kubepug.go index 092627e4..90711c50 100644 --- a/pkg/kubepug/kubepug.go +++ b/pkg/kubepug/kubepug.go @@ -15,11 +15,12 @@ import ( "go.opentelemetry.io/otel" "github.com/zapier/kubechecks/pkg" + "github.com/zapier/kubechecks/pkg/msg" ) const docLinkFmt = "[%s Deprecation Notes](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#%s-v%d%d)" -func CheckApp(ctx context.Context, appName, targetKubernetesVersion string, manifests []string) (pkg.CheckResult, error) { +func CheckApp(ctx context.Context, appName, targetKubernetesVersion string, manifests []string) (msg.CheckResult, error) { _, span := otel.Tracer("Kubechecks").Start(ctx, "KubePug") defer span.End() @@ -33,7 +34,7 @@ func CheckApp(ctx context.Context, appName, targetKubernetesVersion string, mani if err != nil { log.Error().Err(err).Msg("could not create temp directory to write manifests for kubepug check") //return "", err - return pkg.CheckResult{}, err + return msg.CheckResult{}, err } defer os.RemoveAll(tempDir) @@ -44,7 +45,7 @@ func CheckApp(ctx context.Context, appName, targetKubernetesVersion string, mani nextVersion, err := nextKubernetesVersion(targetKubernetesVersion) if err != nil { - return pkg.CheckResult{}, err + return msg.CheckResult{}, err } config := lib.Config{ K8sVersion: fmt.Sprintf("v%s", nextVersion.String()), @@ -57,7 +58,7 @@ func CheckApp(ctx context.Context, appName, targetKubernetesVersion string, mani result, err := kubepug.GetDeprecated() if err != nil { - return pkg.CheckResult{}, err + return msg.CheckResult{}, err } if len(result.DeprecatedAPIs) > 0 || len(result.DeletedAPIs) > 0 { @@ -112,7 +113,7 @@ func CheckApp(ctx context.Context, appName, targetKubernetesVersion string, mani outputString = append(outputString, "No Deprecated or Deleted APIs found.") } - return pkg.CheckResult{ + return msg.CheckResult{ State: checkStatus(result), Summary: "Show kubepug report:", Details: fmt.Sprintf( diff --git a/pkg/local/gitRepos.go b/pkg/local/gitRepos.go index af659515..60b9549d 100644 --- a/pkg/local/gitRepos.go +++ b/pkg/local/gitRepos.go @@ -10,7 +10,7 @@ import ( "github.com/rs/zerolog/log" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/vcs" ) type ReposDirectory struct { @@ -85,7 +85,7 @@ func (rd *ReposDirectory) clone(ctx context.Context, cloneUrl string) string { return "" } - r := repo.Repo{CloneURL: cloneUrl} + r := vcs.Repo{CloneURL: cloneUrl} err = r.CloneRepoLocal(ctx, repoDir) if err != nil { log.Err(err).Str("clone-url", cloneUrl).Msg("failed to clone repository") diff --git a/pkg/message.go b/pkg/msg/message.go similarity index 80% rename from pkg/message.go rename to pkg/msg/message.go index 07d4743b..a4507096 100644 --- a/pkg/message.go +++ b/pkg/msg/message.go @@ -1,4 +1,4 @@ -package pkg +package msg import ( "context" @@ -8,14 +8,15 @@ import ( "sync" "time" - "github.com/spf13/viper" "go.opentelemetry.io/otel" "golang.org/x/exp/constraints" "golang.org/x/exp/slices" + + "github.com/zapier/kubechecks/pkg" ) type CheckResult struct { - State CommitState + State pkg.CommitState Summary, Details string } @@ -40,7 +41,7 @@ func NewMessage(name string, prId, commentId int, vcs toEmoji) *Message { } type toEmoji interface { - ToEmoji(state CommitState) string + ToEmoji(state pkg.CommitState) string } // Message type that allows concurrent updates @@ -61,8 +62,8 @@ type Message struct { deletedAppsSet map[string]struct{} } -func (m *Message) WorstState() CommitState { - state := StateNone +func (m *Message) WorstState() pkg.CommitState { + state := pkg.StateNone for app, r := range m.apps { if m.isDeleted(app) { @@ -70,7 +71,7 @@ func (m *Message) WorstState() CommitState { } for _, result := range r.results { - state = WorstState(state, result.State) + state = pkg.WorstState(state, result.State) } } @@ -124,28 +125,23 @@ func init() { hostname, _ = os.Hostname() } -func (m *Message) SetFooter(start time.Time, commitSha string) { - m.footer = buildFooter(start, commitSha) -} - -func (m *Message) BuildComment(ctx context.Context) string { - return m.buildComment(ctx) -} - -func buildFooter(start time.Time, commitSHA string) string { - showDebug := viper.GetBool("show-debug-info") - if !showDebug { - return fmt.Sprintf("_Done. CommitSHA: %s_\n", commitSHA) +func (m *Message) SetFooter(start time.Time, commitSHA, labelFilter string, showDebugInfo bool) { + if !showDebugInfo { + m.footer = fmt.Sprintf("_Done. CommitSHA: %s_\n", commitSHA) + return } - label := viper.GetString("label-filter") envStr := "" - if label != "" { - envStr = fmt.Sprintf(", Env: %s", label) + if labelFilter != "" { + envStr = fmt.Sprintf(", Env: %s", labelFilter) } duration := time.Since(start) - return fmt.Sprintf("_Done: Pod: %s, Dur: %v, SHA: %s%s_\n", hostname, duration, GitCommit, envStr) + m.footer = fmt.Sprintf("_Done: Pod: %s, Dur: %v, SHA: %s%s_\n", hostname, duration, pkg.GitCommit, envStr) +} + +func (m *Message) BuildComment(ctx context.Context) string { + return m.buildComment(ctx) } // Iterate the map of all apps in this message, building a final comment from their current state @@ -166,10 +162,10 @@ func (m *Message) buildComment(ctx context.Context) string { var checkStrings []string results := m.apps[appName] - appState := StateSuccess + appState := pkg.StateSuccess for _, check := range results.results { var summary string - if check.State == StateNone { + if check.State == pkg.StateNone { summary = check.Summary } else { summary = fmt.Sprintf("%s %s %s", check.Summary, check.State.BareString(), m.vcs.ToEmoji(check.State)) @@ -177,7 +173,7 @@ func (m *Message) buildComment(ctx context.Context) string { msg := fmt.Sprintf("
\n%s\n\n%s\n
", summary, check.Details) checkStrings = append(checkStrings, msg) - appState = WorstState(appState, check.State) + appState = pkg.WorstState(appState, check.State) } sb.WriteString("
\n") diff --git a/pkg/message_test.go b/pkg/msg/message_test.go similarity index 72% rename from pkg/message_test.go rename to pkg/msg/message_test.go index f86f6ea3..faa68bb0 100644 --- a/pkg/message_test.go +++ b/pkg/msg/message_test.go @@ -1,4 +1,4 @@ -package pkg +package msg import ( "context" @@ -6,20 +6,22 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/zapier/kubechecks/pkg" ) type fakeEmojiable struct { emoji string } -func (fe fakeEmojiable) ToEmoji(state CommitState) string { return fe.emoji } +func (fe fakeEmojiable) ToEmoji(state pkg.CommitState) string { return fe.emoji } func TestBuildComment(t *testing.T) { appResults := map[string]*AppResults{ "myapp": { results: []CheckResult{ { - State: StateError, + State: pkg.StateError, Summary: "this failed bigly", Details: "should add some important details here", }, @@ -51,42 +53,42 @@ func TestMessageIsSuccess(t *testing.T) { ) // no apps mean success - assert.Equal(t, StateNone, message.WorstState()) + assert.Equal(t, pkg.StateNone, message.WorstState()) // one app, no checks = success message.AddNewApp(ctx, "some-app") - assert.Equal(t, StateNone, message.WorstState()) + assert.Equal(t, pkg.StateNone, message.WorstState()) // one app, one success = success - message.AddToAppMessage(ctx, "some-app", CheckResult{State: StateSuccess}) - assert.Equal(t, StateSuccess, message.WorstState()) + message.AddToAppMessage(ctx, "some-app", CheckResult{State: pkg.StateSuccess}) + assert.Equal(t, pkg.StateSuccess, message.WorstState()) // one app, one success, one failure = failure - message.AddToAppMessage(ctx, "some-app", CheckResult{State: StateFailure}) - assert.Equal(t, StateFailure, message.WorstState()) + message.AddToAppMessage(ctx, "some-app", CheckResult{State: pkg.StateFailure}) + assert.Equal(t, pkg.StateFailure, message.WorstState()) // one app, two successes, one failure = failure - message.AddToAppMessage(ctx, "some-app", CheckResult{State: StateSuccess}) - assert.Equal(t, StateFailure, message.WorstState()) + message.AddToAppMessage(ctx, "some-app", CheckResult{State: pkg.StateSuccess}) + assert.Equal(t, pkg.StateFailure, message.WorstState()) // one app, two successes, one failure = failure - message.AddToAppMessage(ctx, "some-app", CheckResult{State: StateSuccess}) - assert.Equal(t, StateFailure, message.WorstState()) + message.AddToAppMessage(ctx, "some-app", CheckResult{State: pkg.StateSuccess}) + assert.Equal(t, pkg.StateFailure, message.WorstState()) // two apps: second app's success does not override first app's failure message.AddNewApp(ctx, "some-other-app") - message.AddToAppMessage(ctx, "some-other-app", CheckResult{State: StateSuccess}) - assert.Equal(t, StateFailure, message.WorstState()) + message.AddToAppMessage(ctx, "some-other-app", CheckResult{State: pkg.StateSuccess}) + assert.Equal(t, pkg.StateFailure, message.WorstState()) }) - testcases := map[CommitState]struct{}{ - StateNone: {}, - StateSuccess: {}, - StateRunning: {}, - StateWarning: {}, - StateFailure: {}, - StateError: {}, - StatePanic: {}, + testcases := map[pkg.CommitState]struct{}{ + pkg.StateNone: {}, + pkg.StateSuccess: {}, + pkg.StateRunning: {}, + pkg.StateWarning: {}, + pkg.StateFailure: {}, + pkg.StateError: {}, + pkg.StatePanic: {}, } for state := range testcases { @@ -109,23 +111,23 @@ func TestMultipleItemsWithNewlines(t *testing.T) { ) message.AddNewApp(ctx, "first-app") message.AddToAppMessage(ctx, "first-app", CheckResult{ - State: StateSuccess, + State: pkg.StateSuccess, Summary: "summary-1", Details: "detail-1", }) message.AddToAppMessage(ctx, "first-app", CheckResult{ - State: StateSuccess, + State: pkg.StateSuccess, Summary: "summary-2", Details: "detail-2", }) message.AddNewApp(ctx, "second-app") message.AddToAppMessage(ctx, "second-app", CheckResult{ - State: StateSuccess, + State: pkg.StateSuccess, Summary: "summary-1", Details: "detail-1", }) message.AddToAppMessage(ctx, "second-app", CheckResult{ - State: StateSuccess, + State: pkg.StateSuccess, Summary: "summary-2", Details: "detail-2", }) diff --git a/pkg/server/hook_handler.go b/pkg/server/hook_handler.go index 67fa6614..601ccc27 100644 --- a/pkg/server/hook_handler.go +++ b/pkg/server/hook_handler.go @@ -8,36 +8,28 @@ import ( "github.com/labstack/echo/v4" "github.com/rs/zerolog/log" - "github.com/spf13/viper" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/container" "github.com/zapier/kubechecks/pkg/events" - "github.com/zapier/kubechecks/pkg/repo" "github.com/zapier/kubechecks/pkg/vcs" "github.com/zapier/kubechecks/telemetry" ) type VCSHookHandler struct { - client vcs.Client - cfg *config.ServerConfig - // labelFilter is a string specifying the required label name to filter merge events by; if empty, all merge events will pass the filter. - labelFilter string + ctr container.Container } -func NewVCSHookHandler(cfg *config.ServerConfig) *VCSHookHandler { - labelFilter := viper.GetString("label-filter") - +func NewVCSHookHandler(ctr container.Container) *VCSHookHandler { return &VCSHookHandler{ - client: cfg.VcsClient, - cfg: cfg, - labelFilter: labelFilter, + ctr: ctr, } } func (h *VCSHookHandler) AttachHandlers(grp *echo.Group) { - projectHookPath := fmt.Sprintf("/%s/project", h.cfg.VcsClient.GetName()) + projectHookPath := fmt.Sprintf("/%s/project", h.ctr.VcsClient.GetName()) grp.POST(projectHookPath, h.groupHandler) } @@ -45,13 +37,13 @@ func (h *VCSHookHandler) groupHandler(c echo.Context) error { ctx := context.Background() log.Debug().Msg("Received hook request") // Always verify, even if no secret (no op if no secret) - payload, err := h.client.VerifyHook(c.Request(), h.cfg.WebhookSecret) + payload, err := h.ctr.VcsClient.VerifyHook(c.Request(), h.ctr.Config.WebhookSecret) if err != nil { log.Err(err).Msg("Failed to verify hook") return c.String(http.StatusUnauthorized, "Unauthorized") } - r, err := h.client.ParseHook(c.Request(), payload) + r, err := h.ctr.VcsClient.ParseHook(c.Request(), payload) if err != nil { switch err { case vcs.ErrInvalidType: @@ -71,16 +63,16 @@ func (h *VCSHookHandler) groupHandler(c echo.Context) error { // Takes a constructed Repo, and attempts to run the Kubechecks processing suite against it. // If the Repo is not yet populated, this will fail. -func (h *VCSHookHandler) processCheckEvent(ctx context.Context, repo *repo.Repo) { +func (h *VCSHookHandler) processCheckEvent(ctx context.Context, repo *vcs.Repo) { if !h.passesLabelFilter(repo) { - log.Warn().Str("label-filter", h.labelFilter).Msg("ignoring event, did not have matching label") + log.Warn().Str("label-filter", h.ctr.Config.LabelFilter).Msg("ignoring event, did not have matching label") return } - ProcessCheckEvent(ctx, repo, h.cfg) + ProcessCheckEvent(ctx, repo, h.ctr.Config, h.ctr) } -func ProcessCheckEvent(ctx context.Context, r *repo.Repo, cfg *config.ServerConfig) { +func ProcessCheckEvent(ctx context.Context, r *vcs.Repo, cfg config.ServerConfig, ctr container.Container) { var span trace.Span ctx, span = otel.Tracer("Kubechecks").Start(ctx, "processCheckEvent", trace.WithAttributes( @@ -95,7 +87,7 @@ func ProcessCheckEvent(ctx context.Context, r *repo.Repo, cfg *config.ServerConf defer span.End() // If we've gotten here, we can now begin running checks (or trying to) - cEvent := events.NewCheckEvent(r, cfg) + cEvent := events.NewCheckEvent(r, ctr) err := cEvent.CreateTempDir() if err != nil { @@ -104,7 +96,7 @@ func ProcessCheckEvent(ctx context.Context, r *repo.Repo, cfg *config.ServerConf } defer cEvent.Cleanup(ctx) - err = repo.InitializeGitSettings(cfg.VcsClient.Username(), cfg.VcsClient.Email()) + err = vcs.InitializeGitSettings(ctr.Config, ctr.VcsClient) if err != nil { telemetry.SetError(span, err, "Initialize Git") log.Error().Err(err).Msg("unable to initialize git") @@ -154,7 +146,7 @@ func ProcessCheckEvent(ctx context.Context, r *repo.Repo, cfg *config.ServerConf // and matches the handler's labelFilter. Returns true if there's a matching label or no // "kubechecks:" labels are found, and false if a "kubechecks:" label is found but none match // the labelFilter. -func (h *VCSHookHandler) passesLabelFilter(repo *repo.Repo) bool { +func (h *VCSHookHandler) passesLabelFilter(repo *vcs.Repo) bool { foundKubechecksLabel := false for _, label := range repo.Labels { @@ -165,7 +157,7 @@ func (h *VCSHookHandler) passesLabelFilter(repo *repo.Repo) bool { // Get the remaining string after "kubechecks:" remainingString := strings.TrimPrefix(label, "kubechecks:") - if remainingString == h.labelFilter { + if remainingString == h.ctr.Config.LabelFilter { log.Debug().Str("mr_label", label).Msg("label is match for our filter") return true } @@ -178,7 +170,7 @@ func (h *VCSHookHandler) passesLabelFilter(repo *repo.Repo) bool { } // Return false if we have a label filter, but it did not match any labels on the event - if h.labelFilter != "" { + if h.ctr.Config.LabelFilter != "" { return false } diff --git a/pkg/server/server.go b/pkg/server/server.go index 032af4ab..05ca3884 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -12,46 +12,23 @@ import ( "github.com/labstack/echo/v4/middleware" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/spf13/viper" "github.com/ziflex/lecho/v3" - "github.com/zapier/kubechecks/pkg/app_watcher" - "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/container" "github.com/zapier/kubechecks/pkg/vcs" ) const KubeChecksHooksPathPrefix = "/hooks" type Server struct { - cfg *config.ServerConfig - appWatcher *app_watcher.ApplicationWatcher + ctr container.Container } -func NewServer(ctx context.Context, cfg *config.ServerConfig) *Server { - var appWatcher *app_watcher.ApplicationWatcher - if viper.GetBool("monitor-all-applications") { - argoMap, err := config.BuildAppsMap(ctx) - if err != nil { - log.Fatal().Err(err).Msg("could not build VcsToArgoMap") - } - cfg.VcsToArgoMap = argoMap - - appWatcher, err = app_watcher.NewApplicationWatcher(cfg) - if err != nil { - log.Fatal().Err(err).Msg("could not create ApplicationWatcher") - } - } else { - cfg.VcsToArgoMap = config.NewVcsToArgoMap() - } - - return &Server{cfg: cfg, appWatcher: appWatcher} +func NewServer(ctr container.Container) *Server { + return &Server{ctr: ctr} } func (s *Server) Start(ctx context.Context) { - if s.appWatcher != nil { - go s.appWatcher.Run(ctx, 1) - } - if err := s.ensureWebhooks(ctx); err != nil { log.Warn().Err(err).Msg("failed to create webhooks") } @@ -71,7 +48,7 @@ func (s *Server) Start(ctx context.Context) { hooksGroup := e.Group(s.hooksPrefix()) - ghHooks := NewVCSHookHandler(s.cfg) + ghHooks := NewVCSHookHandler(s.ctr) ghHooks.AttachHandlers(hooksGroup) fmt.Println("Method\tPath") @@ -85,7 +62,7 @@ func (s *Server) Start(ctx context.Context) { } func (s *Server) hooksPrefix() string { - prefix := s.cfg.UrlPrefix + prefix := s.ctr.Config.UrlPrefix serverUrl, err := url.JoinPath("/", prefix, KubeChecksHooksPathPrefix) if err != nil { log.Warn().Err(err).Msg(":whatintarnation:") @@ -95,22 +72,22 @@ func (s *Server) hooksPrefix() string { } func (s *Server) ensureWebhooks(ctx context.Context) error { - if !viper.GetBool("ensure-webhooks") { + if !s.ctr.Config.EnsureWebhooks { return nil } - if !viper.GetBool("monitor-all-applications") { + if !s.ctr.Config.MonitorAllApplications { return errors.New("must enable 'monitor-all-applications' to create webhooks") } - urlBase := viper.GetString("webhook-url-base") + urlBase := s.ctr.Config.WebhookUrlBase if urlBase == "" { return errors.New("must define 'webhook-url-base' to create webhooks") } log.Info().Msg("ensuring all webhooks are created correctly") - vcsClient := s.cfg.VcsClient + vcsClient := s.ctr.VcsClient fullUrl, err := url.JoinPath(urlBase, s.hooksPrefix(), vcsClient.GetName(), "project") if err != nil { @@ -119,7 +96,7 @@ func (s *Server) ensureWebhooks(ctx context.Context) error { } log.Info().Str("webhookUrl", fullUrl).Msg("webhook URL for this kubechecks instance") - for _, repo := range s.cfg.GetVcsRepos() { + for _, repo := range s.ctr.VcsToArgoMap.GetVcsRepos() { wh, err := vcsClient.GetHookByUrl(ctx, repo, fullUrl) if err != nil && !errors.Is(err, vcs.ErrHookNotFound) { log.Error().Err(err).Msgf("failed to get hook for %s:", repo) @@ -127,7 +104,7 @@ func (s *Server) ensureWebhooks(ctx context.Context) error { } if wh == nil { - if err = vcsClient.CreateHook(ctx, repo, fullUrl, s.cfg.WebhookSecret); err != nil { + if err = vcsClient.CreateHook(ctx, repo, fullUrl, s.ctr.Config.WebhookSecret); err != nil { log.Info().Err(err).Msgf("failed to create hook for %s:", repo) } } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index a9e29719..8550d230 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -1,57 +1,57 @@ package server import ( - "context" "testing" "github.com/zapier/kubechecks/pkg/config" + "github.com/zapier/kubechecks/pkg/container" ) func TestHooksPrefix(t *testing.T) { tests := []struct { name string want string - cfg *config.ServerConfig + cfg config.ServerConfig }{ { name: "no-prefix", want: "/hooks", - cfg: &config.ServerConfig{ + cfg: config.ServerConfig{ UrlPrefix: "", }, }, { name: "prefix-no-slash", want: "/test/hooks", - cfg: &config.ServerConfig{ + cfg: config.ServerConfig{ UrlPrefix: "test", }, }, { name: "prefix-trailing-slash", want: "/test/hooks", - cfg: &config.ServerConfig{ + cfg: config.ServerConfig{ UrlPrefix: "test/", }, }, { name: "prefix-leading-slash", want: "/test/hooks", - cfg: &config.ServerConfig{ + cfg: config.ServerConfig{ UrlPrefix: "/test", }, }, { name: "prefix-slash-sandwich", want: "/test/hooks", - cfg: &config.ServerConfig{ + cfg: config.ServerConfig{ UrlPrefix: "/test/", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewServer(context.TODO(), tt.cfg) + s := NewServer(container.Container{Config: tt.cfg}) if got := s.hooksPrefix(); got != tt.want { t.Errorf("hooksPrefix() = %v, want %v", got, tt.want) } diff --git a/pkg/validate/validate.go b/pkg/validate/validate.go index 8c74b95d..7d17a5c3 100644 --- a/pkg/validate/validate.go +++ b/pkg/validate/validate.go @@ -9,25 +9,25 @@ import ( "strings" "github.com/rs/zerolog/log" - "github.com/spf13/viper" "github.com/yannh/kubeconform/pkg/validator" "go.opentelemetry.io/otel" "github.com/zapier/kubechecks/pkg" + "github.com/zapier/kubechecks/pkg/config" "github.com/zapier/kubechecks/pkg/local" + "github.com/zapier/kubechecks/pkg/msg" ) var reposCache = local.NewReposDirectory() -func getSchemaLocations(ctx context.Context, tempRepoPath string) []string { +func getSchemaLocations(ctx context.Context, cfg config.ServerConfig, tempRepoPath string) []string { locations := []string{ // schemas included in kubechecks "default", } // schemas configured globally - schemasLocations := viper.GetStringSlice("schemas-location") - for _, schemasLocation := range schemasLocations { + for _, schemasLocation := range cfg.SchemasLocations { log.Debug().Str("schemas-location", schemasLocation).Msg("viper") schemaPath := reposCache.EnsurePath(ctx, tempRepoPath, schemasLocation) if schemaPath != "" { @@ -52,7 +52,7 @@ func getSchemaLocations(ctx context.Context, tempRepoPath string) []string { return locations } -func ArgoCdAppValidate(ctx context.Context, appName, targetKubernetesVersion, tempRepoPath string, appManifests []string) (pkg.CheckResult, error) { +func ArgoCdAppValidate(ctx context.Context, cfg config.ServerConfig, appName, targetKubernetesVersion, tempRepoPath string, appManifests []string) (msg.CheckResult, error) { _, span := otel.Tracer("Kubechecks").Start(ctx, "ArgoCdAppValidate") defer span.End() @@ -74,7 +74,7 @@ func ArgoCdAppValidate(ctx context.Context, appName, targetKubernetesVersion, te var ( outputString []string - schemaLocations = getSchemaLocations(ctx, tempRepoPath) + schemaLocations = getSchemaLocations(ctx, cfg, tempRepoPath) ) log.Debug().Msgf("cache location: %s", vOpts.Cache) @@ -84,7 +84,7 @@ func ArgoCdAppValidate(ctx context.Context, appName, targetKubernetesVersion, te v, err := validator.New(schemaLocations, vOpts) if err != nil { log.Error().Err(err).Msg("could not create kubeconform validator") - return pkg.CheckResult{}, fmt.Errorf("could not create kubeconform validator: %v", err) + return msg.CheckResult{}, fmt.Errorf("could not create kubeconform validator: %v", err) } result := v.Validate("-", io.NopCloser(strings.NewReader(strings.Join(appManifests, "\n")))) var invalid, failedValidation bool @@ -109,7 +109,7 @@ func ArgoCdAppValidate(ctx context.Context, appName, targetKubernetesVersion, te } } - var cr pkg.CheckResult + var cr msg.CheckResult if invalid { cr.State = pkg.StateWarning } else if failedValidation { diff --git a/pkg/validate/validate_test.go b/pkg/validate/validate_test.go index d8d90183..59672869 100644 --- a/pkg/validate/validate_test.go +++ b/pkg/validate/validate_test.go @@ -11,11 +11,14 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/assert" + + "github.com/zapier/kubechecks/pkg/config" ) func TestDefaultGetSchemaLocations(t *testing.T) { ctx := context.TODO() - schemaLocations := getSchemaLocations(ctx, "/some/other/path") + cfg := config.ServerConfig{} + schemaLocations := getSchemaLocations(ctx, cfg, "/some/other/path") // default schema location is "./schemas" assert.Len(t, schemaLocations, 1) @@ -24,6 +27,7 @@ func TestDefaultGetSchemaLocations(t *testing.T) { func TestGetRemoteSchemaLocations(t *testing.T) { ctx := context.TODO() + cfg := config.ServerConfig{} if os.Getenv("CI") == "" { t.Skip("Skipping testing. Only for CI environments") @@ -35,7 +39,7 @@ func TestGetRemoteSchemaLocations(t *testing.T) { // t.Setenv("KUBECHECKS_SCHEMAS_LOCATION", fixture.URL) // doesn't work because viper needs to initialize from root, which doesn't happen viper.Set("schemas-location", []string{fixture.URL}) - schemaLocations := getSchemaLocations(ctx, "/some/other/path") + schemaLocations := getSchemaLocations(ctx, cfg, "/some/other/path") hasTmpDirPrefix := strings.HasPrefix(schemaLocations[0], "/tmp/schemas") assert.Equal(t, hasTmpDirPrefix, true, "invalid schemas location. Schema location should have prefix /tmp/schemas but has %s", schemaLocations[0]) } diff --git a/pkg/vcs/client.go b/pkg/vcs/client.go index fb2e65e8..dfd9a27f 100644 --- a/pkg/vcs/client.go +++ b/pkg/vcs/client.go @@ -1,12 +1,7 @@ package vcs import ( - "context" "errors" - "net/http" - - "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/pkg/repo" ) const ( @@ -19,37 +14,3 @@ var ( ErrInvalidType = errors.New("invalid event type") ErrHookNotFound = errors.New("webhook not found") ) - -type WebHookConfig struct { - Url string - SecretKey string - Events []string -} - -// Client represents a VCS client -type Client interface { - // PostMessage takes in project name in form "owner/repo" (ie zapier/kubechecks), the PR/MR id, and the actual message - PostMessage(context.Context, *repo.Repo, int, string) *pkg.Message - // UpdateMessage update a message with new content - UpdateMessage(context.Context, *pkg.Message, string) error - // VerifyHook validates a webhook secret and return the body; must be called even if no secret - VerifyHook(*http.Request, string) ([]byte, error) - // ParseHook parses webook payload for valid events - ParseHook(*http.Request, []byte) (*repo.Repo, error) - // CommitStatus sets a status for a specific commit on the remote VCS - CommitStatus(context.Context, *repo.Repo, pkg.CommitState) error - // GetHookByUrl gets a webhook by url - GetHookByUrl(ctx context.Context, repoName, webhookUrl string) (*WebHookConfig, error) - // CreateHook creates a webhook that points at kubechecks - CreateHook(ctx context.Context, repoName, webhookUrl, webhookSecret string) error - // GetName returns the VCS client name (e.g. "github" or "gitlab") - GetName() string - // TidyOutdatedComments either by hiding or deleting them - TidyOutdatedComments(context.Context, *repo.Repo) error - // LoadHook creates an EventRequest from the ID of an actual request - LoadHook(ctx context.Context, repoAndId string) (*repo.Repo, error) - - Username() string - Email() string - ToEmoji(pkg.CommitState) string -} diff --git a/pkg/vcs/github_client/client.go b/pkg/vcs/github_client/client.go index 597224c1..c9f60cdd 100644 --- a/pkg/vcs/github_client/client.go +++ b/pkg/vcs/github_client/client.go @@ -12,26 +12,24 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/shurcooL/githubv4" - "github.com/spf13/viper" "golang.org/x/oauth2" "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/config" "github.com/zapier/kubechecks/pkg/vcs" ) type Client struct { - v4Client *githubv4.Client - username, email string + shurcoolClient *githubv4.Client + googleClient *github.Client + cfg config.ServerConfig - *github.Client + username, email string } -var _ vcs.Client = new(Client) - // CreateGithubClient creates a new GitHub client using the auth token provided. We // can't validate the token at this point, so if it exists we assume it works -func CreateGithubClient() (*Client, error) { +func CreateGithubClient(cfg config.ServerConfig) (*Client, error) { var ( err error googleClient *github.Client @@ -39,7 +37,7 @@ func CreateGithubClient() (*Client, error) { ) // Initialize the GitLab client with access token - t := viper.GetString("vcs-token") + t := cfg.VcsToken if t == "" { log.Fatal().Msg("github token needs to be set") } @@ -50,7 +48,7 @@ func CreateGithubClient() (*Client, error) { ) tc := oauth2.NewClient(ctx, ts) - githubUrl := viper.GetString("vcs-base-url") + githubUrl := cfg.VcsBaseUrl if githubUrl == "" { googleClient = github.NewClient(tc) // If this has failed, we'll catch it on first call @@ -70,8 +68,9 @@ func CreateGithubClient() (*Client, error) { } client := &Client{ - Client: googleClient, - v4Client: shurcoolClient, + cfg: cfg, + googleClient: googleClient, + shurcoolClient: shurcoolClient, } if user != nil { if user.Login != nil { @@ -108,7 +107,7 @@ func (c *Client) VerifyHook(r *http.Request, secret string) ([]byte, error) { } } -func (c *Client) ParseHook(r *http.Request, request []byte) (*repo.Repo, error) { +func (c *Client) ParseHook(r *http.Request, request []byte) (*vcs.Repo, error) { payload, err := github.ParseWebHook(github.WebHookType(r), request) if err != nil { return nil, err @@ -130,13 +129,13 @@ func (c *Client) ParseHook(r *http.Request, request []byte) (*repo.Repo, error) } } -func (c *Client) buildRepoFromEvent(event *github.PullRequestEvent) *repo.Repo { +func (c *Client) buildRepoFromEvent(event *github.PullRequestEvent) *vcs.Repo { var labels []string for _, label := range event.PullRequest.Labels { labels = append(labels, label.GetName()) } - return &repo.Repo{ + return &vcs.Repo{ BaseRef: *event.PullRequest.Base.Ref, HeadRef: *event.PullRequest.Head.Ref, DefaultBranch: *event.Repo.DefaultBranch, @@ -149,6 +148,8 @@ func (c *Client) buildRepoFromEvent(event *github.PullRequestEvent) *repo.Repo { Username: c.username, Email: c.email, Labels: labels, + + Config: c.cfg, } } @@ -168,9 +169,9 @@ func toGithubCommitStatus(state pkg.CommitState) *string { return pkg.Pointer("failure") } -func (c *Client) CommitStatus(ctx context.Context, repo *repo.Repo, status pkg.CommitState) error { +func (c *Client) CommitStatus(ctx context.Context, repo *vcs.Repo, status pkg.CommitState) error { log.Info().Str("repo", repo.Name).Str("sha", repo.SHA).Str("status", status.BareString()).Msg("setting Github commit status") - repoStatus, _, err := c.Repositories.CreateStatus(ctx, repo.Owner, repo.Name, repo.SHA, &github.RepoStatus{ + repoStatus, _, err := c.googleClient.Repositories.CreateStatus(ctx, repo.Owner, repo.Name, repo.SHA, &github.RepoStatus{ State: toGithubCommitStatus(status), Description: pkg.Pointer(status.BareString()), ID: pkg.Pointer(int64(repo.CheckID)), @@ -199,7 +200,7 @@ func parseRepo(cloneUrl string) (string, string) { func (c *Client) GetHookByUrl(ctx context.Context, ownerAndRepoName, webhookUrl string) (*vcs.WebHookConfig, error) { owner, repoName := parseRepo(ownerAndRepoName) - items, _, err := c.Repositories.ListHooks(ctx, owner, repoName, nil) + items, _, err := c.googleClient.Repositories.ListHooks(ctx, owner, repoName, nil) if err != nil { return nil, errors.Wrap(err, "failed to list hooks") } @@ -218,7 +219,7 @@ func (c *Client) GetHookByUrl(ctx context.Context, ownerAndRepoName, webhookUrl func (c *Client) CreateHook(ctx context.Context, ownerAndRepoName, webhookUrl, webhookSecret string) error { owner, repoName := parseRepo(ownerAndRepoName) - _, _, err := c.Repositories.CreateHook(ctx, owner, repoName, &github.Hook{ + _, _, err := c.googleClient.Repositories.CreateHook(ctx, owner, repoName, &github.Hook{ Active: pkg.Pointer(true), Config: map[string]interface{}{ "content_type": "json", @@ -240,7 +241,7 @@ func (c *Client) CreateHook(ctx context.Context, ownerAndRepoName, webhookUrl, w var rePullRequest = regexp.MustCompile(`(.*)/(.*)#(\d+)`) -func (c *Client) LoadHook(ctx context.Context, id string) (*repo.Repo, error) { +func (c *Client) LoadHook(ctx context.Context, id string) (*vcs.Repo, error) { m := rePullRequest.FindStringSubmatch(id) if len(m) != 4 { return nil, errors.New("must be in format OWNER/REPO#PR") @@ -253,12 +254,12 @@ func (c *Client) LoadHook(ctx context.Context, id string) (*repo.Repo, error) { return nil, errors.Wrap(err, "failed to parse int") } - repoInfo, _, err := c.Repositories.Get(ctx, ownerName, repoName) + repoInfo, _, err := c.googleClient.Repositories.Get(ctx, ownerName, repoName) if err != nil { return nil, errors.Wrap(err, "failed to get repo") } - pullRequest, _, err := c.PullRequests.Get(ctx, ownerName, repoName, int(prNumber)) + pullRequest, _, err := c.googleClient.PullRequests.Get(ctx, ownerName, repoName, int(prNumber)) if err != nil { return nil, errors.Wrap(err, "failed to get pull request") } @@ -302,7 +303,7 @@ func (c *Client) LoadHook(ctx context.Context, id string) (*repo.Repo, error) { userEmail = "kubechecks@github.com" } - return &repo.Repo{ + return &vcs.Repo{ BaseRef: baseRef, HeadRef: headRef, DefaultBranch: unPtr(repoInfo.DefaultBranch), @@ -315,6 +316,8 @@ func (c *Client) LoadHook(ctx context.Context, id string) (*repo.Repo, error) { Username: userName, Email: userEmail, Labels: labels, + + Config: c.cfg, }, nil } diff --git a/pkg/vcs/github_client/message.go b/pkg/vcs/github_client/message.go index 3d3a3ad3..cb3f5c96 100644 --- a/pkg/vcs/github_client/message.go +++ b/pkg/vcs/github_client/message.go @@ -8,32 +8,32 @@ import ( "github.com/google/go-github/v53/github" "github.com/rs/zerolog/log" "github.com/shurcooL/githubv4" - "github.com/spf13/viper" "go.opentelemetry.io/otel" "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/msg" + "github.com/zapier/kubechecks/pkg/vcs" "github.com/zapier/kubechecks/telemetry" ) const MaxCommentLength = 64 * 1024 -func (c *Client) PostMessage(ctx context.Context, repo *repo.Repo, prID int, msg string) *pkg.Message { +func (c *Client) PostMessage(ctx context.Context, repo *vcs.Repo, prID int, message string) *msg.Message { _, span := otel.Tracer("Kubechecks").Start(ctx, "PostMessageToMergeRequest") defer span.End() - if len(msg) > MaxCommentLength { - log.Warn().Int("original_length", len(msg)).Msg("trimming the comment size") - msg = msg[:MaxCommentLength] + if len(message) > MaxCommentLength { + log.Warn().Int("original_length", len(message)).Msg("trimming the comment size") + message = message[:MaxCommentLength] } log.Debug().Msgf("Posting message to PR %d in repo %s", prID, repo.FullName) - comment, _, err := c.Issues.CreateComment( + comment, _, err := c.googleClient.Issues.CreateComment( ctx, repo.Owner, repo.Name, prID, - &github.IssueComment{Body: &msg}, + &github.IssueComment{Body: &message}, ) if err != nil { @@ -41,10 +41,10 @@ func (c *Client) PostMessage(ctx context.Context, repo *repo.Repo, prID int, msg log.Error().Err(err).Msg("could not post message to PR") } - return pkg.NewMessage(repo.FullName, prID, int(*comment.ID), c) + return msg.NewMessage(repo.FullName, prID, int(*comment.ID), c) } -func (c *Client) UpdateMessage(ctx context.Context, m *pkg.Message, msg string) error { +func (c *Client) UpdateMessage(ctx context.Context, m *msg.Message, msg string) error { _, span := otel.Tracer("Kubechecks").Start(ctx, "UpdateMessage") defer span.End() @@ -56,7 +56,7 @@ func (c *Client) UpdateMessage(ctx context.Context, m *pkg.Message, msg string) log.Info().Msgf("Updating message for PR %d in repo %s", m.CheckID, m.Name) repoNameComponents := strings.Split(m.Name, "/") - comment, resp, err := c.Issues.EditComment( + comment, resp, err := c.googleClient.Issues.EditComment( ctx, repoNameComponents[0], repoNameComponents[1], @@ -79,7 +79,7 @@ func (c *Client) UpdateMessage(ctx context.Context, m *pkg.Message, msg string) // Pull all comments for the specified PR, and delete any comments that already exist from the bot // This is different from updating an existing message, as this will delete comments from previous runs of the bot // Whereas updates occur mid-execution -func (c *Client) pruneOldComments(ctx context.Context, repo *repo.Repo, comments []*github.IssueComment) error { +func (c *Client) pruneOldComments(ctx context.Context, repo *vcs.Repo, comments []*github.IssueComment) error { _, span := otel.Tracer("Kubechecks").Start(ctx, "pruneOldComments") defer span.End() @@ -87,7 +87,7 @@ func (c *Client) pruneOldComments(ctx context.Context, repo *repo.Repo, comments for _, comment := range comments { if strings.EqualFold(comment.GetUser().GetLogin(), c.username) { - _, err := c.Issues.DeleteComment(ctx, repo.Owner, repo.Name, *comment.ID) + _, err := c.googleClient.Issues.DeleteComment(ctx, repo.Owner, repo.Name, *comment.ID) if err != nil { return fmt.Errorf("failed to delete comment: %w", err) } @@ -97,7 +97,7 @@ func (c *Client) pruneOldComments(ctx context.Context, repo *repo.Repo, comments return nil } -func (c *Client) hideOutdatedMessages(ctx context.Context, repo *repo.Repo, comments []*github.IssueComment) error { +func (c *Client) hideOutdatedMessages(ctx context.Context, repo *vcs.Repo, comments []*github.IssueComment) error { _, span := otel.Tracer("Kubechecks").Start(ctx, "hideOutdatedComments") defer span.End() @@ -120,7 +120,7 @@ func (c *Client) hideOutdatedMessages(ctx context.Context, repo *repo.Repo, comm Classifier: githubv4.ReportedContentClassifiersOutdated, SubjectID: comment.GetNodeID(), } - if err := c.v4Client.Mutate(ctx, &m, input, nil); err != nil { + if err := c.shurcoolClient.Mutate(ctx, &m, input, nil); err != nil { return fmt.Errorf("minimize comment %s: %w", comment.GetNodeID(), err) } } @@ -130,7 +130,7 @@ func (c *Client) hideOutdatedMessages(ctx context.Context, repo *repo.Repo, comm } -func (c *Client) TidyOutdatedComments(ctx context.Context, repo *repo.Repo) error { +func (c *Client) TidyOutdatedComments(ctx context.Context, repo *vcs.Repo) error { _, span := otel.Tracer("Kubechecks").Start(ctx, "TidyOutdatedComments") defer span.End() @@ -138,7 +138,7 @@ func (c *Client) TidyOutdatedComments(ctx context.Context, repo *repo.Repo) erro nextPage := 0 for { - comments, resp, err := c.Issues.ListComments(ctx, repo.Owner, repo.Name, repo.CheckID, &github.IssueListCommentsOptions{ + comments, resp, err := c.googleClient.Issues.ListComments(ctx, repo.Owner, repo.Name, repo.CheckID, &github.IssueListCommentsOptions{ Sort: pkg.Pointer("created"), Direction: pkg.Pointer("asc"), ListOptions: github.ListOptions{Page: nextPage}, @@ -154,7 +154,7 @@ func (c *Client) TidyOutdatedComments(ctx context.Context, repo *repo.Repo) erro nextPage = resp.NextPage } - if strings.ToLower(viper.GetString("tidy-outdated-comments-mode")) == "delete" { + if strings.ToLower(c.cfg.TidyOutdatedCommentsMode) == "delete" { return c.pruneOldComments(ctx, repo, allComments) } return c.hideOutdatedMessages(ctx, repo, allComments) diff --git a/pkg/vcs/gitlab_client/client.go b/pkg/vcs/gitlab_client/client.go index 6ee5e022..21e45f58 100644 --- a/pkg/vcs/gitlab_client/client.go +++ b/pkg/vcs/gitlab_client/client.go @@ -11,28 +11,26 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/spf13/viper" giturls "github.com/whilp/git-urls" "github.com/xanzy/go-gitlab" "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/config" "github.com/zapier/kubechecks/pkg/vcs" ) const GitlabTokenHeader = "X-Gitlab-Token" type Client struct { - *gitlab.Client + c *gitlab.Client + cfg config.ServerConfig username, email string } -var _ vcs.Client = new(Client) - -func CreateGitlabClient() (*Client, error) { +func CreateGitlabClient(cfg config.ServerConfig) (*Client, error) { // Initialize the GitLab client with access token - gitlabToken := viper.GetString("vcs-token") + gitlabToken := cfg.VcsToken if gitlabToken == "" { log.Fatal().Msg("gitlab token needs to be set") } @@ -40,7 +38,7 @@ func CreateGitlabClient() (*Client, error) { var gitlabOptions []gitlab.ClientOptionFunc - gitlabUrl := viper.GetString("vcs-base-url") + gitlabUrl := cfg.VcsBaseUrl if gitlabUrl != "" { gitlabOptions = append(gitlabOptions, gitlab.WithBaseURL(gitlabUrl)) } @@ -55,7 +53,12 @@ func CreateGitlabClient() (*Client, error) { return nil, errors.Wrap(err, "failed to get current user") } - client := &Client{Client: c, username: user.Username, email: user.Email} + client := &Client{ + c: c, + cfg: cfg, + username: user.Username, + email: user.Email, + } if client.username == "" { client.username = vcs.DefaultVcsUsername } @@ -71,7 +74,7 @@ func (c *Client) GetName() string { return "gitlab" } -// Each client has a different way of verifying their payloads; return an err if this isnt valid +// VerifyHook returns an err if the webhook isn't valid func (c *Client) VerifyHook(r *http.Request, secret string) ([]byte, error) { // If we have a secret, and the secret doesn't match, return an error if secret != "" && secret != r.Header.Get(GitlabTokenHeader) { @@ -84,7 +87,7 @@ func (c *Client) VerifyHook(r *http.Request, secret string) ([]byte, error) { } // ParseHook parses and validates a webhook event; return an err if this isn't valid -func (c *Client) ParseHook(r *http.Request, request []byte) (*repo.Repo, error) { +func (c *Client) ParseHook(r *http.Request, request []byte) (*vcs.Repo, error) { eventRequest, err := gitlab.ParseHook(gitlab.HookEventType(r), request) if err != nil { return nil, err @@ -128,7 +131,7 @@ func (c *Client) GetHookByUrl(ctx context.Context, repoName, webhookUrl string) if err != nil { return nil, errors.Wrap(err, "failed to parse repo url") } - webhooks, _, err := c.Client.Projects.ListProjectHooks(pid, nil) + webhooks, _, err := c.c.Projects.ListProjectHooks(pid, nil) if err != nil { return nil, errors.Wrap(err, "failed to list project webhooks") } @@ -156,7 +159,7 @@ func (c *Client) CreateHook(ctx context.Context, repoName, webhookUrl, webhookSe return errors.Wrap(err, "failed to parse repo name") } - _, _, err = c.Client.Projects.AddProjectHook(pid, &gitlab.AddProjectHookOptions{ + _, _, err = c.c.Projects.AddProjectHook(pid, &gitlab.AddProjectHookOptions{ URL: pkg.Pointer(webhookUrl), MergeRequestsEvents: pkg.Pointer(true), Token: pkg.Pointer(webhookSecret), @@ -171,7 +174,7 @@ func (c *Client) CreateHook(ctx context.Context, repoName, webhookUrl, webhookSe var reMergeRequest = regexp.MustCompile(`(.*)!(\d+)`) -func (c *Client) LoadHook(ctx context.Context, id string) (*repo.Repo, error) { +func (c *Client) LoadHook(ctx context.Context, id string) (*vcs.Repo, error) { m := reMergeRequest.FindStringSubmatch(id) if len(m) != 3 { return nil, errors.New("must be in format REPOPATH!MR") @@ -183,17 +186,17 @@ func (c *Client) LoadHook(ctx context.Context, id string) (*repo.Repo, error) { return nil, errors.Wrap(err, "failed to parse merge request number") } - project, _, err := c.Projects.GetProject(repoPath, nil) + project, _, err := c.c.Projects.GetProject(repoPath, nil) if err != nil { return nil, errors.Wrapf(err, "failed to get project '%s'", repoPath) } - mergeRequest, _, err := c.MergeRequests.GetMergeRequest(repoPath, int(mrNumber), nil) + mergeRequest, _, err := c.c.MergeRequests.GetMergeRequest(repoPath, int(mrNumber), nil) if err != nil { return nil, errors.Wrapf(err, "failed to get merge request '%d' in project '%s'", mrNumber, repoPath) } - return &repo.Repo{ + return &vcs.Repo{ BaseRef: mergeRequest.TargetBranch, HeadRef: mergeRequest.SourceBranch, DefaultBranch: project.DefaultBranch, @@ -208,17 +211,19 @@ func (c *Client) LoadHook(ctx context.Context, id string) (*repo.Repo, error) { Username: c.username, Email: c.email, Labels: mergeRequest.Labels, + + Config: c.cfg, }, nil } -func (c *Client) buildRepoFromEvent(event *gitlab.MergeEvent) *repo.Repo { +func (c *Client) buildRepoFromEvent(event *gitlab.MergeEvent) *vcs.Repo { // Convert all labels from this MR to a string array of label names var labels []string for _, label := range event.Labels { labels = append(labels, label.Title) } - return &repo.Repo{ + return &vcs.Repo{ BaseRef: event.ObjectAttributes.TargetBranch, HeadRef: event.ObjectAttributes.SourceBranch, DefaultBranch: event.Project.DefaultBranch, @@ -230,5 +235,7 @@ func (c *Client) buildRepoFromEvent(event *gitlab.MergeEvent) *repo.Repo { Username: c.username, Email: c.email, Labels: labels, + + Config: c.cfg, } } diff --git a/pkg/vcs/gitlab_client/merge.go b/pkg/vcs/gitlab_client/merge.go index 5fc328f7..087d08d0 100644 --- a/pkg/vcs/gitlab_client/merge.go +++ b/pkg/vcs/gitlab_client/merge.go @@ -27,7 +27,7 @@ func (c *Client) GetMergeChanges(ctx context.Context, projectId int, mergeReqId defer span.End() var changes []*Changes - diffs, _, err := c.MergeRequests.ListMergeRequestDiffs(projectId, mergeReqId, &gitlab.ListMergeRequestDiffsOptions{}) + diffs, _, err := c.c.MergeRequests.ListMergeRequestDiffs(projectId, mergeReqId, &gitlab.ListMergeRequestDiffsOptions{}) if err != nil { telemetry.SetError(span, err, "Get MergeRequest Changes") return changes, err diff --git a/pkg/vcs/gitlab_client/message.go b/pkg/vcs/gitlab_client/message.go index 16a20b29..492a9625 100644 --- a/pkg/vcs/gitlab_client/message.go +++ b/pkg/vcs/gitlab_client/message.go @@ -6,37 +6,37 @@ import ( "strings" "github.com/rs/zerolog/log" - "github.com/spf13/viper" "github.com/xanzy/go-gitlab" "go.opentelemetry.io/otel" "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/msg" + "github.com/zapier/kubechecks/pkg/vcs" "github.com/zapier/kubechecks/telemetry" ) const MaxCommentLength = 1_000_000 -func (c *Client) PostMessage(ctx context.Context, repo *repo.Repo, mergeRequestID int, msg string) *pkg.Message { +func (c *Client) PostMessage(ctx context.Context, repo *vcs.Repo, mergeRequestID int, message string) *msg.Message { _, span := otel.Tracer("Kubechecks").Start(ctx, "PostMessageToMergeRequest") defer span.End() - if len(msg) > MaxCommentLength { - log.Warn().Int("original_length", len(msg)).Msg("trimming the comment size") - msg = msg[:MaxCommentLength] + if len(message) > MaxCommentLength { + log.Warn().Int("original_length", len(message)).Msg("trimming the comment size") + message = message[:MaxCommentLength] } - n, _, err := c.Notes.CreateMergeRequestNote( + n, _, err := c.c.Notes.CreateMergeRequestNote( repo.FullName, mergeRequestID, &gitlab.CreateMergeRequestNoteOptions{ - Body: pkg.Pointer(msg), + Body: pkg.Pointer(message), }) if err != nil { telemetry.SetError(span, err, "Create Merge Request Note") log.Error().Err(err).Msg("could not post message to MR") } - return pkg.NewMessage(repo.FullName, mergeRequestID, n.ID, c) + return msg.NewMessage(repo.FullName, mergeRequestID, n.ID, c) } func (c *Client) hideOutdatedMessages(ctx context.Context, projectName string, mergeRequestID int, notes []*gitlab.Note) error { @@ -71,7 +71,7 @@ func (c *Client) hideOutdatedMessages(ctx context.Context, projectName string, m log.Debug().Str("projectName", projectName).Int("mr", mergeRequestID).Msgf("Updating comment %d as outdated", note.ID) - _, _, err := c.Notes.UpdateMergeRequestNote(projectName, mergeRequestID, note.ID, &gitlab.UpdateMergeRequestNoteOptions{ + _, _, err := c.c.Notes.UpdateMergeRequestNote(projectName, mergeRequestID, note.ID, &gitlab.UpdateMergeRequestNoteOptions{ Body: &newBody, }) @@ -79,21 +79,21 @@ func (c *Client) hideOutdatedMessages(ctx context.Context, projectName string, m telemetry.SetError(span, err, "Hide Existing Merge Request Check Note") return fmt.Errorf("could not hide note %d for merge request: %w", note.ID, err) } - } + return nil } -func (c *Client) UpdateMessage(ctx context.Context, m *pkg.Message, msg string) error { +func (c *Client) UpdateMessage(ctx context.Context, m *msg.Message, message string) error { log.Debug().Msgf("Updating message %d for %s", m.NoteID, m.Name) - if len(msg) > MaxCommentLength { - log.Warn().Int("original_length", len(msg)).Msg("trimming the comment size") - msg = msg[:MaxCommentLength] + if len(message) > MaxCommentLength { + log.Warn().Int("original_length", len(message)).Msg("trimming the comment size") + message = message[:MaxCommentLength] } - n, _, err := c.Notes.UpdateMergeRequestNote(m.Name, m.CheckID, m.NoteID, &gitlab.UpdateMergeRequestNoteOptions{ - Body: pkg.Pointer(msg), + n, _, err := c.c.Notes.UpdateMergeRequestNote(m.Name, m.CheckID, m.NoteID, &gitlab.UpdateMergeRequestNoteOptions{ + Body: pkg.Pointer(message), }) if err != nil { @@ -116,7 +116,7 @@ func (c *Client) pruneOldComments(ctx context.Context, projectName string, mrID for _, note := range notes { if note.Author.Username == c.username { log.Debug().Int("mr", mrID).Int("note", note.ID).Msg("deleting old comment") - _, err := c.Notes.DeleteMergeRequestNote(projectName, mrID, note.ID) + _, err := c.c.Notes.DeleteMergeRequestNote(projectName, mrID, note.ID) if err != nil { telemetry.SetError(span, err, "Prune Old Comments") return fmt.Errorf("could not delete old comment: %w", err) @@ -126,7 +126,7 @@ func (c *Client) pruneOldComments(ctx context.Context, projectName string, mrID return nil } -func (c *Client) TidyOutdatedComments(ctx context.Context, repo *repo.Repo) error { +func (c *Client) TidyOutdatedComments(ctx context.Context, repo *vcs.Repo) error { _, span := otel.Tracer("Kubechecks").Start(ctx, "TidyOutdatedMessages") defer span.End() @@ -137,7 +137,7 @@ func (c *Client) TidyOutdatedComments(ctx context.Context, repo *repo.Repo) erro for { // list merge request notes - notes, resp, err := c.Notes.ListMergeRequestNotes(repo.FullName, repo.CheckID, &gitlab.ListMergeRequestNotesOptions{ + notes, resp, err := c.c.Notes.ListMergeRequestNotes(repo.FullName, repo.CheckID, &gitlab.ListMergeRequestNotesOptions{ Sort: pkg.Pointer("asc"), OrderBy: pkg.Pointer("created_at"), ListOptions: gitlab.ListOptions{ @@ -156,7 +156,7 @@ func (c *Client) TidyOutdatedComments(ctx context.Context, repo *repo.Repo) erro nextPage = resp.NextPage } - if strings.ToLower(viper.GetString("tidy-outdated-comments-mode")) == "delete" { + if strings.ToLower(c.cfg.TidyOutdatedCommentsMode) == "delete" { return c.pruneOldComments(ctx, repo.FullName, repo.CheckID, allNotes) } return c.hideOutdatedMessages(ctx, repo.FullName, repo.CheckID, allNotes) diff --git a/pkg/vcs/gitlab_client/pipeline.go b/pkg/vcs/gitlab_client/pipeline.go index 41e26bb4..40bdd7e5 100644 --- a/pkg/vcs/gitlab_client/pipeline.go +++ b/pkg/vcs/gitlab_client/pipeline.go @@ -8,7 +8,7 @@ import ( ) func (c *Client) GetPipelinesForCommit(projectName string, commitSHA string) ([]*gitlab.PipelineInfo, error) { - pipelines, _, err := c.Pipelines.ListProjectPipelines(projectName, &gitlab.ListProjectPipelinesOptions{ + pipelines, _, err := c.c.Pipelines.ListProjectPipelines(projectName, &gitlab.ListProjectPipelinesOptions{ SHA: pkg.Pointer(commitSHA), }) if err != nil { diff --git a/pkg/vcs/gitlab_client/project.go b/pkg/vcs/gitlab_client/project.go index 67489b12..930254fb 100644 --- a/pkg/vcs/gitlab_client/project.go +++ b/pkg/vcs/gitlab_client/project.go @@ -12,13 +12,13 @@ import ( "github.com/zapier/kubechecks/pkg/repo_config" ) -// GetProjectByIDorName gets a project by the given Project Name or ID +// GetProjectByID gets a project by the given Project Name or ID func (c *Client) GetProjectByID(project int) (*gitlab.Project, error) { var proj *gitlab.Project err := backoff.Retry(func() error { var err error var resp *gitlab.Response - proj, resp, err = c.Projects.GetProject(project, nil) + proj, resp, err = c.c.Projects.GetProject(project, nil) return checkReturnForBackoff(resp, err) }, getBackOff()) return proj, err @@ -30,7 +30,7 @@ func (c *Client) GetRepoConfigFile(ctx context.Context, projectId int, mergeReqI // check MR branch for _, file := range repo_config.RepoConfigFilenameVariations() { - b, _, err := c.RepositoryFiles.GetRawFile( + b, _, err := c.c.RepositoryFiles.GetRawFile( projectId, file, &gitlab.GetRawFileOptions{Ref: pkg.Pointer("HEAD")}, diff --git a/pkg/vcs/gitlab_client/status.go b/pkg/vcs/gitlab_client/status.go index fa4680ea..7f6a3874 100644 --- a/pkg/vcs/gitlab_client/status.go +++ b/pkg/vcs/gitlab_client/status.go @@ -12,14 +12,14 @@ import ( "github.com/xanzy/go-gitlab" "github.com/zapier/kubechecks/pkg" - "github.com/zapier/kubechecks/pkg/repo" + "github.com/zapier/kubechecks/pkg/vcs" ) const GitlabCommitStatusContext = "kubechecks" var errNoPipelineStatus = errors.New("nil pipeline status") -func (c *Client) CommitStatus(ctx context.Context, repo *repo.Repo, state pkg.CommitState) error { +func (c *Client) CommitStatus(ctx context.Context, repo *vcs.Repo, state pkg.CommitState) error { description := fmt.Sprintf("%s %s", state.BareString(), c.ToEmoji(state)) status := &gitlab.SetCommitStatusOptions{ @@ -78,7 +78,7 @@ func convertState(state pkg.CommitState) gitlab.BuildStateValue { } func (c *Client) setCommitStatus(projectWithNS string, commitSHA string, status *gitlab.SetCommitStatusOptions) (*gitlab.CommitStatus, error) { - commitStatus, _, err := c.Commits.SetCommitStatus(projectWithNS, commitSHA, status) + commitStatus, _, err := c.c.Commits.SetCommitStatus(projectWithNS, commitSHA, status) return commitStatus, err } diff --git a/pkg/repo/repo.go b/pkg/vcs/repo.go similarity index 88% rename from pkg/repo/repo.go rename to pkg/vcs/repo.go index befee731..fb31f530 100644 --- a/pkg/repo/repo.go +++ b/pkg/vcs/repo.go @@ -1,4 +1,4 @@ -package repo +package vcs import ( "bufio" @@ -14,11 +14,11 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/spf13/viper" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "github.com/zapier/kubechecks/pkg/config" "github.com/zapier/kubechecks/telemetry" ) @@ -38,6 +38,8 @@ type Repo struct { Username string // Username of auth'd client Email string // Email of auth'd client Labels []string // Labels associated with the MR/PR + + Config config.ServerConfig } func (r *Repo) CloneRepoLocal(ctx context.Context, repoDir string) error { @@ -206,13 +208,16 @@ func walk(s string, d fs.DirEntry, err error) error { } func (r *Repo) execCommand(name string, args ...string) *exec.Cmd { - cmd := execCommand(name, args...) + cmd := execCommand(r.Config, name, args...) cmd.Dir = r.RepoDir return cmd } -func censorVcsToken(v *viper.Viper, args []string) []string { - vcsToken := v.GetString("vcs-token") +func censorVcsToken(cfg config.ServerConfig, args []string) []string { + vcsToken := cfg.VcsToken + if len(vcsToken) == 0 { + return args + } var argsToLog []string for _, arg := range args { @@ -221,8 +226,8 @@ func censorVcsToken(v *viper.Viper, args []string) []string { return argsToLog } -func execCommand(name string, args ...string) *exec.Cmd { - argsToLog := censorVcsToken(viper.GetViper(), args) +func execCommand(cfg config.ServerConfig, name string, args ...string) *exec.Cmd { + argsToLog := censorVcsToken(cfg, args) log.Debug().Strs("args", argsToLog).Msg("building command") cmd := exec.Command(name, args...) @@ -230,20 +235,23 @@ func execCommand(name string, args ...string) *exec.Cmd { } // InitializeGitSettings ensures Git auth is set up for cloning -func InitializeGitSettings(username, email string) error { - cmd := execCommand("git", "config", "--global", "user.email", email) +func InitializeGitSettings(cfg config.ServerConfig, vcsClient VcsClient) error { + email := vcsClient.Email() + username := vcsClient.Username() + + cmd := execCommand(cfg, "git", "config", "--global", "user.email", email) err := cmd.Run() if err != nil { return errors.Wrap(err, "failed to set git email address") } - cmd = execCommand("git", "config", "--global", "user.name", username) + cmd = execCommand(cfg, "git", "config", "--global", "user.name", username) err = cmd.Run() if err != nil { return errors.Wrap(err, "failed to set git user name") } - cloneUrl, err := getCloneUrl(username, viper.GetViper()) + cloneUrl, err := getCloneUrl(username, cfg) if err != nil { return errors.Wrap(err, "failed to get clone url") } @@ -260,14 +268,14 @@ func InitializeGitSettings(username, email string) error { } defer outfile.Close() - cmd = execCommand("echo", cloneUrl) + cmd = execCommand(cfg, "echo", cloneUrl) cmd.Stdout = outfile err = cmd.Run() if err != nil { return errors.Wrap(err, "unable to set git credentials") } - cmd = execCommand("git", "config", "--global", "credential.helper", "store") + cmd = execCommand(cfg, "git", "config", "--global", "credential.helper", "store") err = cmd.Run() if err != nil { return errors.Wrap(err, "unable to set git credential usage") @@ -277,10 +285,10 @@ func InitializeGitSettings(username, email string) error { return nil } -func getCloneUrl(user string, cfg *viper.Viper) (string, error) { - vcsBaseUrl := cfg.GetString("vcs-base-url") - vcsType := cfg.GetString("vcs-type") - vcsToken := cfg.GetString("vcs-token") +func getCloneUrl(user string, cfg config.ServerConfig) (string, error) { + vcsBaseUrl := cfg.VcsBaseUrl + vcsType := cfg.VcsType + vcsToken := cfg.VcsToken var hostname, scheme string @@ -296,5 +304,6 @@ func getCloneUrl(user string, cfg *viper.Viper) (string, error) { hostname = parts.Host scheme = parts.Scheme } + return fmt.Sprintf("%s://%s:%s@%s", scheme, user, vcsToken, hostname), nil } diff --git a/pkg/repo/repo_test.go b/pkg/vcs/repo_test.go similarity index 69% rename from pkg/repo/repo_test.go rename to pkg/vcs/repo_test.go index 88aa1cb9..25f4fb50 100644 --- a/pkg/repo/repo_test.go +++ b/pkg/vcs/repo_test.go @@ -1,12 +1,13 @@ -package repo +package vcs import ( "fmt" "testing" - "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/zapier/kubechecks/pkg/config" ) func TestGetCloneUrl(t *testing.T) { @@ -51,15 +52,13 @@ func TestGetCloneUrl(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert.NotEqual(t, "", tc.vcsType) - v := viper.New() - v.Set("vcs-token", testToken) - v.Set("vcs-type", tc.vcsType) - - if tc.vcsBaseUrl != "" { - v.Set("vcs-base-url", tc.vcsBaseUrl) + cfg := config.ServerConfig{ + VcsToken: testToken, + VcsType: tc.vcsType, + VcsBaseUrl: tc.vcsBaseUrl, } - actual, err := getCloneUrl(testUser, v) + actual, err := getCloneUrl(testUser, cfg) require.NoError(t, err) expected := fmt.Sprintf(tc.expected, testUser, testToken) @@ -69,8 +68,13 @@ func TestGetCloneUrl(t *testing.T) { } func TestCensorVcsToken(t *testing.T) { - v := viper.New() - v.Set("vcs-token", "hre") - result := censorVcsToken(v, []string{"one", "two", "three"}) + cfg := config.ServerConfig{VcsToken: "hre"} + result := censorVcsToken(cfg, []string{"one", "two", "three"}) assert.Equal(t, []string{"one", "two", "t********e"}, result) } + +func TestCensorEmptyVcsToken(t *testing.T) { + cfg := config.ServerConfig{VcsToken: ""} + result := censorVcsToken(cfg, []string{"one", "two", "three"}) + assert.Equal(t, []string{"one", "two", "three"}, result) +} diff --git a/pkg/vcs/types.go b/pkg/vcs/types.go new file mode 100644 index 00000000..12e6b3ba --- /dev/null +++ b/pkg/vcs/types.go @@ -0,0 +1,43 @@ +package vcs + +import ( + "context" + "net/http" + + "github.com/zapier/kubechecks/pkg" + "github.com/zapier/kubechecks/pkg/msg" +) + +type WebHookConfig struct { + Url string + SecretKey string + Events []string +} + +// VcsClient represents a VCS client +type VcsClient interface { + // PostMessage takes in project name in form "owner/repo" (ie zapier/kubechecks), the PR/MR id, and the actual message + PostMessage(context.Context, *Repo, int, string) *msg.Message + // UpdateMessage update a message with new content + UpdateMessage(context.Context, *msg.Message, string) error + // VerifyHook validates a webhook secret and return the body; must be called even if no secret + VerifyHook(*http.Request, string) ([]byte, error) + // ParseHook parses webook payload for valid events + ParseHook(*http.Request, []byte) (*Repo, error) + // CommitStatus sets a status for a specific commit on the remote VCS + CommitStatus(context.Context, *Repo, pkg.CommitState) error + // GetHookByUrl gets a webhook by url + GetHookByUrl(ctx context.Context, repoName, webhookUrl string) (*WebHookConfig, error) + // CreateHook creates a webhook that points at kubechecks + CreateHook(ctx context.Context, repoName, webhookUrl, webhookSecret string) error + // GetName returns the VCS client name (e.g. "github" or "gitlab") + GetName() string + // TidyOutdatedComments either by hiding or deleting them + TidyOutdatedComments(context.Context, *Repo) error + // LoadHook creates an EventRequest from the ID of an actual request + LoadHook(ctx context.Context, repoAndId string) (*Repo, error) + + Username() string + Email() string + ToEmoji(pkg.CommitState) string +}