diff --git a/Earthfile b/Earthfile index f8ed5958..f1c2a07b 100644 --- a/Earthfile +++ b/Earthfile @@ -1,5 +1,7 @@ VERSION 0.7 +ARG --global USERARCH + test: BUILD +ci-golang BUILD +ci-helm @@ -22,11 +24,11 @@ release: BUILD +release-helm go-deps: - ARG GOLANG_VERSION="1.19.3" + ARG GOLANG_VERSION="1.21" ARG GOOS=linux - ARG GOARCH=amd64 + ARG GOARCH=$USERARCH - FROM --platform=linux/amd64 golang:$GOLANG_VERSION-bullseye + FROM --platform=linux/$USERARCH golang:$GOLANG_VERSION-bullseye ENV GO111MODULE=on ENV CGO_ENABLED=0 @@ -62,20 +64,26 @@ test-golang: build-binary: ARG GOOS=linux - ARG GOARCH=amd64 + ARG GOARCH=$USERARCH ARG VARIANT ARG --required GIT_TAG ARG --required GIT_COMMIT - FROM --platform=linux/amd64 +go-deps + FROM --platform=linux/$USERARCH +go-deps WORKDIR /src COPY . /src RUN GOARM=${VARIANT#v} go build -ldflags "-X github.com/zapier/kubechecks/pkg.GitCommit=$GIT_COMMIT -X github.com/zapier/kubechecks/pkg.GitTag=$GIT_TAG" -o kubechecks SAVE ARTIFACT kubechecks +build-debug-binary: + FROM --platform=linux/$USERARCH +go-deps + WORKDIR /src + COPY . /src + RUN go build -gcflags="all=-N -l" -ldflags "-X github.com/zapier/kubechecks/pkg.GitCommit=$GIT_COMMIT -X github.com/zapier/kubechecks/pkg.GitTag=$GIT_TAG" -o kubechecks + SAVE ARTIFACT kubechecks + docker: - ARG --required IMAGE_NAME ARG TARGETPLATFORM ARG TARGETARCH ARG TARGETVARIANT @@ -115,15 +123,15 @@ docker: VOLUME /app/policies VOLUME /app/schemas - COPY (+build-binary/kubechecks --platform=linux/amd64 --GOARCH=$TARGETARCH --VARIANT=$TARGETVARIANT) . + COPY (+build-binary/kubechecks --platform=linux/$USERARCH --GOARCH=$TARGETARCH --VARIANT=$TARGETVARIANT) . RUN ./kubechecks help CMD ["./kubechecks", "controller"] - + ARG --required IMAGE_NAME SAVE IMAGE --push $IMAGE_NAME dlv: - FROM golang:1.19 + FROM golang:1.21-bullseye RUN apt update && apt install -y ca-certificates curl git RUN go install github.com/go-delve/delve/cmd/dlv@latest @@ -131,13 +139,15 @@ dlv: SAVE ARTIFACT /go/bin/dlv docker-debug: - ARG IMAGE_NAME="kubechecks:debug" FROM +docker --GIT_TAG=debug --GIT_COMMIT=abcdef - COPY (+dlv/dlv --GOARCH=$GOARCH --VARIANT=$TARGETVARIANT) /usr/local/bin/dlv + COPY (+build-debug-binary/kubechecks --GOARCH=$USERARCH --VARIANT=$TARGETVARIANT) . + + COPY (+dlv/dlv --GOARCH=$USERARCH --VARIANT=$TARGETVARIANT) /usr/local/bin/dlv CMD ["/usr/local/bin/dlv", "--listen=:2345", "--api-version=2", "--headless=true", "--accept-multiclient", "exec", "--continue", "./kubechecks", "controller"] + ARG IMAGE_NAME="kubechecks:debug" SAVE IMAGE --push $IMAGE_NAME fmt-golang: @@ -150,13 +160,13 @@ fmt-golang: && ./hacks/exit-on-changed-files.sh lint-golang: - ARG STATICCHECK_VERSION="2023.1.3" + ARG STATICCHECK_VERSION="2023.1.6" FROM +go-deps # install staticcheck RUN FILE=staticcheck.tgz \ - && URL=https://github.com/dominikh/go-tools/releases/download/$STATICCHECK_VERSION/staticcheck_linux_amd64.tar.gz \ + && URL=https://github.com/dominikh/go-tools/releases/download/$STATICCHECK_VERSION/staticcheck_linux_$USERARCH.tar.gz \ && wget ${URL} \ --output-document ${FILE} \ && tar \ @@ -176,9 +186,9 @@ test-helm: FROM quay.io/helmpack/chart-testing:v${CHART_TESTING_VERSION} # install kubeconform - ARG KUBECONFORM_VERSION="0.5.0" + ARG KUBECONFORM_VERSION="0.6.4" RUN FILE=kubeconform.tgz \ - && URL=https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz \ + && URL=https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${USERARCH}.tar.gz \ && wget ${URL} \ --output-document ${FILE} \ && tar \ @@ -205,7 +215,7 @@ release-helm: ARG HELM_VERSION="3.8.1" RUN FILE=helm.tgz \ - && URL=https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz \ + && URL=https://get.helm.sh/helm-v${HELM_VERSION}-linux-${USERARCH}.tar.gz \ && wget ${URL} \ --output-document ${FILE} \ && tar \ diff --git a/Tiltfile b/Tiltfile index 9dfd27a2..19b787b3 100644 --- a/Tiltfile +++ b/Tiltfile @@ -45,7 +45,9 @@ deploy_ngrok(cfg) # ///////////////////////////////////////////////////////////////////////////// # Load ArgoCD Tiltfile -load('./localdev/argocd/Tiltfile', 'deploy_argo') +load('./localdev/argocd/Tiltfile', 'deploy_argo', 'delete_argocd_apps_on_tilt_down', 'force_argocd_cleanup_on_tilt_down') +# make sure apps get removed (cleanly) before ArgoCD is shutdown +delete_argocd_apps_on_tilt_down() deploy_argo() #load('./localdev/reloader/Tiltfile', 'deploy_reloader') @@ -135,32 +137,21 @@ test_go( ], ) -arch="arm64" if str(local("uname -m")).strip('\n') == "arm64" else "amd64" - earthly_build( context='.', target="+docker-debug", ref='kubechecks', image_arg='IMAGE_NAME', ignore='./dist', - extra_args=[ - '--GOARCH={}'.format(arch), - ] ) - cmd_button('loc:go mod tidy', argv=['go', 'mod', 'tidy'], resource='kubechecks', icon_name='move_up', text='go mod tidy', ) -cmd_button('generate-mocks', - argv=['go', 'generate', './...'], - resource='kubechecks', - icon_name='change_circle', - text='go generate', -) + cmd_button('restart-pod', argv=['kubectl', '-n', 'kubechecks', 'rollout', 'restart', 'deployment/kubechecks'], resource='kubechecks', @@ -173,13 +164,15 @@ k8s_yaml(helm( namespace='kubechecks', name='kubechecks', values='./localdev/kubechecks/values.yaml', - set=['deployment.env.KUBECHECKS_WEBHOOK_URL_BASE=' + get_ngrok_url(cfg), 'deployment.env.NGROK_URL=' + get_ngrok_url(cfg), - 'deployment.env.KUBECHECKS_ARGOCD_WEBHOOK_URL='+ get_ngrok_url(cfg) +'/argocd/api/webhook', - 'deployment.env.KUBECHECKS_ENABLE_CONFTEST=true', - 'deployment.env.KUBECHECKS_VCS_TYPE=' + cfg.get('vcs-type', 'gitlab'), - 'secrets.env.KUBECHECKS_VCS_TOKEN=' + (os.getenv('GITLAB_TOKEN') if 'gitlab' in cfg.get('vcs-type', 'gitlab') else os.getenv('GITHUB_TOKEN')), - 'secrets.env.KUBECHECKS_WEBHOOK_SECRET=' + (os.getenv('KUBECHECKS_WEBHOOK_SECRET') if os.getenv('KUBECHECKS_WEBHOOK_SECRET') != None else ""), - 'secrets.env.KUBECHECKS_OPENAI_API_TOKEN=' + (os.getenv('OPENAI_API_TOKEN') if os.getenv('OPENAI_API_TOKEN') != None else ""),], + set=[ + 'deployment.env[15].name=KUBECHECKS_WEBHOOK_URL_BASE', 'deployment.env[15].value=' + get_ngrok_url(cfg), + 'deployment.env[16].name=NGROK_URL', 'deployment.env[16].value=' + get_ngrok_url(cfg), + 'deployment.env[17].name=KUBECHECKS_ARGOCD_WEBHOOK_URL', 'deployment.env[17].value=' + get_ngrok_url(cfg) +'/argocd/api/webhook', + 'deployment.env[18].name=KUBECHECKS_VCS_TYPE', 'deployment.env[18].value=' + cfg.get('vcs-type', 'gitlab'), + 'secrets.env.KUBECHECKS_VCS_TOKEN=' + (os.getenv('GITLAB_TOKEN') if 'gitlab' in cfg.get('vcs-type', 'gitlab') else os.getenv('GITHUB_TOKEN')), + 'secrets.env.KUBECHECKS_WEBHOOK_SECRET=' + (os.getenv('KUBECHECKS_WEBHOOK_SECRET') if os.getenv('KUBECHECKS_WEBHOOK_SECRET') != None else ""), + 'secrets.env.KUBECHECKS_OPENAI_API_TOKEN=' + (os.getenv('OPENAI_API_TOKEN') if os.getenv('OPENAI_API_TOKEN') != None else ""), + ], )) k8s_resource( @@ -188,7 +181,9 @@ k8s_resource( resource_deps=[ # 'go-build', 'go-test', - 'k8s:namespace' + 'k8s:namespace', + 'argocd', + 'argocd-crds', ], labels=["kubechecks"] ) @@ -218,3 +213,6 @@ install_test_apps(cfg) load("localdev/test_appsets/Tiltfile", "install_test_appsets") install_test_appsets(cfg) + + +force_argocd_cleanup_on_tilt_down() \ No newline at end of file diff --git a/charts/kubechecks/Chart.yaml b/charts/kubechecks/Chart.yaml index ea21292b..8d818044 100644 --- a/charts/kubechecks/Chart.yaml +++ b/charts/kubechecks/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: kubechecks description: A Helm chart for kubechecks -version: 0.4.1 +version: 0.4.2 appVersion: "1.3.3" type: application maintainers: diff --git a/charts/kubechecks/templates/clusterrole.yaml b/charts/kubechecks/templates/clusterrole.yaml new file mode 100644 index 00000000..783ac613 --- /dev/null +++ b/charts/kubechecks/templates/clusterrole.yaml @@ -0,0 +1,8 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubechecks.fullname" . }} +rules: + - apiGroups: ['*'] + resources: ['applications', 'appprojects', 'services'] + verbs: ['*'] diff --git a/charts/kubechecks/templates/clusterrolebinding.yaml b/charts/kubechecks/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..da58c7d8 --- /dev/null +++ b/charts/kubechecks/templates/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubechecks.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubechecks.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "kubechecks.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/cmd/controller_cmd.go b/cmd/controller_cmd.go index 886c9831..03f3899f 100644 --- a/cmd/controller_cmd.go +++ b/cmd/controller_cmd.go @@ -44,9 +44,9 @@ var ControllerCmd = &cobra.Command{ } fmt.Println("Starting KubeChecks:", pkg.GitTag, pkg.GitCommit) - server := server.NewServer(&cfg) - ctx := context.Background() + server := server.NewServer(ctx, &cfg) + go server.Start(ctx) // graceful termination handler. @@ -88,12 +88,11 @@ func init() { RootCmd.AddCommand(ControllerCmd) flags := ControllerCmd.Flags() - stringFlag(flags, "fallback-k8s-version", "Fallback target Kubernetes version for schema / upgrade checks.", - newStringOpts(). - withDefault("1.23.0")) - boolFlag(flags, "show-debug-info", "Set to true to print debug info to the footer of MR comments.") - boolFlag(flags, "enable-conftest", "Set to true to enable conftest policy checking of manifests.") - stringFlag(flags, "label-filter", `(Optional) If set, The label that must be set on an MR (as "kubechecks:") for kubechecks to process the merge request webhook.`) + stringFlag(flags, "fallback-k8s-version", "Fallback target Kubernetes version for schema / upgrade checks (KUBECHECKS_FALLBACK_K8S_VERSION).", + newStringOpts().withDefault("1.23.0")) + boolFlag(flags, "show-debug-info", "Set to true to print debug info to the footer of MR comments (KUBECHECKS_SHOW_DEBUG_INFO).") + boolFlag(flags, "enable-conftest", "Set to true to enable conftest policy checking of manifests (KUBECHECKS_ENABLE_CONFTEST).") + stringFlag(flags, "label-filter", `(Optional) If set, The label that must be set on an MR (as "kubechecks:") for kubechecks to process the merge request webhook (KUBECHECKS_LABEL_FILTER).`) stringFlag(flags, "openai-api-token", "OpenAI API Token.") stringFlag(flags, "webhook-url-base", "The endpoint to listen on for incoming PR/MR event webhooks. For example, 'https://checker.mycompany.com'.") stringFlag(flags, "webhook-url-prefix", "If your application is running behind a proxy that uses path based routing, set this value to match the path prefix. For example, '/hello/world'.") diff --git a/docs/contributing.md b/docs/contributing.md index 746a5cfb..cd5be179 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -106,7 +106,7 @@ Click the Detailed view button at the top, and click the refresh button next to #### Minikube -If you're using minikube with Tilt we recommend following this [guide](https://github.com/tilt-dev/minikube-local) to setup a local registry that Tilt can push to automatically. Without this Tilt will attempt to push up to Docker Hub by default. +If you're using minikube with Tilt we recommend following this [guide](https://github.com/tilt-dev/ctlptl) to setup a local registry that Tilt can push to automatically. Without this Tilt will attempt to push up to Docker Hub by default. ### Code Changes diff --git a/localdev/argocd/Tiltfile b/localdev/argocd/Tiltfile index aebaa70d..a53d08c8 100644 --- a/localdev/argocd/Tiltfile +++ b/localdev/argocd/Tiltfile @@ -72,3 +72,13 @@ def deploy_argo(): labels=["argocd"], resource_deps=['k8s:namespace'] ) + +def delete_argocd_apps_on_tilt_down(): + if config.tilt_subcommand == 'down': + # delete Apps + local("./localdev/argocd/delete-apps.sh") + +def force_argocd_cleanup_on_tilt_down(): + if config.tilt_subcommand == 'down': + # force cleanup of Apps and CRD on tilt down + local("./localdev/argocd/force-cleanup-apps.sh") \ No newline at end of file diff --git a/localdev/argocd/delete-apps.sh b/localdev/argocd/delete-apps.sh new file mode 100755 index 00000000..dc31e9b4 --- /dev/null +++ b/localdev/argocd/delete-apps.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "Deleting ArgoCD test Applications & AppSets (gracefully)..." + +# Delete ApplicationSets +for a in $(kubectl get ApplicationSet -n kubechecks -o=jsonpath='{.items[*].metadata.name}'); do + kubectl delete ApplicationSet $a -n kubechecks --timeout=10s; +done; + +# Delete Applications +for a in $(kubectl get application -n kubechecks -o=jsonpath='{.items[*].metadata.name}'); do + kubectl delete application $a -n kubechecks --timeout=10s; +done; + +exit 0; \ No newline at end of file diff --git a/localdev/argocd/force-cleanup-apps.sh b/localdev/argocd/force-cleanup-apps.sh new file mode 100755 index 00000000..d12ef6df --- /dev/null +++ b/localdev/argocd/force-cleanup-apps.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# check if Applications CRD still exists +exists=$(kubectl get crd | grep "applications.argoproj.io" | wc -l) +if [ "$exists" -eq 0 ]; then + echo "Applications CRD doesn't exist. Exiting..." + exit 0; +fi + +# give time for other processes to cleanup properly +echo "ArgoCD Cleanup: waiting 20 seconds..." +sleep 20 +echo "Cleaning up ArgoCD test Applications and CRDs..." + +# Cleanup Applications +for a in $(kubectl get application -n kubechecks -o=jsonpath='{.items[*].metadata.name}'); do + # remove finalizer from Applications (ArgoCD is probably shutdown by now and deleting apps will hang) + kubectl patch application $a -n kubechecks --type json -p='[{"op": "remove", "path": "/metadata/finalizers"}]'; + kubectl delete application $a -n kubechecks; +done; + +# Cleanup ArgoCD CRDs +for crd in applications.argoproj.io applicationsets.argoproj.io appprojects.argoproj.io; do + kubectl delete crd $crd; +done; + +exit 0; \ No newline at end of file diff --git a/localdev/kubechecks/values.yaml b/localdev/kubechecks/values.yaml index d576f53d..f47241ae 100644 --- a/localdev/kubechecks/values.yaml +++ b/localdev/kubechecks/values.yaml @@ -2,27 +2,43 @@ deployment: annotations: reloader.stakater.com/auto: "true" env: - KUBECHECKS_LOG_LEVEL: debug - KUBECHECKS_ENABLE_WEBHOOK_CONTROLLER: false - KUBECHECKS_ARGOCD_API_INSECURE: true - KUBECHECKS_ARGOCD_API_PATH_PREFIX : '/argocd' - KUBECHECKS_WEBHOOK_URL_PREFIX: 'kubechecks' - KUBECHECKS_NAMESPACE: 'kubechecks' - KUBECHECKS_FALLBACK_K8S_VERSION: "1.25.0" - KUBECHECKS_SHOW_DEBUG_INFO: "true" + - name: KUBECHECKS_LOG_LEVEL + value: debug + - name: KUBECHECKS_ENABLE_WEBHOOK_CONTROLLER + value: "false" + - name: KUBECHECKS_ARGOCD_API_INSECURE + value: "true" + - name: KUBECHECKS_ARGOCD_API_PATH_PREFIX + value : '/argocd' + - name: KUBECHECKS_ARGOCD_NAMESPACE + value: 'kubechecks' + - name: KUBECHECKS_WEBHOOK_URL_PREFIX + value: 'kubechecks' + - name: KUBECHECKS_NAMESPACE + value: 'kubechecks' + - name: KUBECHECKS_FALLBACK_K8S_VERSION + value: "1.25.0" + - name: KUBECHECKS_SHOW_DEBUG_INFO + value: "true" # OTEL - KUBECHECKS_OTEL_COLLECTOR_PORT: "4317" - KUBECHECKS_OTEL_ENABLED: "false" + - name: KUBECHECKS_OTEL_COLLECTOR_PORT + value: "4317" + - name: KUBECHECKS_OTEL_ENABLED + value: "false" # Webhook Management - KUBECHECKS_ENSURE_WEBHOOKS: "true" - KUBECHECKS_MONITOR_ALL_APPLICATIONS: "true" + - name: KUBECHECKS_ENSURE_WEBHOOKS + value: "true" + - name: KUBECHECKS_MONITOR_ALL_APPLICATIONS + value: "true" # # KUBECHECKS_LABEL_FILTER: "test" # On your PR/MR, prefix this with "kubechecks:" # KUBECHECKS_SCHEMAS_LOCATION: https://github.com/zapier/kubecheck-schemas.git - KUBECHECKS_TIDY_OUTDATED_COMMENTS_MODE: "delete" + - name: KUBECHECKS_TIDY_OUTDATED_COMMENTS_MODE + value: "delete" + - name: KUBECHECKS_ENABLE_CONFTEST + value: "true" image: - repository: "" pullPolicy: Always name: "kubechecks" tag: "" diff --git a/localdev/terraform/modules/vcs_files/base_files/.kubechecks.yaml b/localdev/terraform/modules/vcs_files/base_files/.kubechecks.yaml deleted file mode 100644 index 3e5642cb..00000000 --- a/localdev/terraform/modules/vcs_files/base_files/.kubechecks.yaml +++ /dev/null @@ -1,16 +0,0 @@ -applications: - - name: in-cluster-httpbin - cluster: in-cluster - path: apps/httpbin/overlays/in-cluster - additionalPaths: - - apps/httpbin/base - - name: in-cluster-echo-server - cluster: in-cluster - path: apps/echo-server/in-cluster - -applicationSets: - - name: httpdump - paths: - - apps/httpdump/base - - apps/httpdump/overlays/a - - apps/httpdump/overlays/b \ No newline at end of file diff --git a/localdev/test_apps/Tiltfile b/localdev/test_apps/Tiltfile index 6e597e33..20626bcd 100644 --- a/localdev/test_apps/Tiltfile +++ b/localdev/test_apps/Tiltfile @@ -3,27 +3,29 @@ # Test ArgoCD Applications # ///////////////////////////////////////////////////////////////////////////// -def install_test_apps(cfg): - # Load the terraform url we output, default to gitlab if cant find a vcs-type variable - vcsPath = "./localdev/terraform/{}/project.url".format(cfg.get('vcs-type', 'gitlab')) - print("Path to url: " + vcsPath) - projectUrl=str(read_file(vcsPath, "")).strip('\n') - print("Remote Project URL: " + projectUrl) +k8s_kind('Applications', api_version="apiextensions.k8s.io/v1") - k8s_kind('Applications', api_version="apiextensions.k8s.io/v1") +def install_test_apps(cfg): + if config.tilt_subcommand != 'down': + # Load the terraform url we output, default to gitlab if cant find a vcs-type variable + vcsPath = "./localdev/terraform/{}/project.url".format(cfg.get('vcs-type', 'gitlab')) + print("Path to url: " + vcsPath) + projectUrl=str(read_file(vcsPath, "")).strip('\n') + print("Remote Project URL: " + projectUrl) - for app in ["echo-server", "httpbin"]: - print("Creating Test App: " + app) + for app in ["echo-server", "httpbin"]: + print("Creating Test App: " + app) - # read the application YAML and patch the repoURL - objects = read_yaml_stream("localdev/test_apps/{}.yaml".format(app)) - for o in objects: - o['spec']['source']['repoURL'] = projectUrl - k8s_yaml(encode_yaml_stream(objects)) + # read the application YAML and patch the repoURL + objects = read_yaml_stream("localdev/test_apps/{}.yaml".format(app)) + for o in objects: + o['metadata']['namespace'] = "kubechecks" + o['spec']['source']['repoURL'] = projectUrl + k8s_yaml(encode_yaml_stream(objects)) - k8s_resource( - new_name=app, - objects=['in-cluster-{}:application'.format(app)], - labels=["test_apps"], - resource_deps=["argocd-crds","argocd"], - ) + k8s_resource( + new_name=app, + objects=['in-cluster-{}:application'.format(app)], + labels=["test_apps"], + resource_deps=["argocd-crds","argocd"], + ) diff --git a/localdev/test_apps/echo-server.yaml b/localdev/test_apps/echo-server.yaml index 3b9b9826..1afd41cf 100644 --- a/localdev/test_apps/echo-server.yaml +++ b/localdev/test_apps/echo-server.yaml @@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: in-cluster-echo-server + namespace: kubechecks finalizers: - resources-finalizer.argocd.argoproj.io spec: @@ -24,3 +25,4 @@ spec: selfHeal: false syncOptions: - CreateNamespace=true + - ServerSideApply=true diff --git a/localdev/test_apps/httpbin.yaml b/localdev/test_apps/httpbin.yaml index b7d264a7..8af8ebe6 100644 --- a/localdev/test_apps/httpbin.yaml +++ b/localdev/test_apps/httpbin.yaml @@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: in-cluster-httpbin + namespace: kubechecks finalizers: - resources-finalizer.argocd.argoproj.io spec: diff --git a/localdev/test_appsets/httpdump.yaml b/localdev/test_appsets/httpdump.yaml index cf317605..1c11ad99 100644 --- a/localdev/test_appsets/httpdump.yaml +++ b/localdev/test_appsets/httpdump.yaml @@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: name: httpdump + namespace: kubechecks spec: generators: - list: diff --git a/pkg/app_watcher/app_watcher.go b/pkg/app_watcher/app_watcher.go new file mode 100644 index 00000000..39c2ae31 --- /dev/null +++ b/pkg/app_watcher/app_watcher.go @@ -0,0 +1,160 @@ +package app_watcher + +import ( + "context" + "reflect" + + appclientset "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned" + "github.com/rs/zerolog/log" + "github.com/zapier/kubechecks/pkg/config" + "k8s.io/client-go/tools/clientcmd" + + "strings" + "time" + + appv1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + informers "github.com/argoproj/argo-cd/v2/pkg/client/informers/externalversions/application/v1alpha1" + applisters "github.com/argoproj/argo-cd/v2/pkg/client/listers/application/v1alpha1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" +) + +// ApplicationWatcher is the controller that watches ArgoCD Application resources via the Kubernetes API +type ApplicationWatcher struct { + cfg *config.ServerConfig + applicationClientset appclientset.Interface + appInformer cache.SharedIndexInformer + appLister applisters.ApplicationLister +} + +// NewApplicationWatcher creates new instance of ApplicationWatcher. +func NewApplicationWatcher(cfg *config.ServerConfig) (*ApplicationWatcher, error) { + // this assumes kubechecks is running inside the cluster + kubeCfg, err := clientcmd.BuildConfigFromFlags("", "") + if err != nil { + log.Fatal().Msgf("Error building kubeconfig: %s", err.Error()) + } + + appClient := appclientset.NewForConfigOrDie(kubeCfg) + + ctrl := ApplicationWatcher{ + cfg: cfg, + applicationClientset: appClient, + } + + appInformer, appLister := ctrl.newApplicationInformerAndLister(time.Second * 30) + + ctrl.appInformer = appInformer + ctrl.appLister = appLister + + return &ctrl, nil +} + +// Run starts the Application CRD controller. +func (ctrl *ApplicationWatcher) Run(ctx context.Context, processors int) { + log.Info().Msg("starting Application Controller") + + defer runtime.HandleCrash() + + go ctrl.appInformer.Run(ctx.Done()) + + if !cache.WaitForCacheSync(ctx.Done(), ctrl.appInformer.HasSynced) { + log.Error().Msg("Timed out waiting for caches to sync") + return + } + + <-ctx.Done() +} + +// onAdd is the function executed when the informer notifies the +// presence of a new Application in the namespace +func (ctrl *ApplicationWatcher) onApplicationAdded(obj interface{}) { + app, ok := canProcessApp(obj) + if !ok { + return + } + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + log.Error().Err(err).Msg("appwatcher: could not get key for added application") + } + log.Debug().Str("key", key).Msg("appwatcher: onApplicationAdded") + ctrl.cfg.VcsToArgoMap.AddApp(app) +} + +func (ctrl *ApplicationWatcher) onApplicationUpdated(old, new interface{}) { + newApp, newOk := canProcessApp(new) + oldApp, oldOk := canProcessApp(old) + if !newOk || !oldOk { + return + } + + key, err := cache.MetaNamespaceKeyFunc(new) + if err != nil { + log.Warn().Err(err).Msg("appwatcher: could not get key for updated application") + } + + // We want to update when any of Source or Sources parameters has changed + if !reflect.DeepEqual(oldApp.Spec.Source, newApp.Spec.Source) || !reflect.DeepEqual(oldApp.Spec.Sources, newApp.Spec.Sources) { + log.Debug().Str("key", key).Msg("appwatcher: onApplicationUpdated") + ctrl.cfg.VcsToArgoMap.UpdateApp(old.(*appv1alpha1.Application), new.(*appv1alpha1.Application)) + } + +} + +func (ctrl *ApplicationWatcher) onApplicationDeleted(obj interface{}) { + app, ok := canProcessApp(obj) + if !ok { + return + } + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + log.Warn().Err(err).Msg("appwatcher: could not get key for deleted application") + } + + log.Debug().Str("key", key).Msg("appwatcher: onApplicationDeleted") + ctrl.cfg.VcsToArgoMap.DeleteApp(app) +} + +/* +This Go function, named newApplicationInformerAndLister, is part of the ApplicationWatcher struct. It sets up a Kubernetes SharedIndexInformer and a Lister for Argo CD Applications. +A SharedIndexInformer is used to watch changes to a specific type of Kubernetes resource in an efficient manner. It significantly reduces the load on the Kubernetes API server by sharing and caching watches between all controllers that need to observe the object. +Listers use the data from the informer's cache to provide a read-optimized view of the cache which reduces the load on the API Server and hides some complexity. +*/ +func (ctrl *ApplicationWatcher) newApplicationInformerAndLister(refreshTimeout time.Duration) (cache.SharedIndexInformer, applisters.ApplicationLister) { + informer := informers.NewApplicationInformer(ctrl.applicationClientset, "", refreshTimeout, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, + ) + + lister := applisters.NewApplicationLister(informer.GetIndexer()) + informer.AddEventHandler( + cache.ResourceEventHandlerFuncs{ + AddFunc: ctrl.onApplicationAdded, + UpdateFunc: ctrl.onApplicationUpdated, + DeleteFunc: ctrl.onApplicationDeleted, + }, + ) + return informer, lister +} + +func canProcessApp(obj interface{}) (*appv1alpha1.Application, bool) { + app, ok := obj.(*appv1alpha1.Application) + if !ok { + return &appv1alpha1.Application{}, false + } + + for _, src := range app.Spec.Sources { + if isGitRepo(src.RepoURL) { + return app, true + } + } + + if !isGitRepo(app.Spec.Source.RepoURL) { + return app, false + } + + return app, true +} + +func isGitRepo(url string) bool { + return strings.Contains(url, "gitlab.com") || strings.Contains(url, "github.com") +} diff --git a/pkg/app_watcher/app_watcher_test.go b/pkg/app_watcher/app_watcher_test.go new file mode 100644 index 00000000..e47dc85c --- /dev/null +++ b/pkg/app_watcher/app_watcher_test.go @@ -0,0 +1,150 @@ +package app_watcher + +import ( + "context" + "testing" + "time" + + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + appclientsetfake "github.com/argoproj/argo-cd/v2/pkg/client/clientset/versioned/fake" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/zapier/kubechecks/pkg/config" +) + +func initTestObjects() *ApplicationWatcher { + // Setup the fake Application client set and informer. + testApp1 := &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-1", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{RepoURL: "https://gitlab.com/test/repo.git"}, + }, + } + testApp2 := &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-2", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{RepoURL: "https://github.com/test/repo.git"}, + }, + } + + clientset := appclientsetfake.NewSimpleClientset(testApp1, testApp2) + ctrl := &ApplicationWatcher{ + applicationClientset: clientset, + cfg: &config.ServerConfig{ + VcsToArgoMap: config.NewVcsToArgoMap(), + }, + } + + appInformer, appLister := ctrl.newApplicationInformerAndLister(time.Second * 1) + ctrl.appInformer = appInformer + ctrl.appLister = appLister + + return ctrl +} + +func TestApplicationAdded(t *testing.T) { + ctrl := initTestObjects() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go ctrl.Run(ctx, 1) + + time.Sleep(time.Second * 1) + + assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 2) + + _, err := ctrl.applicationClientset.ArgoprojV1alpha1().Applications("default").Create(ctx, &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-3", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{RepoURL: "https://gitlab.com/test/repo-3.git"}, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Error(err) + } + + time.Sleep(time.Second * 1) + assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 3) +} + +func TestApplicationUpdated(t *testing.T) { + ctrl := initTestObjects() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go ctrl.Run(ctx, 1) + + time.Sleep(time.Second * 1) + + assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 2) + + oldAppDirectory := ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + newAppDirectory := ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo-3.git") + assert.Equal(t, oldAppDirectory.Count(), 1) + assert.Equal(t, newAppDirectory.Count(), 0) + // + _, err := ctrl.applicationClientset.ArgoprojV1alpha1().Applications("default").Update(ctx, &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-1", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{RepoURL: "https://gitlab.com/test/repo-3.git"}, + }, + }, metav1.UpdateOptions{}) + if err != nil { + t.Error(err) + } + time.Sleep(time.Second * 1) + oldAppDirectory = ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + newAppDirectory = ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo-3.git") + assert.Equal(t, oldAppDirectory.Count(), 0) + assert.Equal(t, newAppDirectory.Count(), 1) +} + +func TestApplicationDeleted(t *testing.T) { + ctrl := initTestObjects() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go ctrl.Run(ctx, 1) + + time.Sleep(time.Second * 1) + + assert.Equal(t, len(ctrl.cfg.VcsToArgoMap.GetMap()), 2) + + appDirectory := ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + assert.Equal(t, appDirectory.Count(), 1) + // + err := ctrl.applicationClientset.ArgoprojV1alpha1().Applications("default").Delete(ctx, "test-app-1", metav1.DeleteOptions{}) + if err != nil { + t.Error(err) + } + time.Sleep(time.Second * 1) + + appDirectory = ctrl.cfg.VcsToArgoMap.GetAppsInRepo("https://gitlab.com/test/repo.git") + assert.Equal(t, appDirectory.Count(), 0) +} + +// TestIsGitRepo will test various URLs against the isGitRepo function. +func TestIsGitRepo(t *testing.T) { + tests := []struct { + url string + expected bool + }{ + {"https://github.com/user/repo.git", true}, + {"https://gitlab.com/user/repo.git", true}, + {"ssh://gitlab.com/user/repo.git", true}, + {"user@github.com:user/repo.git", true}, + {"https://bitbucket.org/user/repo.git", false}, + {"user@gitlab.invalid/user/repo.git", false}, + {"http://myownserver.com/git/repo.git", false}, + } + + for _, test := range tests { + if result := isGitRepo(test.url); result != test.expected { + t.Errorf("isGitRepo(%q) = %v; want %v", test.url, result, test.expected) + } + } +} diff --git a/pkg/config/app_directory.go b/pkg/config/app_directory.go index 603e6fe5..2d123d34 100644 --- a/pkg/config/app_directory.go +++ b/pkg/config/app_directory.go @@ -150,7 +150,7 @@ func (d *AppDirectory) AddApp(app v1alpha1.Application) { Str("appName", app.Name). Str("cluster-name", app.Spec.Destination.Name). Str("cluster-server", app.Spec.Destination.Server). - Msg("found app") + Msg("add app") d.appsMap[app.Name] = app d.AddDir(app.Name, getSourcePath(app)) } @@ -163,6 +163,36 @@ func (d *AppDirectory) AddFile(appName, path string) { d.appFiles[path] = append(d.appFiles[path], appName) } +func (d *AppDirectory) RemoveApp(app v1alpha1.Application) { + log.Debug(). + Str("appName", app.Name). + Str("cluster-name", app.Spec.Destination.Name). + Str("cluster-server", app.Spec.Destination.Server). + Msg("delete app") + + // remove app from appsMap + delete(d.appsMap, app.Name) + + // Clean up app from appDirs + sourcePath := getSourcePath(app) + d.appDirs[sourcePath] = removeFromSlice[string](d.appDirs[sourcePath], app.Name, func(a, b string) bool { return a == b }) + + // Clean up app from appFiles + src := app.Spec.GetSource() + srcPath := src.Path + if helm := src.Helm; helm != nil { + for _, param := range helm.FileParameters { + path := filepath.Join(srcPath, param.Path) + d.appFiles[path] = removeFromSlice[string](d.appFiles[path], app.Name, func(a, b string) bool { return a == b }) + } + + for _, valueFilePath := range helm.ValueFiles { + path := filepath.Join(srcPath, valueFilePath) + d.appFiles[path] = removeFromSlice[string](d.appFiles[path], app.Name, func(a, b string) bool { return a == b }) + } + } +} + func mergeMaps[T any](first map[string]T, second map[string]T, combine func(T, T) T) map[string]T { result := make(map[string]T) for key, value := range first { @@ -186,3 +216,12 @@ func mergeLists[T any](a []T, b []T) []T { func takeFirst[T any](a, _ T) T { return a } + +func removeFromSlice[T any](slice []T, element T, equal func(T, T) bool) []T { + for i, j := range slice { + if equal(j, element) { + return append(slice[:i], slice[i+1:]...) + } + } + return slice +} diff --git a/pkg/config/app_directory_test.go b/pkg/config/app_directory_test.go index 8453d677..a7243f6c 100644 --- a/pkg/config/app_directory_test.go +++ b/pkg/config/app_directory_test.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "reflect" "testing" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" @@ -99,3 +100,31 @@ func TestShouldInclude(t *testing.T) { }) } } + +// TestRemoveFromSlice performs tests on the removeFromSlice function. +func TestRemoveFromSlice(t *testing.T) { + // Test for integers + ints := []int{1, 2, 3, 4, 5} + intsAfterRemoval := []int{1, 2, 4, 5} + intsTest := func(t *testing.T) { + result := removeFromSlice(ints, 3, func(a, b int) bool { return a == b }) + if !reflect.DeepEqual(result, intsAfterRemoval) { + t.Errorf("Expected %v, got %v", intsAfterRemoval, result) + } + } + + // Test for strings + strings := []string{"apple", "banana", "cherry", "date"} + stringsAfterRemoval := []string{"apple", "cherry", "date"} + stringsTest := func(t *testing.T) { + result := removeFromSlice(strings, "banana", func(a, b string) bool { return a == b }) + if !reflect.DeepEqual(result, stringsAfterRemoval) { + t.Errorf("Expected %v, got %v", stringsAfterRemoval, result) + } + } + + // Execute subtests + t.Run("Integers", intsTest) + t.Run("Strings", stringsTest) + // Add more subtests for different generic types if necessary +} diff --git a/pkg/config/config.go b/pkg/config/config.go index ace5b34b..9f4d8f4d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -12,21 +12,21 @@ import ( "github.com/zapier/kubechecks/pkg/vcs" ) -type repoURL struct { +type RepoURL struct { Host, Path string } -func (r repoURL) CloneURL() string { +func (r RepoURL) CloneURL() string { return fmt.Sprintf("git@%s:%s", r.Host, r.Path) } -func buildNormalizedRepoUrl(host, path string) repoURL { +func buildNormalizedRepoUrl(host, path string) RepoURL { path = strings.TrimPrefix(path, "/") path = strings.TrimSuffix(path, ".git") - return repoURL{host, path} + return RepoURL{host, path} } -func normalizeRepoUrl(s string) (repoURL, error) { +func NormalizeRepoUrl(s string) (RepoURL, error) { var parser func(string) (*url.URL, error) if strings.HasPrefix(s, "http") { @@ -37,20 +37,43 @@ func normalizeRepoUrl(s string) (repoURL, error) { r, err := parser(s) if err != nil { - return repoURL{}, err + return RepoURL{}, err } return buildNormalizedRepoUrl(r.Host, r.Path), nil } -func (v2a *VcsToArgoMap) AddApp(app v1alpha1.Application) { +func (v2a *VcsToArgoMap) AddApp(app *v1alpha1.Application) { if app.Spec.Source == nil { log.Warn().Msgf("%s/%s: no source, skipping", app.Namespace, app.Name) return } appDirectory := v2a.GetAppsInRepo(app.Spec.Source.RepoURL) - appDirectory.ProcessApp(app) + appDirectory.ProcessApp(*app) +} + +func (v2a *VcsToArgoMap) UpdateApp(old *v1alpha1.Application, new *v1alpha1.Application) { + if new.Spec.Source == nil { + log.Warn().Msgf("%s/%s: no source, skipping", new.Namespace, new.Name) + return + } + + oldAppDirectory := v2a.GetAppsInRepo(old.Spec.Source.RepoURL) + oldAppDirectory.RemoveApp(*old) + + newAppDirectory := v2a.GetAppsInRepo(new.Spec.Source.RepoURL) + newAppDirectory.ProcessApp(*new) +} + +func (v2a *VcsToArgoMap) DeleteApp(app *v1alpha1.Application) { + if app.Spec.Source == nil { + log.Warn().Msgf("%s/%s: no source, skipping", app.Namespace, app.Name) + return + } + + oldAppDirectory := v2a.GetAppsInRepo(app.Spec.Source.RepoURL) + oldAppDirectory.RemoveApp(*app) } type ServerConfig struct { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 33296f17..3502a5db 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -4,38 +4,138 @@ import ( "fmt" "testing" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestNormalizeStrings(t *testing.T) { testCases := []struct { input string - expected repoURL + expected RepoURL }{ { input: "git@github.com:one/two", - expected: repoURL{"github.com", "one/two"}, + expected: RepoURL{"github.com", "one/two"}, }, { input: "https://github.com/one/two", - expected: repoURL{"github.com", "one/two"}, + expected: RepoURL{"github.com", "one/two"}, }, { input: "git@gitlab.com:djeebus/helm-test.git", - expected: repoURL{"gitlab.com", "djeebus/helm-test"}, + expected: RepoURL{"gitlab.com", "djeebus/helm-test"}, }, { input: "https://gitlab.com/djeebus/helm-test.git", - expected: repoURL{"gitlab.com", "djeebus/helm-test"}, + expected: RepoURL{"gitlab.com", "djeebus/helm-test"}, }, } for _, tc := range testCases { t.Run(fmt.Sprintf("case %s", tc.input), func(t *testing.T) { - actual, err := normalizeRepoUrl(tc.input) + actual, err := NormalizeRepoUrl(tc.input) require.NoError(t, err) assert.Equal(t, tc.expected, actual) }) } } + +// TestBuildNormalizedRepoURL tests the buildNormalizedRepoUrl function. +func TestBuildNormalizedRepoURL(t *testing.T) { + tests := []struct { + host string + path string + expected RepoURL + }{ + { + host: "example.com", + path: "/repository.git", + expected: RepoURL{ + Host: "example.com", + Path: "repository", + }, + }, + // ... additional test cases + } + + for _, tc := range tests { + result := buildNormalizedRepoUrl(tc.host, tc.path) + assert.Equal(t, tc.expected, result) + } +} + +// TestAddApp tests the AddApp method from the VcsToArgoMap type. +func TestAddApp(t *testing.T) { + // Setup your mocks and expected calls here. + + v2a := NewVcsToArgoMap() // This would be mocked accordingly. + app1 := &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-1", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argo-cd.git", + Path: "test-app-1", + }, + }, + } + + v2a.AddApp(app1) + appDir := v2a.GetAppsInRepo("https://github.com/argoproj/argo-cd.git") + + assert.Equal(t, appDir.Count(), 1) + assert.Equal(t, len(appDir.appDirs["test-app-1"]), 1) + + // Assertions to verify the behavior here. + app2 := &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-2", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argo-cd.git", + Path: "test-app-2", + }, + }, + } + + v2a.AddApp(app2) + assert.Equal(t, appDir.Count(), 2) + assert.Equal(t, len(appDir.appDirs["test-app-2"]), 1) +} + +func TestDeleteApp(t *testing.T) { + // Setup your mocks and expected calls here. + + v2a := NewVcsToArgoMap() // This would be mocked accordingly. + app1 := &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-1", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argo-cd.git", + Path: "test-app-1", + }, + }, + } + // Assertions to verify the behavior here. + app2 := &v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{Name: "test-app-2", Namespace: "default"}, + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + RepoURL: "https://github.com/argoproj/argo-cd.git", + Path: "test-app-2", + }, + }, + } + + v2a.AddApp(app1) + v2a.AddApp(app2) + appDir := v2a.GetAppsInRepo("https://github.com/argoproj/argo-cd.git") + + assert.Equal(t, appDir.Count(), 2) + assert.Equal(t, len(appDir.appDirs["test-app-1"]), 1) + assert.Equal(t, len(appDir.appDirs["test-app-2"]), 1) + + v2a.DeleteApp(app2) + assert.Equal(t, appDir.Count(), 1) + assert.Equal(t, len(appDir.appDirs["test-app-2"]), 0) +} diff --git a/pkg/config/vcstoargomap.go b/pkg/config/vcstoargomap.go index 47e82f22..31f61390 100644 --- a/pkg/config/vcstoargomap.go +++ b/pkg/config/vcstoargomap.go @@ -12,15 +12,19 @@ import ( ) type VcsToArgoMap struct { - appDirByRepo map[repoURL]*AppDirectory + appDirByRepo map[RepoURL]*AppDirectory } func NewVcsToArgoMap() VcsToArgoMap { return VcsToArgoMap{ - appDirByRepo: make(map[repoURL]*AppDirectory), + appDirByRepo: make(map[RepoURL]*AppDirectory), } } +func (v2a *VcsToArgoMap) GetMap() map[RepoURL]*AppDirectory { + return v2a.appDirByRepo +} + func BuildAppsMap(ctx context.Context) (VcsToArgoMap, error) { result := NewVcsToArgoMap() argoClient := argo_client.GetArgoClient() @@ -30,14 +34,14 @@ func BuildAppsMap(ctx context.Context) (VcsToArgoMap, error) { return result, errors.Wrap(err, "failed to list applications") } for _, app := range apps.Items { - result.AddApp(app) + result.AddApp(&app) } return result, nil } func (v2a *VcsToArgoMap) GetAppsInRepo(repoCloneUrl string) *AppDirectory { - repoUrl, err := normalizeRepoUrl(repoCloneUrl) + repoUrl, err := NormalizeRepoUrl(repoCloneUrl) if err != nil { log.Warn().Err(err).Msgf("failed to parse %s", repoCloneUrl) } diff --git a/pkg/conftest/conftest.go b/pkg/conftest/conftest.go index cb766822..3e87402c 100644 --- a/pkg/conftest/conftest.go +++ b/pkg/conftest/conftest.go @@ -41,7 +41,7 @@ func Conftest(ctx context.Context, app *v1alpha1.Application, repoPath string) ( policiesLocations := viper.GetStringSlice("policies-location") var locations []string for _, policiesLocation := range policiesLocations { - log.Debug().Str("schemas-location", policiesLocation).Msg("viper") + log.Debug().Str("policies-location", policiesLocation).Msg("viper") schemaPath := reposCache.EnsurePath(ctx, repoPath, policiesLocation) if schemaPath != "" { locations = append(locations, schemaPath) diff --git a/pkg/server/server.go b/pkg/server/server.go index fd2abe11..032af4ab 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/viper" "github.com/ziflex/lecho/v3" + "github.com/zapier/kubechecks/pkg/app_watcher" "github.com/zapier/kubechecks/pkg/config" "github.com/zapier/kubechecks/pkg/vcs" ) @@ -22,21 +23,36 @@ import ( const KubeChecksHooksPathPrefix = "/hooks" type Server struct { - cfg *config.ServerConfig + cfg *config.ServerConfig + appWatcher *app_watcher.ApplicationWatcher } -func NewServer(cfg *config.ServerConfig) *Server { - return &Server{cfg: cfg} +func NewServer(ctx context.Context, cfg *config.ServerConfig) *Server { + var appWatcher *app_watcher.ApplicationWatcher + if viper.GetBool("monitor-all-applications") { + argoMap, err := config.BuildAppsMap(ctx) + if err != nil { + log.Fatal().Err(err).Msg("could not build VcsToArgoMap") + } + cfg.VcsToArgoMap = argoMap + + appWatcher, err = app_watcher.NewApplicationWatcher(cfg) + if err != nil { + log.Fatal().Err(err).Msg("could not create ApplicationWatcher") + } + } else { + cfg.VcsToArgoMap = config.NewVcsToArgoMap() + } + + return &Server{cfg: cfg, appWatcher: appWatcher} } func (s *Server) Start(ctx context.Context) { - if argoMap, err := s.buildVcsToArgoMap(ctx); err != nil { - log.Warn().Err(err).Msg("failed to build vcs app map from argo") - } else { - s.cfg.VcsToArgoMap = argoMap + if s.appWatcher != nil { + go s.appWatcher.Run(ctx, 1) } - if err := s.ensureWebhooks(); err != nil { + if err := s.ensureWebhooks(ctx); err != nil { log.Warn().Err(err).Msg("failed to create webhooks") } @@ -78,7 +94,7 @@ func (s *Server) hooksPrefix() string { return strings.TrimSuffix(serverUrl, "/") } -func (s *Server) ensureWebhooks() error { +func (s *Server) ensureWebhooks(ctx context.Context) error { if !viper.GetBool("ensure-webhooks") { return nil } @@ -94,7 +110,6 @@ func (s *Server) ensureWebhooks() error { log.Info().Msg("ensuring all webhooks are created correctly") - ctx := context.TODO() vcsClient := s.cfg.VcsClient fullUrl, err := url.JoinPath(urlBase, s.hooksPrefix(), vcsClient.GetName(), "project") @@ -120,13 +135,3 @@ func (s *Server) ensureWebhooks() error { return nil } - -func (s *Server) buildVcsToArgoMap(ctx context.Context) (config.VcsToArgoMap, error) { - if !viper.GetBool("monitor-all-applications") { - return config.NewVcsToArgoMap(), nil - } - - log.Debug().Msg("building VCS to Application Map") - - return config.BuildAppsMap(ctx) -} diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index bffa5541..a9e29719 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -1,6 +1,7 @@ package server import ( + "context" "testing" "github.com/zapier/kubechecks/pkg/config" @@ -50,7 +51,7 @@ func TestHooksPrefix(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewServer(tt.cfg) + s := NewServer(context.TODO(), tt.cfg) if got := s.hooksPrefix(); got != tt.want { t.Errorf("hooksPrefix() = %v, want %v", got, tt.want) } diff --git a/pkg/vcs/gitlab_client/client.go b/pkg/vcs/gitlab_client/client.go index f4dbc15e..ab4099e9 100644 --- a/pkg/vcs/gitlab_client/client.go +++ b/pkg/vcs/gitlab_client/client.go @@ -12,7 +12,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/viper" - "github.com/whilp/git-urls" + giturls "github.com/whilp/git-urls" "github.com/xanzy/go-gitlab" "github.com/zapier/kubechecks/pkg"