From faecbfeb8ec19b2a6eb128508d7f61a1d1daa5ee Mon Sep 17 00:00:00 2001 From: Hongwei Liu Date: Tue, 30 Jul 2024 20:26:32 +0800 Subject: [PATCH] feat(STONEINTG-997): save build PLR info to PLR metadata 1.Save build source branch info to build PLR metadata as below * save pr group to annotation test.appstudio.openshift.io/pr-group * save pr group in hash format to label test.appstudio.openshift.io/pr-group-sha 2.Save build PLR start time to snapshot annotation test.appstudio.openshift.io/pipelinerunstarttime Signed-off-by: Hongwei Liu --- docs/build_pipeline_controller.md | 6 + gitops/snapshot.go | 9 ++ .../buildpipeline/buildpipeline_adapter.go | 58 ++++++++ .../buildpipeline_adapter_test.go | 38 +++++ .../buildpipeline/buildpipeline_controller.go | 2 + tekton/build_pipeline.go | 47 +++++++ tekton/build_pipeline_test.go | 133 ++++++++++++++++++ 7 files changed, 293 insertions(+) create mode 100644 tekton/build_pipeline_test.go diff --git a/docs/build_pipeline_controller.md b/docs/build_pipeline_controller.md index 92079fa1a..2594e6643 100644 --- a/docs/build_pipeline_controller.md +++ b/docs/build_pipeline_controller.md @@ -12,6 +12,7 @@ flowchart TD %% Node definitions predicate((PREDICATE:
Filter events related to
PipelineRuns)) new_pipeline_run{Pipeline created?} +new_pipeline_run_without_prgroup{PR group is added to pipelineRun metadata?} get_pipeline_run{Pipeline updated?} failed_pipeline_run{Pipeline failed?} finalizer_exists{Does the finalizer already exist?} @@ -24,15 +25,20 @@ add_finalizer(Add finalizer to build PLR) remove_finalizer(Remove finalizer from build PLR) error[Return error] continue[Continue processing] +update_metadata(add PR group info to build pipelineRun metadata) %% Node connections predicate --> get_pipeline_run predicate --> new_pipeline_run +predicate --> new_pipeline_run_without_prgroup predicate --> failed_pipeline_run new_pipeline_run --Yes--> finalizer_exists finalizer_exists --No--> add_finalizer add_finalizer --> continue failed_pipeline_run --Yes --> remove_finalizer +new_pipeline_run_without_prgroup --No --> update_metadata +new_pipeline_run_without_prgroup --Yes --> continue +update_metadata --> continue get_pipeline_run --Yes --> retrieve_associated_entity get_pipeline_run --No --> error retrieve_associated_entity --No --> error diff --git a/gitops/snapshot.go b/gitops/snapshot.go index 50f95a479..92ba6277c 100644 --- a/gitops/snapshot.go +++ b/gitops/snapshot.go @@ -76,6 +76,15 @@ const ( // SnapshotStatusReportAnnotation contains metadata of tests related to status reporting to git provider SnapshotStatusReportAnnotation = "test.appstudio.openshift.io/git-reporter-status" + // PRGroupAnnotation contains the pr group name + PRGroupAnnotation = "test.appstudio.openshift.io/pr-group" + + // PRGroupHashLabel contains the pr group name in sha format + PRGroupHashLabel = "test.appstudio.openshift.io/pr-group-sha" + + // BuildPipelineRunStartTime contains the start time of build pipelineRun + BuildPipelineRunStartTime = "test.appstudio.openshift.io/pipelinerunstarttime" + // BuildPipelineRunPrefix contains the build pipeline run related labels and annotations BuildPipelineRunPrefix = "build.appstudio" diff --git a/internal/controller/buildpipeline/buildpipeline_adapter.go b/internal/controller/buildpipeline/buildpipeline_adapter.go index cd247d4e8..4d4b4c985 100644 --- a/internal/controller/buildpipeline/buildpipeline_adapter.go +++ b/internal/controller/buildpipeline/buildpipeline_adapter.go @@ -30,6 +30,7 @@ import ( "github.com/konflux-ci/integration-service/loader" "github.com/konflux-ci/integration-service/tekton" "github.com/konflux-ci/operator-toolkit/controller" + "github.com/konflux-ci/operator-toolkit/metadata" applicationapiv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1" tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -149,6 +150,33 @@ func (a *Adapter) EnsurePipelineIsFinalized() (controller.OperationResult, error return controller.ContinueProcessing() } +// EnsurePRGroupAnnotated is an operation that will ensure that the pr group info +// is added to build pipelineRun metadata label and annotation once it is created, +// then these label and annotation will be copied to component snapshot when it is created +func (a *Adapter) EnsurePRGroupAnnotated() (controller.OperationResult, error) { + if metadata.HasLabel(a.pipelineRun, gitops.PRGroupHashLabel) && metadata.HasAnnotation(a.pipelineRun, gitops.PRGroupAnnotation) { + a.logger.Info("build pipelineRun has had pr group info in metadata, no need to update") + return controller.ContinueProcessing() + } + + err := a.addPRGroupToBuildPLRMetadata(a.pipelineRun) + if err != nil { + if errors.IsNotFound(err) { + a.logger.Error(err, "failed to add pr group info to build pipelineRun metadata due to notfound pipelineRun") + return controller.StopProcessing() + } else { + a.logger.Error(err, "failed to add pr group info to build pipelineRun metadata") + return controller.RequeueWithError(err) + } + + } + + a.logger.LogAuditEvent("pr group info is updated to build pipelineRun metadata", a.pipelineRun, h.LogActionUpdate, + "pipelineRun.Name", a.pipelineRun.Name) + + return controller.ContinueProcessing() +} + // getImagePullSpecFromPipelineRun gets the full image pullspec from the given build PipelineRun, // In case the Image pullspec can't be composed, an error will be returned. func (a *Adapter) getImagePullSpecFromPipelineRun(pipelineRun *tektonv1.PipelineRun) (string, error) { @@ -218,6 +246,10 @@ func (a *Adapter) prepareSnapshotForPipelineRun(pipelineRun *tektonv1.PipelineRu snapshot.Labels[gitops.BuildPipelineRunFinishTimeLabel] = strconv.FormatInt(time.Now().Unix(), 10) } + if pipelineRun.Status.StartTime != nil { + snapshot.Annotations[gitops.BuildPipelineRunStartTime] = strconv.FormatInt(pipelineRun.Status.StartTime.Time.Unix(), 10) + } + return snapshot, nil } @@ -329,3 +361,29 @@ func (a *Adapter) updatePipelineRunWithCustomizedError(canRemoveFinalizer *bool, return controller.RequeueOnErrorOrContinue(cerr) } + +// addPRGroupToBuildPLRMetadata will add pr-group info gotten from souce-branch to annotation +// and also the string in sha format to metadata label +func (a *Adapter) addPRGroupToBuildPLRMetadata(pipelineRun *tektonv1.PipelineRun) error { + prGroupName := tekton.GetPRGroupNameFromBuildPLR(pipelineRun) + if prGroupName != "" { + prGroupHashName := tekton.GenerateSHA(prGroupName) + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + var err error + pipelineRun, err = a.loader.GetPipelineRun(a.context, a.client, pipelineRun.Name, pipelineRun.Namespace) + if err != nil { + return err + } + + patch := client.MergeFrom(pipelineRun.DeepCopy()) + + _ = metadata.SetAnnotation(&pipelineRun.ObjectMeta, gitops.PRGroupAnnotation, prGroupName) + _ = metadata.SetLabel(&pipelineRun.ObjectMeta, gitops.PRGroupHashLabel, prGroupHashName) + + return a.client.Patch(a.context, pipelineRun, patch) + }) + } + a.logger.Info("can't find source branch info in build PLR, not need to update build pipelineRun metadata") + return nil +} diff --git a/internal/controller/buildpipeline/buildpipeline_adapter_test.go b/internal/controller/buildpipeline/buildpipeline_adapter_test.go index 2f790b46f..0d43bd87b 100644 --- a/internal/controller/buildpipeline/buildpipeline_adapter_test.go +++ b/internal/controller/buildpipeline/buildpipeline_adapter_test.go @@ -29,6 +29,7 @@ import ( "github.com/konflux-ci/integration-service/helpers" "github.com/konflux-ci/integration-service/loader" "github.com/konflux-ci/integration-service/tekton" + "github.com/konflux-ci/operator-toolkit/metadata" "knative.dev/pkg/apis" v1 "knative.dev/pkg/apis/duck/v1" @@ -267,6 +268,8 @@ var _ = Describe("Pipeline Adapter", Ordered, func() { "build.appstudio.openshift.io/repo": "https://github.com/devfile-samples/devfile-sample-go-basic?rev=c713067b0e65fb3de50d1f7c457eb51c2ab0dbb0", "foo": "bar", "chains.tekton.dev/signed": "true", + "pipelinesascode.tekton.dev/source-branch": "sourceBranch", + "pipelinesascode.tekton.dev/url-org": "redhat", }, }, Spec: tektonv1.PipelineRunSpec{ @@ -317,6 +320,7 @@ var _ = Describe("Pipeline Adapter", Ordered, func() { Value: *tektonv1.NewStructuredValues(SampleCommit), }, }, + StartTime: &metav1.Time{Time: time.Now()}, }, Status: v1.Status{ Conditions: v1.Conditions{ @@ -410,6 +414,8 @@ var _ = Describe("Pipeline Adapter", Ordered, func() { Expect(expectedSnapshot.Labels).NotTo(BeNil()) Expect(expectedSnapshot.Labels).Should(HaveKeyWithValue(Equal(gitops.BuildPipelineRunNameLabel), Equal(buildPipelineRun.Name))) Expect(expectedSnapshot.Labels).Should(HaveKeyWithValue(Equal(gitops.ApplicationNameLabel), Equal(hasApp.Name))) + Expect(metadata.HasAnnotation(expectedSnapshot, gitops.BuildPipelineRunStartTime)).To(BeTrue()) + Expect(expectedSnapshot.Annotations[gitops.BuildPipelineRunStartTime]).NotTo(BeNil()) }) It("ensures that Labels and Annotations were copied to snapshot from pipelinerun", func() { @@ -1014,6 +1020,38 @@ var _ = Describe("Pipeline Adapter", Ordered, func() { }) + When("add pr group to the build pipelineRun annotations and labels", func() { + BeforeEach(func() { + adapter = NewAdapter(ctx, buildPipelineRun, hasComp, hasApp, logger, loader.NewMockLoader(), k8sClient) + }) + It("add pr group to the build pipelineRun annotations and labels", func() { + existingBuildPLR := new(tektonv1.PipelineRun) + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: buildPipelineRun.Namespace, + Name: buildPipelineRun.Name, + }, existingBuildPLR) + Expect(err).NotTo(HaveOccurred()) + Expect(metadata.HasAnnotation(existingBuildPLR, gitops.PRGroupAnnotation)).To(BeFalse()) + Expect(metadata.HasLabel(existingBuildPLR, gitops.PRGroupHashLabel)).To(BeFalse()) + + // Add label and annotation to PLR + result, err := adapter.EnsurePRGroupAnnotated() + Expect(err).NotTo(HaveOccurred()) + Expect(result.CancelRequest).To(BeFalse()) + Expect(result.RequeueRequest).To(BeFalse()) + + err = adapter.client.Get(adapter.context, types.NamespacedName{ + Namespace: buildPipelineRun.Namespace, + Name: buildPipelineRun.Name, + }, existingBuildPLR) + Expect(err).NotTo(HaveOccurred()) + Expect(metadata.HasAnnotation(existingBuildPLR, gitops.PRGroupAnnotation)).To(BeTrue()) + Expect(existingBuildPLR.Annotations).Should(HaveKeyWithValue(Equal(gitops.PRGroupAnnotation), Equal("sourceBranch"))) + Expect(metadata.HasLabel(existingBuildPLR, gitops.PRGroupHashLabel)).To(BeTrue()) + Expect(existingBuildPLR.Labels[gitops.PRGroupHashLabel]).NotTo(BeNil()) + }) + }) + When("running pipeline with deletion timestamp is processed", func() { var runningDeletingBuildPipeline *tektonv1.PipelineRun diff --git a/internal/controller/buildpipeline/buildpipeline_controller.go b/internal/controller/buildpipeline/buildpipeline_controller.go index 643bb42b3..0f7bab44a 100644 --- a/internal/controller/buildpipeline/buildpipeline_controller.go +++ b/internal/controller/buildpipeline/buildpipeline_controller.go @@ -117,6 +117,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return controller.ReconcileHandler([]controller.Operation{ adapter.EnsurePipelineIsFinalized, + adapter.EnsurePRGroupAnnotated, adapter.EnsureSnapshotExists, }) } @@ -124,6 +125,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // AdapterInterface is an interface defining all the operations that should be defined in an Integration adapter. type AdapterInterface interface { EnsurePipelineIsFinalized() (controller.OperationResult, error) + EnsurePRGroupAnnotated() (controller.OperationResult, error) EnsureSnapshotExists() (controller.OperationResult, error) } diff --git a/tekton/build_pipeline.go b/tekton/build_pipeline.go index 9b73dcb29..4d933fd9a 100644 --- a/tekton/build_pipeline.go +++ b/tekton/build_pipeline.go @@ -18,8 +18,10 @@ package tekton import ( "context" + "crypto/sha256" "encoding/json" "fmt" + "strings" h "github.com/konflux-ci/integration-service/helpers" "github.com/konflux-ci/operator-toolkit/metadata" @@ -27,6 +29,20 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + // master branch in github/gitlab + MasterBranch = "master" + + // main branch in github/gitlab + MainBranch = "main" + + // PipelineAsCodeSourceBranchAnnotation is the branch name of the the pull request is created from + PipelineAsCodeSourceBranchAnnotation = "pipelinesascode.tekton.dev/source-branch" + + // PipelineAsCodeSourceRepoOrg is the repo org build PLR is triggered by + PipelineAsCodeSourceRepoOrg = "pipelinesascode.tekton.dev/url-org" +) + // AnnotateBuildPipelineRun sets annotation for a build pipelineRun in defined context and returns that pipeline func AnnotateBuildPipelineRun(ctx context.Context, pipelineRun *tektonv1.PipelineRun, key, value string, cl client.Client) error { patch := client.MergeFrom(pipelineRun.DeepCopy()) @@ -40,6 +56,19 @@ func AnnotateBuildPipelineRun(ctx context.Context, pipelineRun *tektonv1.Pipelin return nil } +// LabelBuildPipelineRun sets annotation for a build pipelineRun in defined context and returns that pipeline +func LabelBuildPipelineRun(ctx context.Context, pipelineRun *tektonv1.PipelineRun, key, value string, cl client.Client) error { + patch := client.MergeFrom(pipelineRun.DeepCopy()) + + _ = metadata.SetLabel(&pipelineRun.ObjectMeta, key, value) + + err := cl.Patch(ctx, pipelineRun, patch) + if err != nil { + return err + } + return nil +} + // AnnotateBuildPipelineRunWithCreateSnapshotAnnotation sets annotation test.appstudio.openshift.io/create-snapshot-status to build pipelineRun with // a message that reflects either success or failure for creating a snapshot func AnnotateBuildPipelineRunWithCreateSnapshotAnnotation(ctx context.Context, pipelineRun *tektonv1.PipelineRun, cl client.Client, ensureSnapshotExistsErr error) error { @@ -67,3 +96,21 @@ func AnnotateBuildPipelineRunWithCreateSnapshotAnnotation(ctx context.Context, p } return AnnotateBuildPipelineRun(ctx, pipelineRun, h.CreateSnapshotAnnotationName, string(jsonResult), cl) } + +// GetPRGroupNameFromBuildPLR gets the PR group from the substring before @ from +// the source-branch pac annotation, for main, it generate PR group with {source-branch}-{url-org} +func GetPRGroupNameFromBuildPLR(pipelineRun *tektonv1.PipelineRun) string { + if prGroup, found := pipelineRun.ObjectMeta.Annotations[PipelineAsCodeSourceBranchAnnotation]; found { + if prGroup == MainBranch || prGroup == MasterBranch && metadata.HasAnnotation(pipelineRun, PipelineAsCodeSourceRepoOrg) { + prGroup = prGroup + "-" + pipelineRun.ObjectMeta.Annotations[PipelineAsCodeSourceRepoOrg] + } + return strings.Split(prGroup, "@")[0] + } + return "" +} + +// GenerateSHA generate a 63 charactors sha string used by pipelineRun and snapshot label +func GenerateSHA(str string) string { + hash := sha256.Sum256([]byte(str)) + return fmt.Sprintf("%x", hash)[0:62] +} diff --git a/tekton/build_pipeline_test.go b/tekton/build_pipeline_test.go new file mode 100644 index 000000000..67ee2cc6b --- /dev/null +++ b/tekton/build_pipeline_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 Red Hat Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tekton_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "knative.dev/pkg/apis" + v1 "knative.dev/pkg/apis/duck/v1" + "time" + + tekton "github.com/konflux-ci/integration-service/tekton" + tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("build pipeline", func() { + + var ( + buildPipelineRun *tektonv1.PipelineRun + ) + + BeforeEach(func() { + buildPipelineRun = &tektonv1.PipelineRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pipelinerun-build-sample", + Namespace: "default", + Labels: map[string]string{ + "pipelines.appstudio.openshift.io/type": "build", + "pipelines.openshift.io/used-by": "build-cloud", + "pipelines.openshift.io/runtime": "nodejs", + "pipelines.openshift.io/strategy": "s2i", + "appstudio.openshift.io/component": "component-sample", + "build.appstudio.redhat.com/target_branch": "main", + "pipelinesascode.tekton.dev/event-type": "pull_request", + }, + Annotations: map[string]string{ + "appstudio.redhat.com/updateComponentOnSuccess": "false", + "pipelinesascode.tekton.dev/on-target-branch": "[main,master]", + "build.appstudio.openshift.io/repo": "https://github.com/devfile-samples/devfile-sample-go-basic?rev=c713067b0e65fb3de50d1f7c457eb51c2ab0dbb0", + "foo": "bar", + "chains.tekton.dev/signed": "true", + "pipelinesascode.tekton.dev/source-branch": "sourceBranch", + "pipelinesascode.tekton.dev/url-org": "redhat", + }, + }, + Spec: tektonv1.PipelineRunSpec{ + PipelineRef: &tektonv1.PipelineRef{ + Name: "build-pipeline-pass", + ResolverRef: tektonv1.ResolverRef{ + Resolver: "bundle", + Params: tektonv1.Params{ + {Name: "bundle", + Value: tektonv1.ParamValue{Type: "string", StringVal: "quay.io/redhat-appstudio/example-tekton-bundle:test"}, + }, + {Name: "name", + Value: tektonv1.ParamValue{Type: "string", StringVal: "test-task"}, + }, + }, + }, + }, + Params: []tektonv1.Param{ + { + Name: "output-image", + Value: tektonv1.ParamValue{ + Type: tektonv1.ParamTypeString, + StringVal: "quay.io/redhat-appstudio/example-tekton-bundle:test", + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, buildPipelineRun)).Should(Succeed()) + + buildPipelineRun.Status = tektonv1.PipelineRunStatus{ + PipelineRunStatusFields: tektonv1.PipelineRunStatusFields{ + StartTime: &metav1.Time{Time: time.Now()}, + }, + Status: v1.Status{ + Conditions: v1.Conditions{ + apis.Condition{ + Reason: "Completed", + Status: "True", + Type: apis.ConditionSucceeded, + }, + }, + }, + } + Expect(k8sClient.Status().Update(ctx, buildPipelineRun)).Should(Succeed()) + }) + + AfterEach(func() { + err := k8sClient.Delete(ctx, buildPipelineRun) + Expect(err == nil || k8serrors.IsNotFound(err)).To(BeTrue()) + }) + + Context("When a build pipelineRun exists", func() { + It("can get PR group from build pipelineRun", func() { + prGroup := tekton.GetPRGroupNameFromBuildPLR(buildPipelineRun) + Expect(prGroup).To(Equal("sourceBranch")) + Expect(tekton.GenerateSHA(prGroup)).NotTo(BeNil()) + }) + + It("can get PR group from build pipelineRun is source branch is main", func() { + buildPipelineRun.Annotations[tekton.PipelineAsCodeSourceBranchAnnotation] = "main" + prGroup := tekton.GetPRGroupNameFromBuildPLR(buildPipelineRun) + Expect(prGroup).To(Equal("main-redhat")) + Expect(tekton.GenerateSHA(prGroup)).NotTo(BeNil()) + }) + + It("can get PR group from build pipelineRun is source branch has @ charactor", func() { + buildPipelineRun.Annotations[tekton.PipelineAsCodeSourceBranchAnnotation] = "myfeature@change1" + prGroup := tekton.GetPRGroupNameFromBuildPLR(buildPipelineRun) + Expect(prGroup).To(Equal("myfeature")) + Expect(tekton.GenerateSHA(prGroup)).NotTo(BeNil()) + }) + }) +})