From 844d9e867358c0ab84bf9ec6b32725053240ca8c Mon Sep 17 00:00:00 2001 From: liamfallon Date: Thu, 28 Nov 2024 13:36:58 +0000 Subject: [PATCH] rebase --- api/porch/types.go | 94 ------------ pkg/engine/engine.go | 340 +------------------------------------------ 2 files changed, 2 insertions(+), 432 deletions(-) diff --git a/api/porch/types.go b/api/porch/types.go index f1d0b837..c903ce10 100644 --- a/api/porch/types.go +++ b/api/porch/types.go @@ -530,97 +530,3 @@ type PackageStatus struct { // published package revision belonging to this package LatestRevision string `json:"latestRevision,omitempty"` } - -// Function represents a kpt function discovered in a repository -// Function resources are created automatically by discovery in a registered Repository. -// Function resource names will be computed as : -// to ensure uniqueness of names, and will follow formatting of -// [DNS Subdomain Names](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names). -// +genclient -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// +k8s:openapi-gen=true -type Function struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec FunctionSpec `json:"spec,omitempty"` - Status FunctionStatus `json:"status,omitempty"` -} - -// FunctionList -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -type FunctionList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - - Items []Function `json:"items"` -} - -type FunctionType string - -const ( - FunctionTypeValidator FunctionType = "validator" - FunctionTypeMutator FunctionType = "mutator" -) - -// FunctionSpec defines the desired state of a Function -type FunctionSpec struct { - // Image specifies the function image, such as 'gcr.io/kpt-fn/gatekeeper:v0.2'. - Image string `json:"image"` - - // RepositoryRef references the repository in which the function is located. - RepositoryRef RepositoryRef `json:"repositoryRef"` - - // FunctionType specifies the function types (mutator, validator or/and others). - FunctionTypes []FunctionType `json:"functionTypes,omitempty"` - - FunctionConfigs []FunctionConfig `json:"functionConfigs,omitempty"` - - // Keywords are used as filters to provide correlation in function discovery. - Keywords []string `json:"keywords,omitempty"` - - // Description is a short description of the function. - Description string `json:"description"` - - // `DocumentationUrl specifies the URL of comprehensive function documentation` - DocumentationUrl string `json:"documentationUrl,omitempty"` - - // InputTypes specifies to which input KRM types the function applies. Specified as Group Version Kind. - // For example: - // - // inputTypes: - // - kind: RoleBinding - // # If version is unspecified, applies to all versions - // apiVersion: rbac.authorization.k8s.io - // - kind: ClusterRoleBinding - // apiVersion: rbac.authorization.k8s.io/v1 - // InputTypes []metav1.TypeMeta - - // OutputTypes specifies types of any KRM resources the function creates - // For example: - // - // outputTypes: - // - kind: ConfigMap - // apiVersion: v1 - // OutputTypes []metav1.TypeMeta - -} - -// FunctionConfig specifies all the valid types of the function config for this function. -// If unspecified, defaults to v1/ConfigMap. For example, function `set-namespace` accepts both `ConfigMap` and `SetNamespace` -type FunctionConfig struct { - metav1.TypeMeta `json:",inline"` - // Experimental: requiredFields tells necessary fields and is aimed to help users write the FunctionConfig. - // Otherwise, users can get the required fields info from the function evaluation error message. - RequiredFields []string `json:"requiredFields,omitempty"` -} - -// FunctionRef is a reference to a Function resource. -type FunctionRef struct { - // Name is the name of the Function resource referenced. The resource is expected to be within the same namespace. - Name string `json:"name"` -} - -// FunctionStatus defines the observed state of Function -type FunctionStatus struct { -} diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index cea17e2a..6893e579 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -503,350 +503,14 @@ func (cad *cadEngine) UpdatePackageResources(ctx context.Context, repositoryObj return nil, nil, err } - runnerOptions := cad.runnerOptionsResolver(old.GetNamespace()) - - mutations := []mutation{ - &mutationReplaceResources{ - newResources: new, - oldResources: old, - }, - } - prevResources, err := oldPackage.repoPackageRevision.GetResources(ctx) - if err != nil { - return nil, nil, fmt.Errorf("cannot get package resources: %w", err) - } - resources := repository.PackageResources{ - Contents: prevResources.Spec.Resources, - } - appliedResources, renderStatus, err := applyResourceMutations(ctx, draft, resources, mutations) - if err != nil { - return nil, nil, err - } - - if len(appliedResources.Contents) > 0 { - // render the package - // Render failure will not fail the overall API operation. - // The render error and result is captured as part of renderStatus above - // and is returned in packageresourceresources API's status field. We continue with - // saving the non-rendered resources to avoid losing user's changes. - // and supress this err. - _, renderStatus, _ = applyResourceMutations(ctx, - draft, - appliedResources, - []mutation{&renderPackageMutation{ - runnerOptions: runnerOptions, - runtime: cad.runtime, - }}) - } + renderStatus, err := cad.taskHandler.DoPRResourceMutations(ctx, pr2Update, draft, oldRes, newRes) // No lifecycle change when updating package resources; updates are done. repoPkgRev, err := draft.Close(ctx, "") if err != nil { return nil, renderStatus, err } - return &PackageRevision{ - repoPackageRevision: repoPkgRev, - }, renderStatus, nil -} - -// applyResourceMutations mutates the resources and returns the most recent renderResult. -func applyResourceMutations(ctx context.Context, draft repository.PackageDraft, baseResources repository.PackageResources, mutations []mutation) (applied repository.PackageResources, renderStatus *api.RenderStatus, err error) { - var lastApplied mutation - for _, m := range mutations { - updatedResources, taskResult, err := m.Apply(ctx, baseResources) - if taskResult == nil && err == nil { - // a nil taskResult means nothing changed - continue - } - - var task *api.Task - if taskResult != nil { - task = taskResult.Task - } - if taskResult != nil && task.Type == api.TaskTypeEval { - renderStatus = taskResult.RenderStatus - } - if err != nil { - return updatedResources, renderStatus, err - } - - // if the last applied mutation was a render mutation, and so is this one, skip it - if lastApplied != nil && isRenderMutation(m) && isRenderMutation(lastApplied) { - continue - } - lastApplied = m - - if err := draft.UpdateResources(ctx, &api.PackageRevisionResources{ - Spec: api.PackageRevisionResourcesSpec{ - Resources: updatedResources.Contents, - }, - }, task); err != nil { - return updatedResources, renderStatus, err - } - baseResources = updatedResources - applied = updatedResources - } - - return applied, renderStatus, nil -} - -type updatePackageMutation struct { - cloneTask *api.Task - updateTask *api.Task - repoOpener RepositoryOpener - referenceResolver ReferenceResolver - namespace string - pkgName string -} - -func (m *updatePackageMutation) Apply(ctx context.Context, resources repository.PackageResources) (repository.PackageResources, *api.TaskResult, error) { - ctx, span := tracer.Start(ctx, "updatePackageMutation::Apply", trace.WithAttributes()) - defer span.End() - - currUpstreamPkgRef, err := m.currUpstream() - if err != nil { - return repository.PackageResources{}, nil, err - } - - targetUpstream := m.updateTask.Update.Upstream - if targetUpstream.Type == api.RepositoryTypeGit || targetUpstream.Type == api.RepositoryTypeOCI || targetUpstream.Type == api.RepositoryTypeDB { - return repository.PackageResources{}, nil, fmt.Errorf("update is not supported for non-porch upstream packages") - } - - originalResources, err := (&PackageFetcher{ - repoOpener: m.repoOpener, - referenceResolver: m.referenceResolver, - }).FetchResources(ctx, currUpstreamPkgRef, m.namespace) - if err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("error fetching the resources for package %s with ref %+v", - m.pkgName, *currUpstreamPkgRef) - } - - upstreamRevision, err := (&PackageFetcher{ - repoOpener: m.repoOpener, - referenceResolver: m.referenceResolver, - }).FetchRevision(ctx, targetUpstream.UpstreamRef, m.namespace) - if err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("error fetching revision for target upstream %s", targetUpstream.UpstreamRef.Name) - } - upstreamResources, err := upstreamRevision.GetResources(ctx) - if err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("error fetching resources for target upstream %s", targetUpstream.UpstreamRef.Name) - } - - klog.Infof("performing pkg upgrade operation for pkg %s resource counts local[%d] original[%d] upstream[%d]", - m.pkgName, len(resources.Contents), len(originalResources.Spec.Resources), len(upstreamResources.Spec.Resources)) - - // May be have packageUpdater part of engine to make it easy for testing ? - updatedResources, err := (&defaultPackageUpdater{}).Update(ctx, - resources, - repository.PackageResources{ - Contents: originalResources.Spec.Resources, - }, - repository.PackageResources{ - Contents: upstreamResources.Spec.Resources, - }) - if err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("error updating the package to revision %s", targetUpstream.UpstreamRef.Name) - } - - newUpstream, newUpstreamLock, err := upstreamRevision.GetLock() - if err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("error fetching the resources for package revisions %s", targetUpstream.UpstreamRef.Name) - } - if err := kpt.UpdateKptfileUpstream("", updatedResources.Contents, newUpstream, newUpstreamLock); err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("failed to apply upstream lock to package %q: %w", m.pkgName, err) - } - - // ensure merge-key comment is added to newly added resources. - result, err := ensureMergeKey(ctx, updatedResources) - if err != nil { - klog.Infof("failed to add merge key comments: %v", err) - } - return result, &api.TaskResult{Task: m.updateTask}, nil -} - -// Currently assumption is that downstream packages will be forked from a porch package. -// As per current implementation, upstream package ref is stored in a new update task but this may -// change so the logic of figuring out current upstream will live in this function. -func (m *updatePackageMutation) currUpstream() (*api.PackageRevisionRef, error) { - if m.cloneTask == nil || m.cloneTask.Clone == nil { - return nil, fmt.Errorf("package %s does not have original upstream info", m.pkgName) - } - upstream := m.cloneTask.Clone.Upstream - if upstream.Type == api.RepositoryTypeGit || upstream.Type == api.RepositoryTypeOCI || upstream.Type == api.RepositoryTypeDB { - return nil, fmt.Errorf("upstream package must be porch native package. Found it to be %s", upstream.Type) - } - return upstream.UpstreamRef, nil -} - -func findCloneTask(pr *api.PackageRevision) *api.Task { - if len(pr.Spec.Tasks) == 0 { - return nil - } - firstTask := pr.Spec.Tasks[0] - if firstTask.Type == api.TaskTypeClone { - return &firstTask - } - return nil -} - -func writeResourcesToDirectory(dir string, resources repository.PackageResources) error { - for k, v := range resources.Contents { - p := filepath.Join(dir, k) - dir := filepath.Dir(p) - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("failed to create directory %q: %w", dir, err) - } - if err := os.WriteFile(p, []byte(v), 0644); err != nil { - return fmt.Errorf("failed to write file %q: %w", dir, err) - } - } - return nil -} - -func loadResourcesFromDirectory(dir string) (repository.PackageResources, error) { - // TODO: return abstraction instead of loading everything - result := repository.PackageResources{ - Contents: map[string]string{}, - } - if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - rel, err := filepath.Rel(dir, path) - if err != nil { - return fmt.Errorf("cannot compute relative path %q, %q, %w", dir, path, err) - } - - contents, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("cannot read file %q: %w", dir, err) - } - result.Contents[rel] = string(contents) - return nil - }); err != nil { - return repository.PackageResources{}, err - } - - return result, nil -} - -type mutationReplaceResources struct { - newResources *api.PackageRevisionResources - oldResources *api.PackageRevisionResources -} - -func (m *mutationReplaceResources) Apply(ctx context.Context, resources repository.PackageResources) (repository.PackageResources, *api.TaskResult, error) { - ctx, span := tracer.Start(ctx, "mutationReplaceResources::Apply", trace.WithAttributes()) - defer span.End() - - patch := &api.PackagePatchTaskSpec{} - - old := resources.Contents - new, err := healConfig(old, m.newResources.Spec.Resources) - if err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("failed to heal resources: %w", err) - } - - for k, newV := range new { - oldV, ok := old[k] - // New config or changed config - if !ok { - patchSpec := api.PatchSpec{ - File: k, - PatchType: api.PatchTypeCreateFile, - Contents: newV, - } - patch.Patches = append(patch.Patches, patchSpec) - } else if newV != oldV { - patchSpec, err := GeneratePatch(k, oldV, newV) - if err != nil { - return repository.PackageResources{}, nil, fmt.Errorf("error generating patch: %w", err) - } - if patchSpec.Contents == "" { - continue - } - patch.Patches = append(patch.Patches, patchSpec) - } - } - for k := range old { - // Deleted config - if _, ok := new[k]; !ok { - patchSpec := api.PatchSpec{ - File: k, - PatchType: api.PatchTypeDeleteFile, - } - patch.Patches = append(patch.Patches, patchSpec) - } - } - // If patch is empty, don't create a Task. - var taskResult *api.TaskResult - if len(patch.Patches) > 0 { - taskResult = &api.TaskResult{ - Task: &api.Task{ - Type: api.TaskTypePatch, - Patch: patch, - }, - } - } - return repository.PackageResources{Contents: new}, taskResult, nil -} - -func healConfig(old, new map[string]string) (map[string]string, error) { - // Copy comments from old config to new - oldResources, err := (&packageReader{ - input: repository.PackageResources{Contents: old}, - extra: map[string]string{}, - }).Read() - if err != nil { - return nil, fmt.Errorf("failed to read old packge resources: %w", err) - } - - var filter kio.FilterFunc = func(r []*yaml.RNode) ([]*yaml.RNode, error) { - for _, n := range r { - for _, original := range oldResources { - if n.GetNamespace() == original.GetNamespace() && - n.GetName() == original.GetName() && - n.GetApiVersion() == original.GetApiVersion() && - n.GetKind() == original.GetKind() { - comments.CopyComments(original, n) - } - } - } - return r, nil - } - - out := &packageWriter{ - output: repository.PackageResources{ - Contents: map[string]string{}, - }, - } - - extra := map[string]string{} - - if err := (kio.Pipeline{ - Inputs: []kio.Reader{&packageReader{ - input: repository.PackageResources{Contents: new}, - extra: extra, - }}, - Filters: []kio.Filter{filter}, - Outputs: []kio.Writer{out}, - ContinueOnEmptyResult: true, - }).Execute(); err != nil { - return nil, err - } - - healed := out.output.Contents - - for k, v := range extra { - healed[k] = v - } - - return healed, nil + return repoPkgRev, renderStatus, nil } // isRecloneAndReplay determines if an update should be handled using reclone-and-replay semantics.