diff --git a/pkg/apis/v1alpha1/conditions.go b/pkg/apis/v1alpha1/conditions.go index 6c75b3361..776d40408 100644 --- a/pkg/apis/v1alpha1/conditions.go +++ b/pkg/apis/v1alpha1/conditions.go @@ -159,15 +159,14 @@ const ( // -- RUNNABLE ConditionType - RunTemplateReady ConditionReasons const ( - ReadyRunTemplateReason = "Ready" - NotFoundRunTemplateReason = "RunTemplateNotFound" - StampedObjectRejectedByAPIServerRunTemplateReason = "StampedObjectRejectedByAPIServer" - OutputPathNotSatisfiedRunTemplateReason = "OutputPathNotSatisfied" - TemplateStampFailureRunTemplateReason = "TemplateStampFailure" - FailedToListCreatedObjectsReason = "FailedToListCreatedObjects" - SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason = "SetOfImmutableStampedObjectsIncludesNoHealthyObject" - UnknownErrorReason = "UnknownError" - ClientBuilderErrorResourcesSubmittedReason = "ClientBuilderError" - SucceededStampedObjectConditionReason = "SucceededCondition" - UnknownStampedObjectConditionReason = "Unknown" + ReadyRunTemplateReason = "Ready" + NotFoundRunTemplateReason = "RunTemplateNotFound" + StampedObjectRejectedByAPIServerRunTemplateReason = "StampedObjectRejectedByAPIServer" + OutputPathNotSatisfiedRunTemplateReason = "OutputPathNotSatisfied" + TemplateStampFailureRunTemplateReason = "TemplateStampFailure" + FailedToListCreatedObjectsReason = "FailedToListCreatedObjects" + UnknownErrorReason = "UnknownError" + ClientBuilderErrorResourcesSubmittedReason = "ClientBuilderError" + SucceededStampedObjectConditionReason = "SucceededCondition" + UnknownStampedObjectConditionReason = "Unknown" ) diff --git a/pkg/conditions/deliverable_conditions.go b/pkg/conditions/deliverable_conditions.go index ec2ac04c9..0bc0448e8 100644 --- a/pkg/conditions/deliverable_conditions.go +++ b/pkg/conditions/deliverable_conditions.go @@ -120,7 +120,7 @@ func AddConditionForResourceSubmittedDeliverable(conditionManager *ConditionMana if typedErr.StampedObject == nil { (*conditionManager).AddPositive(MissingPassThroughInputCondition(typedErr.PassThroughInput, typedErr.GetQualifiedResource())) } else { - (*conditionManager).AddPositive(MissingValueAtPathCondition(isOwner, typedErr.StampedObject, typedErr.JsonPathExpression(), typedErr.GetQualifiedResource())) + (*conditionManager).AddPositive(MissingValueAtPathCondition(isOwner, typedErr.StampedObject, typedErr.JsonPathExpression(), typedErr.GetQualifiedResource(), typedErr.Healthy)) } default: (*conditionManager).AddPositive(UnknownResourceErrorCondition(isOwner, typedErr)) diff --git a/pkg/conditions/owner_conditions.go b/pkg/conditions/owner_conditions.go index e4a82c3e3..af6645bdd 100644 --- a/pkg/conditions/owner_conditions.go +++ b/pkg/conditions/owner_conditions.go @@ -61,17 +61,31 @@ func TemplateObjectRetrievalFailureCondition(isOwner bool, err error) metav1.Con } } -func MissingValueAtPathCondition(isOwner bool, obj *unstructured.Unstructured, expression string, qualifiedResource string) metav1.Condition { +func MissingValueAtPathCondition(isOwner bool, obj *unstructured.Unstructured, expression string, qualifiedResource string, health metav1.ConditionStatus) metav1.Condition { var namespaceMsg string if obj.GetNamespace() != "" { namespaceMsg = fmt.Sprintf(" in namespace [%s]", obj.GetNamespace()) } + + var message string + + switch health { + case metav1.ConditionTrue: + message = fmt.Sprintf("cannot read value [%s] from healthy object [%s/%s]%s, contact Platform Eng", + expression, qualifiedResource, obj.GetName(), namespaceMsg) + case metav1.ConditionFalse: + message = fmt.Sprintf("cannot read value [%s] from unhealthy object [%s/%s]%s, examine object, particularly whether it is receiving proper inputs", + expression, qualifiedResource, obj.GetName(), namespaceMsg) + default: + message = fmt.Sprintf("waiting to read value [%s] from object [%s/%s]%s", + expression, qualifiedResource, obj.GetName(), namespaceMsg) + } + return metav1.Condition{ - Type: getConditionType(isOwner), - Status: metav1.ConditionUnknown, - Reason: v1alpha1.MissingValueAtPathResourcesSubmittedReason, - Message: fmt.Sprintf("waiting to read value [%s] from resource [%s/%s]%s", - expression, qualifiedResource, obj.GetName(), namespaceMsg), + Type: getConditionType(isOwner), + Status: metav1.ConditionUnknown, + Reason: v1alpha1.MissingValueAtPathResourcesSubmittedReason, + Message: message, } } @@ -102,15 +116,6 @@ func BlueprintsFailedToListCreatedObjectsCondition(isOwner bool, err error) meta } } -func NoHealthyImmutableObjectsCondition(isOwner bool, err error) metav1.Condition { - return metav1.Condition{ - Type: getConditionType(isOwner), - Status: metav1.ConditionFalse, - Reason: v1alpha1.SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason, - Message: err.Error(), - } -} - func UnknownResourceErrorCondition(isOwner bool, err error) metav1.Condition { return metav1.Condition{ Type: getConditionType(isOwner), diff --git a/pkg/conditions/owner_conditions_test.go b/pkg/conditions/owner_conditions_test.go index c8cfaca3b..8a4767494 100644 --- a/pkg/conditions/owner_conditions_test.go +++ b/pkg/conditions/owner_conditions_test.go @@ -17,6 +17,7 @@ package conditions_test import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,18 +39,49 @@ var _ = Describe("Conditions", func() { }) Context("stamped object has a namespace", func() { - It("has the correct message", func() { + var healthy metav1.ConditionStatus + BeforeEach(func() { obj.SetNamespace("my-ns") + }) + + Context("healthy is true", func() { + BeforeEach(func() { + healthy = metav1.ConditionTrue + }) + + It("has the correct message", func() { + condition := conditions.MissingValueAtPathCondition(true, obj, "spec.foo", "widget.thing.io", healthy) + Expect(condition.Message).To(Equal("cannot read value [spec.foo] from healthy object [widget.thing.io/my-widget] in namespace [my-ns], contact Platform Eng")) + }) + }) + + Context("healthy is false", func() { + BeforeEach(func() { + healthy = metav1.ConditionFalse + }) + + It("has the correct message", func() { + condition := conditions.MissingValueAtPathCondition(true, obj, "spec.foo", "widget.thing.io", healthy) + Expect(condition.Message).To(Equal("cannot read value [spec.foo] from unhealthy object [widget.thing.io/my-widget] in namespace [my-ns], examine object, particularly whether it is receiving proper inputs")) + }) + }) + + Context("healthy is unknown", func() { + BeforeEach(func() { + healthy = metav1.ConditionUnknown + }) - condition := conditions.MissingValueAtPathCondition(true, obj, "spec.foo", "widget.thing.io") - Expect(condition.Message).To(Equal("waiting to read value [spec.foo] from resource [widget.thing.io/my-widget] in namespace [my-ns]")) + It("has the correct message", func() { + condition := conditions.MissingValueAtPathCondition(true, obj, "spec.foo", "widget.thing.io", healthy) + Expect(condition.Message).To(Equal("waiting to read value [spec.foo] from object [widget.thing.io/my-widget] in namespace [my-ns]")) + }) }) }) Context("stamped object does not have a namespace", func() { It("has the correct message", func() { - condition := conditions.MissingValueAtPathCondition(true, obj, "spec.foo", "widget.thing.io") - Expect(condition.Message).To(Equal("waiting to read value [spec.foo] from resource [widget.thing.io/my-widget]")) + condition := conditions.MissingValueAtPathCondition(true, obj, "spec.foo", "widget.thing.io", metav1.ConditionUnknown) + Expect(condition.Message).To(Equal("waiting to read value [spec.foo] from object [widget.thing.io/my-widget]")) }) }) }) diff --git a/pkg/conditions/workload_conditions.go b/pkg/conditions/workload_conditions.go index fa297c451..ee1b863f1 100644 --- a/pkg/conditions/workload_conditions.go +++ b/pkg/conditions/workload_conditions.go @@ -88,13 +88,11 @@ func AddConditionForResourceSubmittedWorkload(conditionManager *ConditionManager (*conditionManager).AddPositive(TemplateRejectedByAPIServerCondition(isOwner, typedErr)) case cerrors.ListCreatedObjectsError: (*conditionManager).AddPositive(BlueprintsFailedToListCreatedObjectsCondition(isOwner, typedErr)) - case cerrors.NoHealthyImmutableObjectsError: - (*conditionManager).AddPositive(NoHealthyImmutableObjectsCondition(isOwner, typedErr)) case cerrors.RetrieveOutputError: if typedErr.StampedObject == nil { (*conditionManager).AddPositive(MissingPassThroughInputCondition(typedErr.PassThroughInput, typedErr.GetQualifiedResource())) } else { - (*conditionManager).AddPositive(MissingValueAtPathCondition(isOwner, typedErr.StampedObject, typedErr.JsonPathExpression(), typedErr.GetQualifiedResource())) + (*conditionManager).AddPositive(MissingValueAtPathCondition(isOwner, typedErr.StampedObject, typedErr.JsonPathExpression(), typedErr.GetQualifiedResource(), typedErr.Healthy)) } case cerrors.ResolveTemplateOptionError: (*conditionManager).AddPositive(ResolveTemplateOptionsErrorCondition(isOwner, typedErr)) diff --git a/pkg/controllers/deliverable_reconciler_test.go b/pkg/controllers/deliverable_reconciler_test.go index f01845269..0042f2d6f 100644 --- a/pkg/controllers/deliverable_reconciler_test.go +++ b/pkg/controllers/deliverable_reconciler_test.go @@ -796,6 +796,7 @@ var _ = Describe("DeliverableReconciler", func() { var retrieveError cerrors.RetrieveOutputError var wrappedError error var stampedObject *unstructured.Unstructured + var healthy metav1.ConditionStatus JustBeforeEach(func() { stampedObject = &unstructured.Unstructured{} @@ -814,6 +815,7 @@ var _ = Describe("DeliverableReconciler", func() { BlueprintType: cerrors.Delivery, StampedObject: stampedObject, QualifiedResource: "mything.thing.io", + Healthy: healthy, } rlzr.RealizeStub = func(ctx context.Context, resourceRealizer realizer.ResourceRealizer, deliveryName string, resources []realizer.OwnerResource, statuses statuses.ResourceStatuses) error { @@ -950,9 +952,37 @@ var _ = Describe("DeliverableReconciler", func() { wrappedError = stamp.NewJsonPathError("this.wont.find.anything", errors.New("some error")) }) - It("calls the condition manager to report", func() { - _, _ = reconciler.Reconcile(ctx, req) - Expect(conditionManager.AddPositiveArgsForCall(1)).To(Equal(conditions.MissingValueAtPathCondition(true, stampedObject, "this.wont.find.anything", "mything.thing.io"))) + Context("and the RetrieveOutputError reports object as healthy", func() { + BeforeEach(func() { + healthy = metav1.ConditionTrue + }) + + It("calls the condition manager to report", func() { + _, _ = reconciler.Reconcile(ctx, req) + Expect(conditionManager.AddPositiveArgsForCall(1)).To(Equal(conditions.MissingValueAtPathCondition(true, stampedObject, "this.wont.find.anything", "mything.thing.io", metav1.ConditionTrue))) + }) + }) + + Context("and the RetrieveOutputError reports object as unhealthy", func() { + BeforeEach(func() { + healthy = metav1.ConditionFalse + }) + + It("calls the condition manager to report", func() { + _, _ = reconciler.Reconcile(ctx, req) + Expect(conditionManager.AddPositiveArgsForCall(1)).To(Equal(conditions.MissingValueAtPathCondition(true, stampedObject, "this.wont.find.anything", "mything.thing.io", metav1.ConditionFalse))) + }) + }) + + Context("and the RetrieveOutputError reports object health as unknown", func() { + BeforeEach(func() { + healthy = metav1.ConditionUnknown + }) + + It("calls the condition manager to report", func() { + _, _ = reconciler.Reconcile(ctx, req) + Expect(conditionManager.AddPositiveArgsForCall(1)).To(Equal(conditions.MissingValueAtPathCondition(true, stampedObject, "this.wont.find.anything", "mything.thing.io", metav1.ConditionUnknown))) + }) }) It("does not return an error", func() { diff --git a/pkg/controllers/workload_reconciler_test.go b/pkg/controllers/workload_reconciler_test.go index 430c1dcff..81df1751d 100644 --- a/pkg/controllers/workload_reconciler_test.go +++ b/pkg/controllers/workload_reconciler_test.go @@ -850,8 +850,9 @@ var _ = Describe("WorkloadReconciler", func() { It("calls the condition manager to report", func() { _, _ = reconciler.Reconcile(ctx, req) - Expect(conditionManager.AddPositiveArgsForCall(1)).To( - Equal(conditions.MissingValueAtPathCondition(true, stampedObject, "this.wont.find.anything", "mything.thing.io"))) + var emptyConditionStatus metav1.ConditionStatus + Expect(conditionManager.AddPositiveArgsForCall(1)). + To(Equal(conditions.MissingValueAtPathCondition(true, stampedObject, "this.wont.find.anything", "mything.thing.io", emptyConditionStatus))) }) It("does not return an error", func() { diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index d74a42853..58c728218 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -19,6 +19,7 @@ import ( "strings" kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -131,6 +132,7 @@ type RetrieveOutputError struct { BlueprintType string QualifiedResource string PassThroughInput string + Healthy metav1.ConditionStatus } func (e RetrieveOutputError) Error() string { @@ -193,15 +195,6 @@ type NoHealthyImmutableObjectsError struct { BlueprintType string } -func (e NoHealthyImmutableObjectsError) Error() string { - return fmt.Errorf("unable to retrieve outputs for resource [%s] in %s [%s]: %w", - e.ResourceName, - e.BlueprintType, - e.BlueprintName, - e.Err, - ).Error() -} - func WrapUnhandledError(err error) error { if IsUnhandledErrorType(err) { return NewUnhandledError(err) diff --git a/pkg/realizer/component.go b/pkg/realizer/component.go index 181751af8..108722530 100644 --- a/pkg/realizer/component.go +++ b/pkg/realizer/component.go @@ -206,25 +206,23 @@ func (r *resourceRealizer) doImmutable(ctx context.Context, resource OwnerResour var output *templates.Output - if latestSuccessfulObject == nil { - for _, obj := range allRunnableStampedObjects { - log.V(logger.DEBUG).Info("failed to retrieve output from any object", "considered", obj) - } - - return template, stampedObject, nil, passThrough, templateName, errors.NoHealthyImmutableObjectsError{ - Err: fmt.Errorf("failed to find any healthy object in the set of immutable stamped objects"), - ResourceName: resource.Name, - BlueprintName: blueprintName, - BlueprintType: errors.SupplyChain, - } - } - output, err = stampReader.Output(latestSuccessfulObject) if err != nil { - qualifiedResource, rErr := utils.GetQualifiedResource(mapper, latestSuccessfulObject) + var ( + qualifiedResource string + rErr error + objectToReport *unstructured.Unstructured + ) + if latestSuccessfulObject == nil { + objectToReport = stampedObject + } else { + objectToReport = latestSuccessfulObject + } + + qualifiedResource, rErr = utils.GetQualifiedResource(mapper, objectToReport) if rErr != nil { - log.Error(err, "failed to retrieve qualified resource name", "object", latestSuccessfulObject) + log.Error(err, "failed to retrieve qualified resource name", "object", objectToReport) qualifiedResource = "could not fetch - see the log line for 'failed to retrieve qualified resource name'" } diff --git a/pkg/realizer/component_test.go b/pkg/realizer/component_test.go index 8408a6278..94de5931d 100644 --- a/pkg/realizer/component_test.go +++ b/pkg/realizer/component_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gstruct" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,10 +37,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/vmware-tanzu/cartographer/pkg/apis/v1alpha1" + cerrors "github.com/vmware-tanzu/cartographer/pkg/errors" "github.com/vmware-tanzu/cartographer/pkg/realizer" "github.com/vmware-tanzu/cartographer/pkg/realizer/realizerfakes" "github.com/vmware-tanzu/cartographer/pkg/repository" "github.com/vmware-tanzu/cartographer/pkg/repository/repositoryfakes" + "github.com/vmware-tanzu/cartographer/pkg/stamp" "github.com/vmware-tanzu/cartographer/pkg/templates" ) @@ -264,26 +267,37 @@ var _ = Describe("Resource", func() { When("call to list objects succeeds", func() { BeforeEach(func() { - stampedObjectWithTime := expectedObject.DeepCopy() - - stampedObjectWithTime.SetCreationTimestamp(metav1.NewTime(time.Unix(1, 0))) - - fakeOwnerRepo.ListUnstructuredReturns([]*unstructured.Unstructured{stampedObjectWithTime}, nil) + templateAPI.Spec.TemplateSpec.HealthRule = &v1alpha1.HealthRule{ + SingleConditionType: "Succeeded", + } }) - When("no returned object meets the healthRule", func() { + When("a healthy object is returned", func() { BeforeEach(func() { - templateAPI.Spec.TemplateSpec.HealthRule = &v1alpha1.HealthRule{ - SingleConditionType: "Ready", + stampedObjectWithTime := expectedObject.DeepCopy() + stampedObjectWithTime.SetCreationTimestamp(metav1.NewTime(time.Unix(1, 0))) + stampedObjectWithTime.Object["status"] = map[string]any{ + "conditions": []map[string]any{ + { + "type": "Succeeded", + "status": "True", + "lastTransitionTime": "2023-08-17T14:30:28Z", + "reason": "", + "message": "", + }, + }, } + + fakeOwnerRepo.ListUnstructuredReturns([]*unstructured.Unstructured{stampedObjectWithTime}, nil) }) - It("creates a stamped object, but returns an error and no output", func() { + + It("creates a stamped object and returns the outputs and stampedObjects", func() { template, returnedStampedObject, out, isPassThrough, templateRefName, err := r.Do(ctx, resource, blueprintName, outputs, fakeMapper) + Expect(err).NotTo(HaveOccurred()) Expect(template).ToNot(BeNil()) Expect(isPassThrough).To(BeFalse()) Expect(templateRefName).To(Equal("image-template-1")) Expect(returnedStampedObject.Object).To(Equal(expectedObject.Object)) - Expect(out).To(BeNil()) Expect(fakeOwnerRepo.EnsureImmutableObjectExistsOnClusterCallCount()).To(Equal(1)) @@ -308,21 +322,34 @@ var _ = Describe("Resource", func() { Expect(stampedObject.Object["data"]).To(Equal(map[string]interface{}{"player_current_lives": "some-url", "some_other_info": "some-revision"})) Expect(metadataValues["labels"]).To(Equal(map[string]interface{}{"expected-labels-from-labeler-placeholder": "labeler"})) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("unable to retrieve outputs for resource [resource-1] in supply chain [supply-chain-name]: failed to find any healthy object in the set of immutable stamped objects")) - Expect(reflect.TypeOf(err).String()).To(Equal("errors.NoHealthyImmutableObjectsError")) + Expect(out.Source.Revision).To(Equal("some-revision")) + Expect(out.Source.URL).To(Equal("some-url")) }) }) - When("at least one returned object meets the healthRule", func() { + When("no healthy object is returned", func() { BeforeEach(func() { - templateAPI.Spec.TemplateSpec.HealthRule = &v1alpha1.HealthRule{ - AlwaysHealthy: &runtime.RawExtension{Raw: []byte{}}, - } + stampedObjectWithTime := expectedObject.DeepCopy() + stampedObjectWithTime.SetCreationTimestamp(metav1.NewTime(time.Unix(1, 0))) + fakeOwnerRepo.ListUnstructuredReturns([]*unstructured.Unstructured{stampedObjectWithTime}, nil) + + fakeMapper.RESTMappingReturns(&meta.RESTMapping{ + Resource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmap", + }, + GroupVersionKind: schema.GroupVersionKind{ + Group: "", + Version: "", + Kind: "", + }, + Scope: nil, + }, nil) }) - It("creates a stamped object and returns the outputs and stampedObjects", func() { - template, returnedStampedObject, out, isPassThrough, templateRefName, err := r.Do(ctx, resource, blueprintName, outputs, fakeMapper) - Expect(err).ToNot(HaveOccurred()) + + It("returns the expected outputs", func() { + template, returnedStampedObject, out, isPassThrough, templateRefName, _ := r.Do(ctx, resource, blueprintName, outputs, fakeMapper) Expect(template).ToNot(BeNil()) Expect(isPassThrough).To(BeFalse()) Expect(templateRefName).To(Equal("image-template-1")) @@ -351,8 +378,34 @@ var _ = Describe("Resource", func() { Expect(stampedObject.Object["data"]).To(Equal(map[string]interface{}{"player_current_lives": "some-url", "some_other_info": "some-revision"})) Expect(metadataValues["labels"]).To(Equal(map[string]interface{}{"expected-labels-from-labeler-placeholder": "labeler"})) - Expect(out.Source.Revision).To(Equal("some-revision")) - Expect(out.Source.URL).To(Equal("some-url")) + Expect(out).To(BeNil()) + }) + + It("returns the expected error", func() { + _, _, _, _, _, err := r.Do(ctx, resource, blueprintName, outputs, fakeMapper) + Expect(err).To(HaveOccurred()) + + Expect(err).To(BeAssignableToTypeOf(cerrors.RetrieveOutputError{})) + + Expect(err).To(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "BlueprintName": Equal("supply-chain-name"), + "BlueprintType": Equal("supply chain"), + "QualifiedResource": Equal("configmap"), + "PassThroughInput": Equal(""), + "StampedObject": Equal(&expectedObject), + "ResourceName": Equal("resource-1"), + "Healthy": And( + BeAssignableToTypeOf(metav1.ConditionUnknown), + BeEquivalentTo(""), + ), + "Err": And( + BeAssignableToTypeOf(stamp.JsonPathError{}), + BeEquivalentTo(stamp.NewJsonPathError( + "data.player_current_lives", + fmt.Errorf("failed to evaluate path of empty object"), + )), + ), + })) }) }) }) diff --git a/pkg/realizer/realizer.go b/pkg/realizer/realizer.go index 61df460d1..f14ebbbd9 100644 --- a/pkg/realizer/realizer.go +++ b/pkg/realizer/realizer.go @@ -19,6 +19,7 @@ package realizer import ( "context" "crypto/sha256" + "errors" "fmt" "reflect" "time" @@ -33,6 +34,7 @@ import ( "k8s.io/utils/strings/slices" "github.com/vmware-tanzu/cartographer/pkg/apis/v1alpha1" + cerrors "github.com/vmware-tanzu/cartographer/pkg/errors" "github.com/vmware-tanzu/cartographer/pkg/events" "github.com/vmware-tanzu/cartographer/pkg/logger" "github.com/vmware-tanzu/cartographer/pkg/realizer/healthcheck" @@ -119,14 +121,6 @@ func (r *realizer) Realize(ctx context.Context, resourceRealizer ResourceRealize "object", stampedObject) } - if err != nil { - log.Error(err, "failed to realize resource") - - if firstError == nil { - firstError = err - } - } - outs.AddOutput(resource.Name, out) previousResourceStatus := resourceStatuses.GetPreviousResourceStatus(resource.Name) @@ -164,7 +158,18 @@ func (r *realizer) Realize(ctx context.Context, resourceRealizer ResourceRealize additionalConditions = []metav1.Condition{r.healthyConditionEvaluator(template.GetHealthRule(), realizedResource, stampedObject)} } } + + var typedErr cerrors.RetrieveOutputError + ok := errors.As(err, &typedErr) + if ok { + if len(additionalConditions) > 0 { + typedErr.Healthy = additionalConditions[0].Status + err = typedErr + } + } + resourceStatuses.Add(realizedResource, err, isPassThrough, additionalConditions...) + if slices.Contains(resourceStatuses.ChangedConditionTypes(realizedResource.Name), v1alpha1.ResourceHealthy) { newStatus := metav1.ConditionUnknown newHealthyCondition := resourceStatuses.GetCurrent().ConditionsForResourceNamed(realizedResource.Name).ConditionWithType(v1alpha1.ResourceHealthy) @@ -173,8 +178,15 @@ func (r *realizer) Realize(ctx context.Context, resourceRealizer ResourceRealize } events.FromContextOrDie(ctx).ResourceEventf(events.NormalType, events.ResourceHealthyStatusChangedReason, "[%s] found healthy status in [%Q] changed to [%s]", stampedObject, realizedResource.Name, newStatus) } - } + if err != nil { + log.Error(err, "failed to realize resource") + + if firstError == nil { + firstError = err + } + } + } return firstError } diff --git a/pkg/stamp/reader.go b/pkg/stamp/reader.go index 1d44e4564..e9f82f0e9 100644 --- a/pkg/stamp/reader.go +++ b/pkg/stamp/reader.go @@ -83,7 +83,10 @@ type SourceOutputReader struct { func (r *SourceOutputReader) Output(stampedObject *unstructured.Unstructured) (*templates.Output, error) { if stampedObject == nil { - return nil, fmt.Errorf("failed to evaluate path of empty object") + return nil, JsonPathError{ + Err: fmt.Errorf("failed to evaluate path of empty object"), + expression: r.template.Spec.URLPath, + } } // TODO: We don't need a Builder evaluator := eval.EvaluatorBuilder() @@ -122,7 +125,10 @@ type ConfigOutputReader struct { func (r *ConfigOutputReader) Output(stampedObject *unstructured.Unstructured) (*templates.Output, error) { if stampedObject == nil { - return nil, fmt.Errorf("failed to evaluate path of empty object") + return nil, JsonPathError{ + Err: fmt.Errorf("failed to evaluate path of empty object"), + expression: r.template.Spec.ConfigPath, + } } evaluator := eval.EvaluatorBuilder() config, err := evaluator.EvaluateJsonPath(r.template.Spec.ConfigPath, stampedObject.UnstructuredContent()) @@ -151,7 +157,10 @@ type ImageOutputReader struct { func (r *ImageOutputReader) Output(stampedObject *unstructured.Unstructured) (*templates.Output, error) { if stampedObject == nil { - return nil, fmt.Errorf("failed to evaluate path of empty object") + return nil, JsonPathError{ + Err: fmt.Errorf("failed to evaluate path of empty object"), + expression: r.template.Spec.ImagePath, + } } evaluator := eval.EvaluatorBuilder() image, err := evaluator.EvaluateJsonPath(r.template.Spec.ImagePath, stampedObject.UnstructuredContent()) diff --git a/tests/integration/supplychain/workload_reconciler_test.go b/tests/integration/supplychain/workload_reconciler_test.go index 5e8d47e20..e35c910b5 100644 --- a/tests/integration/supplychain/workload_reconciler_test.go +++ b/tests/integration/supplychain/workload_reconciler_test.go @@ -17,6 +17,7 @@ package supplychain_test import ( "context" "encoding/json" + "fmt" "time" . "github.com/onsi/ginkgo" @@ -382,11 +383,12 @@ var _ = Describe("WorkloadReconciler", func() { Context("supply chain with immutable template", func() { var ( - expectedValue string - healthRuleSpecification string - lifecycleSpecification string - immutableTemplateBase string - workload v1alpha1.Workload + expectedValue string + healthRuleSpecification string + lifecycleSpecification string + immutableTemplateBase string + workload v1alpha1.Workload + configPathThatWillBeFound string ) BeforeEach(func() { @@ -397,7 +399,7 @@ var _ = Describe("WorkloadReconciler", func() { metadata: name: my-config-template spec: - configPath: spec.foo + configPath: %s lifecycle: %s template: apiVersion: test.run/v1alpha1 @@ -409,6 +411,8 @@ var _ = Describe("WorkloadReconciler", func() { %s ` + configPathThatWillBeFound = "spec.foo" + followOnTemplateYaml := utils.HereYaml(` --- apiVersion: carto.run/v1alpha1 @@ -543,7 +547,7 @@ var _ = Describe("WorkloadReconciler", func() { Context("without a healthRule", func() { BeforeEach(func() { healthRuleSpecification = "" - templateYaml := utils.HereYamlF(immutableTemplateBase, lifecycleSpecification, healthRuleSpecification) + templateYaml := utils.HereYamlF(immutableTemplateBase, configPathThatWillBeFound, lifecycleSpecification, healthRuleSpecification) template := utils.CreateObjectOnClusterFromYamlDefinition(ctx, c, templateYaml) cleanups = append(cleanups, template) }) @@ -637,7 +641,7 @@ var _ = Describe("WorkloadReconciler", func() { Context("with an alwaysHealthy healthRule", func() { BeforeEach(func() { healthRuleSpecification = "healthRule:\n alwaysHealthy: {}" - templateYaml := utils.HereYamlF(immutableTemplateBase, lifecycleSpecification, healthRuleSpecification) + templateYaml := utils.HereYamlF(immutableTemplateBase, configPathThatWillBeFound, lifecycleSpecification, healthRuleSpecification) template := utils.CreateObjectOnClusterFromYamlDefinition(ctx, c, templateYaml) cleanups = append(cleanups, template) }) @@ -665,7 +669,7 @@ var _ = Describe("WorkloadReconciler", func() { Context("which is not satisfied", func() { BeforeEach(func() { healthRuleSpecification = "healthRule:\n singleConditionType: Ready" - templateYaml := utils.HereYamlF(immutableTemplateBase, lifecycleSpecification, healthRuleSpecification) + templateYaml := utils.HereYamlF(immutableTemplateBase, configPathThatWillBeFound, lifecycleSpecification, healthRuleSpecification) template := utils.CreateObjectOnClusterFromYamlDefinition(ctx, c, templateYaml) cleanups = append(cleanups, template) }) @@ -674,9 +678,7 @@ var _ = Describe("WorkloadReconciler", func() { itStampsTheTemplatedObjectOnce() }) It("results in proper status", func() { - getConditionOfType := func(element interface{}) string { - return element.(metav1.Condition).Type - } + createdObject := getTestObjAtIndex(ctx, testNS, 0, 1) Eventually(func() []metav1.Condition { workload := &v1alpha1.Workload{} @@ -690,18 +692,18 @@ var _ = Describe("WorkloadReconciler", func() { return workload.Status.Resources[0].Conditions }).Should(MatchAllElements(getConditionOfType, Elements{ "ResourceSubmitted": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), - "Reason": Equal(v1alpha1.SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason), - "Message": ContainSubstring("unable to retrieve outputs for resource [my-first-resource] in supply chain [my-supply-chain]: failed to find any healthy object in the set of immutable stamped objects"), + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf("waiting to read value [spec.foo] from object [testobjs.test.run/%s] in namespace [%s]", createdObject.Name, testNS)), }), "Healthy": MatchFields(IgnoreExtras, Fields{ "Status": Equal(metav1.ConditionUnknown), "Reason": Equal("ReadyCondition"), - "Message": ContainSubstring("condition with type [Ready] not found on resource status"), + "Message": Equal("condition with type [Ready] not found on resource status"), }), "Ready": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), - "Reason": Equal(v1alpha1.SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason), + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), }), })) @@ -714,15 +716,15 @@ var _ = Describe("WorkloadReconciler", func() { "Status": Equal(metav1.ConditionTrue), }), "ResourcesSubmitted": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), - "Reason": Equal(v1alpha1.SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason), + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), }), "ResourcesHealthy": MatchFields(IgnoreExtras, Fields{ "Status": Equal(metav1.ConditionUnknown), "Reason": Equal("HealthyConditionRule"), }), "Ready": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), + "Status": Equal(metav1.ConditionUnknown), }), })) @@ -732,23 +734,43 @@ var _ = Describe("WorkloadReconciler", func() { When("the healthRule is subsequently satisfied", func() { It("results in a healthy workload and propagates outputs", func() { // update the object - opts := []client.ListOption{ - client.InNamespace(testNS), + testToUpdate := getTestObjAtIndex(ctx, testNS, 0, 1) + testToUpdate.Status.Conditions = []metav1.Condition{ + { + Type: "Ready", + Status: "True", + Reason: "Ready", + LastTransitionTime: metav1.Now(), + }, } - testsList := &resources.TestObjList{} + err := c.Status().Update(ctx, testToUpdate) + Expect(err).NotTo(HaveOccurred()) - Eventually(func() ([]resources.TestObj, error) { - err := c.List(ctx, testsList, opts...) - return testsList.Items, err - }).Should(HaveLen(1)) + itResultsInAHealthyWorkload() - testToUpdate := &testsList.Items[0] + Eventually(func() v1alpha1.Output { + workload := &v1alpha1.Workload{} + Expect(c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload)).To(Succeed()) + if len(workload.Status.Resources[0].Outputs) < 1 { + return v1alpha1.Output{} + } + return workload.Status.Resources[0].Outputs[0] + }).Should(MatchFields(IgnoreExtras, Fields{ + "Name": Equal("config"), + "Preview": Equal("some-address\n"), + })) + }) + }) + When("the healthRule is subsequently violated", func() { + It("results in an unhealthy workload that reports the absence of outputs", func() { + // update the object + testToUpdate := getTestObjAtIndex(ctx, testNS, 0, 1) testToUpdate.Status.Conditions = []metav1.Condition{ { Type: "Ready", - Status: "True", - Reason: "Ready", + Status: "False", + Reason: "SomeReason", LastTransitionTime: metav1.Now(), }, } @@ -757,21 +779,227 @@ var _ = Describe("WorkloadReconciler", func() { Expect(err).NotTo(HaveOccurred()) // assert expected state - itResultsInAHealthyWorkload() + Eventually(func() []metav1.Condition { + workload := &v1alpha1.Workload{} + err := c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload) + Expect(err).NotTo(HaveOccurred()) + + if len(workload.Status.Resources) < 2 { + return []metav1.Condition{} + } + + return workload.Status.Resources[0].Conditions + }).Should(MatchAllElements(getConditionOfType, Elements{ + "ResourceSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "cannot read value [spec.foo] from unhealthy object [testobjs.test.run/%s] in namespace [%s], examine object, particularly whether it is receiving proper inputs", + testToUpdate.Name, + testNS, + )), + }), + "Healthy": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionFalse), + "Reason": Equal("ReadyCondition"), + }), + "Ready": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionFalse), + "Reason": Equal("ReadyCondition"), + }), + })) workload := &v1alpha1.Workload{} err = c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload) Expect(err).NotTo(HaveOccurred()) - Expect(workload.Status.Resources[0].Outputs).To(HaveLen(1)) - Expect(workload.Status.Resources[0].Outputs[0]).To(MatchFields(IgnoreExtras, Fields{ - "Name": Equal("config"), - "Preview": Equal("some-address\n"), + Expect(workload.Status.Conditions).To(MatchAllElements(getConditionOfType, Elements{ + "SupplyChainReady": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionTrue), + }), + "ResourcesSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "cannot read value [spec.foo] from unhealthy object [testobjs.test.run/%s] in namespace [%s], examine object, particularly whether it is receiving proper inputs", + testToUpdate.Name, + testNS, + )), + }), + "ResourcesHealthy": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionFalse), + "Reason": Equal("HealthyConditionRule"), + }), + "Ready": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionFalse), + }), })) + + Expect(workload.Status.Resources[0].Outputs).To(HaveLen(0)) }) }) }) }) + + Context("whose output will not be found", func() { + var createdObj *resources.TestObj + BeforeEach(func() { + configPathThatWillNotBeFound := "status.someOutput" + healthRuleSpecification = "healthRule:\n singleConditionType: Ready" + templateYaml := utils.HereYamlF(immutableTemplateBase, configPathThatWillNotBeFound, lifecycleSpecification, healthRuleSpecification) + template := utils.CreateObjectOnClusterFromYamlDefinition(ctx, c, templateYaml) + cleanups = append(cleanups, template) + }) + + Context("while healthy", func() { + BeforeEach(func() { + createdObj = getTestObjAtIndex(ctx, testNS, 0, 1) + createdObj.Status.Conditions = []metav1.Condition{ + { + Type: "Ready", + Status: "True", + Reason: "Ready", + LastTransitionTime: metav1.Now(), + }, + } + + Expect(c.Status().Update(ctx, createdObj)).To(Succeed()) + }) + + It("returns an ResourceSubmitted error directing the reader to a Platform Eng", func() { + Eventually(func() []metav1.Condition { + workload := &v1alpha1.Workload{} + Expect(c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload)).To(Succeed()) + + if len(workload.Status.Resources) < 2 { + return []metav1.Condition{} + } + return workload.Status.Resources[0].Conditions + }).Should(MatchElements(getConditionOfType, IgnoreExtras, Elements{ + "ResourceSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "cannot read value [status.someOutput] from healthy object [testobjs.test.run/%s] in namespace [%s], contact Platform Eng", + createdObj.Name, + testNS, + )), + }), + })) + + workload := &v1alpha1.Workload{} + Expect(c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload)).To(Succeed()) + + Expect(workload.Status.Conditions).To(MatchElements(getConditionOfType, IgnoreExtras, Elements{ + "ResourcesSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "cannot read value [status.someOutput] from healthy object [testobjs.test.run/%s] in namespace [%s], contact Platform Eng", + createdObj.Name, + testNS, + )), + }), + })) + }) + }) + + Context("while unhealthy", func() { + BeforeEach(func() { + createdObj = getTestObjAtIndex(ctx, testNS, 0, 1) + createdObj.Status.Conditions = []metav1.Condition{ + { + Type: "Ready", + Status: "False", + Reason: "SomeReason", + LastTransitionTime: metav1.Now(), + }, + } + + Expect(c.Status().Update(ctx, createdObj)).To(Succeed()) + }) + + It("returns an ResourceSubmitted error directing the reader to examine inputs", func() { + Eventually(func() []metav1.Condition { + workload := &v1alpha1.Workload{} + Expect(c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload)).To(Succeed()) + + if len(workload.Status.Resources) < 2 { + return []metav1.Condition{} + } + return workload.Status.Resources[0].Conditions + }).Should(MatchElements(getConditionOfType, IgnoreExtras, Elements{ + "ResourceSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "cannot read value [status.someOutput] from unhealthy object [testobjs.test.run/%s] in namespace [%s], examine object, particularly whether it is receiving proper inputs", + createdObj.Name, + testNS, + )), + }), + })) + + workload := &v1alpha1.Workload{} + Expect(c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload)).To(Succeed()) + + Expect(workload.Status.Conditions).To(MatchElements(getConditionOfType, IgnoreExtras, Elements{ + "ResourcesSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "cannot read value [status.someOutput] from unhealthy object [testobjs.test.run/%s] in namespace [%s], examine object, particularly whether it is receiving proper inputs", + createdObj.Name, + testNS, + )), + }), + })) + }) + }) + + Context("while health unknown", func() { + BeforeEach(func() { + createdObj = getTestObjAtIndex(ctx, testNS, 0, 1) + Expect(c.Status().Update(ctx, createdObj)).To(Succeed()) + }) + It("returns an ResourceSubmitted error suggesting the reader wait", func() { + Eventually(func() []metav1.Condition { + workload := &v1alpha1.Workload{} + Expect(c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload)).To(Succeed()) + + if len(workload.Status.Resources) < 2 { + return []metav1.Condition{} + } + return workload.Status.Resources[0].Conditions + }).Should(MatchElements(getConditionOfType, IgnoreExtras, Elements{ + "ResourceSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "waiting to read value [status.someOutput] from object [testobjs.test.run/%s] in namespace [%s]", + createdObj.Name, + testNS, + )), + }), + })) + + workload := &v1alpha1.Workload{} + Expect(c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload)).To(Succeed()) + + Expect(workload.Status.Conditions).To(MatchElements(getConditionOfType, IgnoreExtras, Elements{ + "ResourcesSubmitted": MatchFields(IgnoreExtras, Fields{ + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf( + "waiting to read value [status.someOutput] from object [testobjs.test.run/%s] in namespace [%s]", + createdObj.Name, + testNS, + )), + }), + })) + }) + }) + }) }) Context("tekton template", func() { @@ -782,7 +1010,7 @@ var _ = Describe("WorkloadReconciler", func() { Context("without a healthRule", func() { BeforeEach(func() { healthRuleSpecification = "" - templateYaml := utils.HereYamlF(immutableTemplateBase, lifecycleSpecification, healthRuleSpecification) + templateYaml := utils.HereYamlF(immutableTemplateBase, configPathThatWillBeFound, lifecycleSpecification, healthRuleSpecification) template := utils.CreateObjectOnClusterFromYamlDefinition(ctx, c, templateYaml) cleanups = append(cleanups, template) }) @@ -931,6 +1159,8 @@ var _ = Describe("WorkloadReconciler", func() { return element.(metav1.Condition).Type } + createdObject := getTestObjAtIndex(ctx, testNS, 0, 1) + Eventually(func() []metav1.Condition { workload := &v1alpha1.Workload{} err := c.Get(ctx, client.ObjectKey{Name: "workload-joe", Namespace: testNS}, workload) @@ -943,9 +1173,9 @@ var _ = Describe("WorkloadReconciler", func() { return workload.Status.Resources[0].Conditions }).Should(MatchAllElements(getConditionOfType, Elements{ "ResourceSubmitted": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), - "Reason": Equal(v1alpha1.SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason), - "Message": ContainSubstring("unable to retrieve outputs for resource [my-first-resource] in supply chain [my-supply-chain]: failed to find any healthy object in the set of immutable stamped objects"), + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), + "Message": Equal(fmt.Sprintf("waiting to read value [spec.foo] from object [testobjs.test.run/%s] in namespace [%s]", createdObject.Name, testNS)), }), "Healthy": MatchFields(IgnoreExtras, Fields{ "Status": Equal(metav1.ConditionUnknown), @@ -953,8 +1183,8 @@ var _ = Describe("WorkloadReconciler", func() { "Message": ContainSubstring("condition with type [Succeeded] not found on resource status"), }), "Ready": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), - "Reason": Equal(v1alpha1.SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason), + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), }), })) @@ -967,15 +1197,15 @@ var _ = Describe("WorkloadReconciler", func() { "Status": Equal(metav1.ConditionTrue), }), "ResourcesSubmitted": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), - "Reason": Equal(v1alpha1.SetOfImmutableStampedObjectsIncludesNoHealthyObjectReason), + "Status": Equal(metav1.ConditionUnknown), + "Reason": Equal(v1alpha1.MissingValueAtPathResourcesSubmittedReason), }), "ResourcesHealthy": MatchFields(IgnoreExtras, Fields{ "Status": Equal(metav1.ConditionUnknown), "Reason": Equal("HealthyConditionRule"), }), "Ready": MatchFields(IgnoreExtras, Fields{ - "Status": Equal(metav1.ConditionFalse), + "Status": Equal(metav1.ConditionUnknown), }), })) @@ -1135,3 +1365,22 @@ var _ = Describe("WorkloadReconciler", func() { }) }) }) + +func getTestObjAtIndex(ctx context.Context, namespace string, index int, numObjectsExpected int) *resources.TestObj { + opts := []client.ListOption{ + client.InNamespace(namespace), + } + + testsList := &resources.TestObjList{} + + Eventually(func() ([]resources.TestObj, error) { + err := c.List(ctx, testsList, opts...) + return testsList.Items, err + }).Should(HaveLen(numObjectsExpected)) + + return &testsList.Items[index] +} + +func getConditionOfType(element interface{}) string { + return element.(metav1.Condition).Type +}