diff --git a/e2e/tests/free5gc/009.sh b/e2e/tests/free5gc/009.sh index f4393c7f..76d64930 100755 --- a/e2e/tests/free5gc/009.sh +++ b/e2e/tests/free5gc/009.sh @@ -61,17 +61,15 @@ if [[ $lifecycle == "Published" ]]; then info "Copied to $smf_pkg_rev, pulling" fi -# We need to put this entire section in a retry loop, because it is possible -# for a controller to come in and change the package after we pull it. This -# in general is something we should not be seeing, but is not really a failure -# state, so we will work around it in here. A separate issues has been filed to -# debug why a controller is unexpectedly changing the package. - - -# Calls porchctl, but does not immediatelly die on conflict errors (e.g.: "the object has been modified; please apply your changes to the latest version and try again"), +# Calls porchctl, but does not immediatelly die due to: +# - conflict errors (e.g.: "the object has been modified; please apply your changes to the latest version and try again") +# - readiness errors (e.g.: "readiness conditions not met") +# Calls porchctl, but does not immediatelly die due to: +# - conflict errors (e.g.: "the object has been modified; please apply your changes to the latest version and try again") +# - readiness errors (e.g.: "readiness conditions not met") # but returns with a non-zero code instead. # It dies on any other error as usual. -function porchctl_enable_conflict { +function porchctl_enable_err_check { # do not immediatelly die on error set +o pipefail set +o errexit @@ -81,8 +79,8 @@ function porchctl_enable_conflict { set -o pipefail set -o errexit - if [[ $output =~ "modified" ]]; then - info "Capacity update failed due to concurrent change, retrying" + if [[ $output =~ "modified" ]] || [[ $output =~ "readiness" ]] ; then + info "Capacity update failed due to $output, retrying" retries=$((retries - 1)) return 1 fi @@ -92,8 +90,17 @@ function porchctl_enable_conflict { return 0 } -retries=5 + +function get_pkgrev_conditions { + conditions=$(kubectl --kubeconfig "$kubeconfig" get packagerevision "$smf_pkg_rev" -o jsonpath='{range .status.conditions[*]}{.type}{" : "}{.status}{"\n"}{end}') + info $conditions +} + +retries=20 while [[ $retries -gt 0 ]]; do + info "Getting conditons before PULL" + get_pkgrev_conditions + rm -rf $ws porchctl rpkg pull -n default "$smf_pkg_rev" $ws @@ -107,20 +114,27 @@ while [[ $retries -gt 0 ]]; do diff -r /tmp/$ws $ws || echo + info "Getting conditons before PUSH" + get_pkgrev_conditions + modified=false info "Pushing update" - if ! porchctl_enable_conflict rpkg push -n default "$smf_pkg_rev" $ws ; then + if ! porchctl_enable_err_check rpkg push -n default "$smf_pkg_rev" $ws ; then continue fi + sleep 10 + info "Getting conditons before PROPOSE" + get_pkgrev_conditions + info "Proposing update" - if ! porchctl_enable_conflict rpkg propose -n default "$smf_pkg_rev" ; then + if ! porchctl_enable_err_check rpkg propose -n default "$smf_pkg_rev" ; then continue fi k8s_wait_exists "packagerev" "$smf_pkg_rev" info "approving package $smf_pkg_rev update" - if ! porchctl_enable_conflict rpkg approve -n default "$smf_pkg_rev" ; then + if ! porchctl_enable_err_check rpkg approve -n default "$smf_pkg_rev" ; then continue fi info "approved package $smf_pkg_rev update"