Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix for 009 smf scaling flakiness #307

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Fix for 009 smf scaling flakiness #307

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9217265
Fix for 009 smf scaling flakiness
efiacor Sep 26, 2024
92bc97a
Add logging for pkg condtions in 009
efiacor Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 29 additions & 15 deletions e2e/tests/free5gc/009.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,17 +61,15 @@ if [[ $lifecycle == "Published" ]]; then
info "Copied to $smf_pkg_rev, pulling"
fi

# We need to put this entire section in a retry loop, because it is possible
# for a controller to come in and change the package after we pull it. This
# in general is something we should not be seeing, but is not really a failure
# state, so we will work around it in here. A separate issues has been filed to
# debug why a controller is unexpectedly changing the package.


# Calls porchctl, but does not immediatelly die on conflict errors (e.g.: "the object has been modified; please apply your changes to the latest version and try again"),
# Calls porchctl, but does not immediatelly die due to:
# - conflict errors (e.g.: "the object has been modified; please apply your changes to the latest version and try again")
# - readiness errors (e.g.: "readiness conditions not met")
# Calls porchctl, but does not immediatelly die due to:
# - conflict errors (e.g.: "the object has been modified; please apply your changes to the latest version and try again")
# - readiness errors (e.g.: "readiness conditions not met")
# but returns with a non-zero code instead.
# It dies on any other error as usual.
function porchctl_enable_conflict {
function porchctl_enable_err_check {
# do not immediatelly die on error
set +o pipefail
set +o errexit
Expand All @@ -81,8 +79,8 @@ function porchctl_enable_conflict {
set -o pipefail
set -o errexit

if [[ $output =~ "modified" ]]; then
info "Capacity update failed due to concurrent change, retrying"
if [[ $output =~ "modified" ]] || [[ $output =~ "readiness" ]] ; then
info "Capacity update failed due to $output, retrying"
retries=$((retries - 1))
return 1
fi
Expand All @@ -92,8 +90,17 @@ function porchctl_enable_conflict {
return 0
}

retries=5

function get_pkgrev_conditions {
conditions=$(kubectl --kubeconfig "$kubeconfig" get packagerevision "$smf_pkg_rev" -o jsonpath='{range .status.conditions[*]}{.type}{" : "}{.status}{"\n"}{end}')
info $conditions
}

retries=20
while [[ $retries -gt 0 ]]; do
info "Getting conditons before PULL"
get_pkgrev_conditions

rm -rf $ws
porchctl rpkg pull -n default "$smf_pkg_rev" $ws

Expand All @@ -107,20 +114,27 @@ while [[ $retries -gt 0 ]]; do

diff -r /tmp/$ws $ws || echo

info "Getting conditons before PUSH"
get_pkgrev_conditions

modified=false
info "Pushing update"
if ! porchctl_enable_conflict rpkg push -n default "$smf_pkg_rev" $ws ; then
if ! porchctl_enable_err_check rpkg push -n default "$smf_pkg_rev" $ws ; then
continue
fi

sleep 10
info "Getting conditons before PROPOSE"
get_pkgrev_conditions

info "Proposing update"
if ! porchctl_enable_conflict rpkg propose -n default "$smf_pkg_rev" ; then
if ! porchctl_enable_err_check rpkg propose -n default "$smf_pkg_rev" ; then
continue
fi
k8s_wait_exists "packagerev" "$smf_pkg_rev"

info "approving package $smf_pkg_rev update"
if ! porchctl_enable_conflict rpkg approve -n default "$smf_pkg_rev" ; then
if ! porchctl_enable_err_check rpkg approve -n default "$smf_pkg_rev" ; then
continue
fi
info "approved package $smf_pkg_rev update"
Expand Down