Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: TOOLS-2996 ns SC check for manage roster stage observed #331

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
with:
submodules: true
- name: Get Python version from Pipfile
run: echo "PYTHON_VERSION=$(grep "python_full_version" Pipfile | cut -d ' ' -f 3 - | tr -d '"')" >> $GITHUB_ENV
run: echo "PYTHON_VERSION=$(grep "python_version" Pipfile | cut -d ' ' -f 3 - | tr -d '"')" >> $GITHUB_ENV
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v3
with:
Expand Down
7 changes: 3 additions & 4 deletions lib/live_cluster/client/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -2401,12 +2401,11 @@ async def info_cluster_stable(
resp = await self._info(req)

if "error" in resp.lower():
if "cluster not specified size" in resp or "unstable cluster" in resp:
raise ASInfoClusterStableError(resp)

raise ASInfoResponseError(
info_server_response_err = ASInfoResponseError(
ErrorsMsgs.INFO_SERVER_ERROR_RESPONSE, resp
)
logger.error(info_server_response_err)
raise info_server_response_err

return resp

Expand Down
63 changes: 61 additions & 2 deletions lib/live_cluster/manage_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
from lib.base_controller import CommandHelp, ModifierHelp, ShellException
from lib.utils.lookup_dict import PrefixDict
from .client import (
ASInfoClusterStableError,
ASInfoResponseError,
ASInfoError,
ASProtocolError,
BoolConfigType,
Expand Down Expand Up @@ -2721,7 +2721,7 @@ def _check_and_log_cluster_stable(self, stable_data):
warning_str = "The cluster is unstable. It is advised that you do not manage the roster. Run 'info network' for more information."

for resp in stable_data.values():
if isinstance(resp, ASInfoClusterStableError):
if isinstance(resp, ASInfoResponseError):
logger.warning(warning_str)
return False

Expand All @@ -2748,6 +2748,35 @@ def _check_and_log_nodes_in_observed(self, observed, nodes):
return False

return True

async def _check_ns_is_strong_consistency(self, ns):
"""
Check if a namespace is in strong consistency mode.
"""
try:
namespace_stats = await self.cluster.info_namespace_statistics(ns, nodes='all')
if isinstance(namespace_stats, Exception):
raise namespace_stats

if not namespace_stats:
logger.error("namespace {} not found".format(ns))
return False

namespace_stats = list(namespace_stats.values())[0] if len(namespace_stats.values()) > 0 else None
if not namespace_stats:
logger.error("namespace {} not does not exist".format(ns))
return False

strong_consistency = namespace_stats.get("strong-consistency", "false").lower() == 'true'
if strong_consistency is False:
logger.error("namespace {} is not in strong consistency mode".format(ns))
return strong_consistency

except Exception as e:
logger.error("Error while checking namespace strong consistency mode: {}".format(e))
raise e

return strong_consistency


@CommandHelp(
Expand Down Expand Up @@ -2789,6 +2818,13 @@ async def _do_default(self, line):
modifiers=self.modifiers,
mods=self.mods,
)

# to be run against a SC namespace only
ns_strong_consistency = await self._check_ns_is_strong_consistency(ns)
if isinstance(ns_strong_consistency, Exception):
return
elif not ns_strong_consistency:
return

current_roster = asyncio.create_task(
self.cluster.info_roster(ns, nodes="principal")
Expand Down Expand Up @@ -2864,6 +2900,14 @@ async def _do_default(self, line):
modifiers=self.modifiers,
mods=self.mods,
)

# to be run against a SC namespace only
ns_strong_consistency = await self._check_ns_is_strong_consistency(ns)
if isinstance(ns_strong_consistency, Exception):
return
elif not ns_strong_consistency:
return

current_roster = asyncio.create_task(
self.cluster.info_roster(ns, nodes="principal")
)
Expand Down Expand Up @@ -2957,6 +3001,13 @@ async def _do_default(self, line):
mods=self.mods,
)

# to be run against a SC namespace only
ns_strong_consistency = await self._check_ns_is_strong_consistency(ns)
if isinstance(ns_strong_consistency, Exception):
return
elif not ns_strong_consistency:
return

if warn:
current_roster = asyncio.create_task(
self.cluster.info_roster(ns, nodes="principal")
Expand Down Expand Up @@ -3014,6 +3065,14 @@ def __init__(self):

async def _do_default(self, line):
ns = self.mods["ns"][0]

# to be run against a SC namespace only
ns_strong_consistency = await self._check_ns_is_strong_consistency(ns)
if isinstance(ns_strong_consistency, Exception):
return
elif not ns_strong_consistency:
return

current_roster = asyncio.create_task(
self.cluster.info_roster(ns, nodes="principal")
)
Expand Down
6 changes: 4 additions & 2 deletions test/unit/live_cluster/client/test_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
ASInfoConfigError,
ASInfoResponseError,
)
from lib.live_cluster.client.constants import ErrorsMsgs

with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
Expand Down Expand Up @@ -2053,9 +2054,10 @@ async def test_info_cluster_stable(self):

async def test_info_cluster_stable_with_errors(self):
self.info_mock.return_value = "ERROR::cluster not specified size"
expected = ASInfoClusterStableError("ERROR::cluster not specified size")
expected = ASInfoResponseError(ErrorsMsgs.INFO_SERVER_ERROR_RESPONSE, "ERROR::cluster not specified size")

actual = await self.node.info_cluster_stable(cluster_size=3, namespace="bar")
print("LOLLL", actual)

self.info_mock.assert_called_with(
"cluster-stable:size=3;namespace=bar", self.ip
Expand All @@ -2067,7 +2069,7 @@ async def test_info_cluster_stable_with_errors(self):
)

self.info_mock.return_value = "ERROR::unstable cluster"
expected = ASInfoClusterStableError("ERROR::unstable cluster")
expected = ASInfoResponseError(ErrorsMsgs.INFO_SERVER_ERROR_RESPONSE, "ERROR::unstable cluster")

actual = await self.node.info_cluster_stable(cluster_size=3, namespace="bar")

Expand Down
Loading
Loading