From 22997c32616354a56e912b07d8e354b5f59d3ed5 Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Mon, 11 Nov 2024 17:11:43 +0200 Subject: [PATCH] [mariadb] Fix race condition for grastate.dat There seems to be a race condition involving the grastate.dat file. Upon creation of a new mariad-server pod the file would exist however, it is not populated for a short period of time. It seems to take around 15-20 seconds for this file to be populated. However there is a separate thread which is attempting to read the file and tends to end in an IndexError exception killing the thread which maintains the grastate.dat file until the pod is restarted. This patchset adds a loop to check for up to 60 seconds for the file to be populated before attempting to continue, thus giving the file time to be populated. Change-Id: I2f2a801aa4528a7af61797419422572be1c82e75 --- charts/infra/mariadb/templates/bin/_start.py.tpl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/charts/infra/mariadb/templates/bin/_start.py.tpl b/charts/infra/mariadb/templates/bin/_start.py.tpl index e0efb0c10..aee55d625 100644 --- a/charts/infra/mariadb/templates/bin/_start.py.tpl +++ b/charts/infra/mariadb/templates/bin/_start.py.tpl @@ -530,8 +530,19 @@ def get_grastate_val(key): """ logger.debug("Reading grastate.dat key={0}".format(key)) try: - with open("/var/lib/mysql/grastate.dat", "r") as myfile: - grastate_raw = [s.strip() for s in myfile.readlines()] + # This attempts to address a potential race condition with the initial + # creation of the grastate.date file where the file would exist + # however, it is not immediately populated. Testing indicated it could + # take 15-20 seconds for the file to be populated. So loop and keep + # checking up to 60 seconds. If it still isn't populated afterwards, + # the IndexError will still occur as we are seeing now without the loop. + time_end = time.time() + 60 + while time.time() < time_end: + with open("/var/lib/mysql/grastate.dat", "r") as myfile: + grastate_raw = [s.strip() for s in myfile.readlines()] + if grastate_raw: + break + time.sleep(1) return [i for i in grastate_raw if i.startswith("{0}:".format(key))][0].split(':')[1].strip() except IndexError: