From eec3d8d614414cd22acdd068636062b4335712f9 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sun, 21 Jan 2024 13:08:00 +0100 Subject: [PATCH 1/3] fix: Adding VOMS extensions without having environment variables set --- src/DIRAC/Core/Security/Locations.py | 36 +++++++++++++++++++ src/DIRAC/Core/Security/VOMS.py | 40 ++++++--------------- tests/Integration/Framework/Test_ProxyDB.py | 6 ++++ 3 files changed, 53 insertions(+), 29 deletions(-) diff --git a/src/DIRAC/Core/Security/Locations.py b/src/DIRAC/Core/Security/Locations.py index 3c9fa6cdeb9..ede9ed25fc2 100644 --- a/src/DIRAC/Core/Security/Locations.py +++ b/src/DIRAC/Core/Security/Locations.py @@ -5,6 +5,10 @@ from DIRAC import gConfig g_SecurityConfPath = "/DIRAC/Security" +DEFAULT_VOMSES_LOCATION = f"{DIRAC.rootPath}/etc/grid-security/vomses" +SYSTEM_VOMSES_LOCATION = "/etc/vomses" +DEFAULT_VOMSDIR_LOCATION = f"{DIRAC.rootPath}/etc/grid-security/vomsdir" +SYSTEM_VOMSDIR_LOCATION = "/etc/grid-security/vomsdir" def getProxyLocation(): @@ -172,3 +176,35 @@ def getDefaultProxyLocation(): # /tmp/x509up_u proxyName = "x509up_u%d" % os.getuid() return f"/tmp/{proxyName}" + + +def getVomsesLocation(): + """Get the location of the directory containing the vomses files""" + if "X509_VOMSES" in os.environ: + return os.environ["X509_VOMSES"] + elif os.path.isdir(DEFAULT_VOMSES_LOCATION): + return DEFAULT_VOMSES_LOCATION + elif os.path.isdir(SYSTEM_VOMSES_LOCATION): + return SYSTEM_VOMSES_LOCATION + else: + raise Exception( + "The env variable X509_VOMSES is not set. " + "DIRAC does not know where to look for etc/grid-security/vomses. " + "Please set X509_VOMSES." + ) + + +def getVomsdirLocation(): + """Get the location of the directory containing the vomsdir files""" + if "X509_VOMS_DIR" in os.environ: + return os.environ["X509_VOMS_DIR"] + elif os.path.isdir(DEFAULT_VOMSDIR_LOCATION): + return DEFAULT_VOMSDIR_LOCATION + elif os.path.isdir(SYSTEM_VOMSDIR_LOCATION): + return SYSTEM_VOMSDIR_LOCATION + else: + raise Exception( + "The env variable X509_VOMS_DIR is not set. " + "DIRAC does not know where to look for etc/grid-security/vomsdir. " + "Please set X509_VOMS_DIR." + ) diff --git a/src/DIRAC/Core/Security/VOMS.py b/src/DIRAC/Core/Security/VOMS.py index 407a1aba449..67031da21d9 100644 --- a/src/DIRAC/Core/Security/VOMS.py +++ b/src/DIRAC/Core/Security/VOMS.py @@ -8,6 +8,7 @@ from DIRAC import S_OK, S_ERROR, gConfig, rootPath, gLogger from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Security import Locations from DIRAC.Core.Security.ProxyFile import multiProxyArgument, deleteMultiProxy from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.Utilities.Subprocess import shellCall @@ -188,34 +189,7 @@ def getVOMSProxyInfo(self, proxy, option=False): self._unlinkFiles(proxyDict["file"]) def getVOMSESLocation(self): - # Transition code to new behaviour - if "DIRAC_VOMSES" not in os.environ and "X509_VOMSES" not in os.environ: - os.environ["X509_VOMSES"] = os.path.join(rootPath, "etc", "grid-security", "vomses") - gLogger.notice( - "You did not set X509_VOMSES in your bashrc. DIRAC searches $DIRAC/etc/grid-security/vomses . " - "Please use X509_VOMSES, this auto discovery will be dropped." - ) - elif "DIRAC_VOMSES" in os.environ and "X509_VOMSES" in os.environ: - os.environ["X509_VOMSES"] = f"{os.environ['DIRAC_VOMSES']}:{os.environ['X509_VOMSES']}" - gLogger.notice( - "You set both variables DIRAC_VOMSES and X509_VOMSES in your bashrc. " - "DIRAC_VOMSES will be dropped in a future version, please use only X509_VOMSES" - ) - elif "DIRAC_VOMSES" in os.environ and "X509_VOMSES" not in os.environ: - os.environ["X509_VOMSES"] = os.environ["DIRAC_VOMSES"] - gLogger.notice( - "You set the variables DIRAC_VOMSES in your bashrc. " - "DIRAC_VOMSES will be dropped in a future version, please use X509_VOMSES" - ) - # End of transition code - if "X509_VOMSES" not in os.environ: - raise Exception( - "The env variable X509_VOMSES is not set. " - "DIRAC does not know where to look for etc/grid-security/vomses. " - "Please set X509_VOMSES in your bashrc." - ) - vomsesPaths = os.environ["X509_VOMSES"].split(":") - for vomsesPath in vomsesPaths: + for vomsesPath in Locations.getVomsesLocation().split(":"): if not os.path.exists(vomsesPath): continue if os.path.isfile(vomsesPath): @@ -284,7 +258,15 @@ def setVOMSAttributes(self, proxy, attribute=None, vo=None): cmd += ["-r"] cmd += ["-timeout", "12"] - result = shellCall(self._secCmdTimeout, shlex.join(cmd)) + result = shellCall( + self._secCmdTimeout, + shlex.join(cmd), + env=os.environ + | { + "X509_CERT_DIR": Locations.getCAsLocation(), + "X509_VOMS_DIR": Locations.getVomsdirLocation(), + }, + ) deleteMultiProxy(proxyDict) diff --git a/tests/Integration/Framework/Test_ProxyDB.py b/tests/Integration/Framework/Test_ProxyDB.py index bf1c7afb4e9..6d64db2a290 100644 --- a/tests/Integration/Framework/Test_ProxyDB.py +++ b/tests/Integration/Framework/Test_ProxyDB.py @@ -631,10 +631,16 @@ def test_getRemoveProxy(self): continue VOMS_PROXY_INIT_CMD = DIRAC.Core.Security.VOMS.VOMS_PROXY_INIT_CMD DIRAC.Core.Security.VOMS.VOMS_PROXY_INIT_CMD = "voms-proxy-fake" + clear_x509_env = False + if "X509_VOMS_DIR" not in os.environ: + os.environ["X509_VOMS_DIR"] = "/etc/grid-security/vomsdir" + clear_x509_env = True try: result = db.getVOMSProxy(dn, group, time, role) finally: DIRAC.Core.Security.VOMS.VOMS_PROXY_INIT_CMD = VOMS_PROXY_INIT_CMD + if clear_x509_env: + del os.environ["X509_VOMS_DIR"] self.assertFalse(result["OK"], "Must be fail.") gLogger.info(f"Msg: {result['Message']}") From 1e609b323b0492a04ef7cbbaa8630b8226f8d99f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 22 Jan 2024 11:10:53 +0100 Subject: [PATCH 2/3] feat: Add DIRAC_DISABLE_GCONFIG_REFRESH environment variable --- .../environment_variable_configuration.rst | 4 ++++ src/DIRAC/ConfigurationSystem/private/Refresher.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst index c08cee8deb4..caddb2023dd 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst @@ -18,6 +18,10 @@ DIRAC_DEPRECATED_FAIL If set, the use of functions or objects that are marked ``@deprecated`` will fail. Useful for example in continuous integration tests against future versions of DIRAC +DIRAC_DISABLE_GCONFIG_REFRESH + If set, attempting to start the ``gConfig`` refresh thread will result in an exception. + This is used by DiracX to accidental use of vanilla DIRAC in contexts where it won't work. + DIRAC_FEWER_CFG_LOCKS If ``true`` or ``yes`` or ``on`` or ``1`` or ``y`` or ``t``, DIRAC will reduce the number of locks used when accessing the CS for better performance (default, ``no``). diff --git a/src/DIRAC/ConfigurationSystem/private/Refresher.py b/src/DIRAC/ConfigurationSystem/private/Refresher.py index 9da1603b50f..28e17a8e818 100755 --- a/src/DIRAC/ConfigurationSystem/private/Refresher.py +++ b/src/DIRAC/ConfigurationSystem/private/Refresher.py @@ -58,6 +58,8 @@ def refreshConfigurationIfNeeded(self): except _thread.error: pass # Launch the refresh + if os.environ.get("DIRAC_DISABLE_GCONFIG_REFRESH", "false").lower() in ("yes", "true"): + raise NotImplementedError("DIRAC_DISABLE_GCONFIG_REFRESH is set") thd = threading.Thread(target=self._refreshInThread) thd.daemon = True thd.start() From 61dc5598512c6a1e89d13d96fd3d91050ea15dcd Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 22 Jan 2024 11:11:24 +0100 Subject: [PATCH 3/3] refactor: Split voms-proxy-init command for use with DiracX --- src/DIRAC/Core/Security/Locations.py | 6 +++ src/DIRAC/Core/Security/ProxyFile.py | 2 + src/DIRAC/Core/Security/VOMS.py | 58 +++++++++++++++------------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/DIRAC/Core/Security/Locations.py b/src/DIRAC/Core/Security/Locations.py index ede9ed25fc2..6e01b51c4da 100644 --- a/src/DIRAC/Core/Security/Locations.py +++ b/src/DIRAC/Core/Security/Locations.py @@ -45,6 +45,12 @@ def getCAsLocation(): casPath = retVal["Value"] if os.path.isdir(casPath): return casPath + # Other locations + return getCAsLocationNoConfig() + + +def getCAsLocationNoConfig(): + """Retrieve the CA's files location""" # Look up the X509_CERT_DIR environment variable if "X509_CERT_DIR" in os.environ: casPath = os.environ["X509_CERT_DIR"] diff --git a/src/DIRAC/Core/Security/ProxyFile.py b/src/DIRAC/Core/Security/ProxyFile.py index 5a0b3f47864..d693d267d3a 100644 --- a/src/DIRAC/Core/Security/ProxyFile.py +++ b/src/DIRAC/Core/Security/ProxyFile.py @@ -135,6 +135,8 @@ def multiProxyArgument(proxy=False): return S_ERROR(DErrno.EPROXYFIND) if isinstance(proxy, str): proxyLoc = proxy + else: + raise NotImplementedError(f"Unknown proxy type ({type(proxy)})") # Load proxy proxy = X509Chain() retVal = proxy.loadProxyFromFile(proxyLoc) diff --git a/src/DIRAC/Core/Security/VOMS.py b/src/DIRAC/Core/Security/VOMS.py index 67031da21d9..31148e37af2 100644 --- a/src/DIRAC/Core/Security/VOMS.py +++ b/src/DIRAC/Core/Security/VOMS.py @@ -18,6 +18,37 @@ VOMS_PROXY_INIT_CMD = "voms-proxy-init" +def voms_init_cmd( + vo: str, attribute: str | None, chain: X509Chain, in_fn: str, out_fn: str, vomsesPath: str | None +) -> list[str]: + secs = chain.getRemainingSecs()["Value"] - 300 + if secs < 0: + return S_ERROR(DErrno.EVOMS, "Proxy length is less that 300 secs") + hours = int(secs / 3600) + mins = int((secs - hours * 3600) / 60) + + bitStrength = chain.getStrength()["Value"] + + cmd = [VOMS_PROXY_INIT_CMD] + if chain.isLimitedProxy()["Value"]: + cmd.append("-limited") + cmd += ["-cert", in_fn] + cmd += ["-key", in_fn] + cmd += ["-out", out_fn] + cmd += ["-voms"] + cmd += [f"{vo}:{attribute}" if attribute and attribute != "NoRole" else vo] + cmd += ["-valid", f"{hours}:{mins}"] + cmd += ["-bits", str(bitStrength)] + if vomsesPath: + cmd += ["-vomses", vomsesPath] + + if chain.isRFC().get("Value"): + cmd += ["-r"] + cmd += ["-timeout", "12"] + + return cmd + + class VOMS: def __init__(self, *args, **kwargs): """Create VOMS class, setting specific timeout for VOMS shell commands.""" @@ -225,38 +256,13 @@ def setVOMSAttributes(self, proxy, attribute=None, vo=None): chain = proxyDict["chain"] proxyLocation = proxyDict["file"] - secs = chain.getRemainingSecs()["Value"] - 300 - if secs < 0: - return S_ERROR(DErrno.EVOMS, "Proxy length is less that 300 secs") - hours = int(secs / 3600) - mins = int((secs - hours * 3600) / 60) - - # Ask VOMS a proxy the same strength as the one we already have - bitStrength = chain.getStrength()["Value"] - retVal = self._generateTemporalFile() if not retVal["OK"]: deleteMultiProxy(proxyDict) return retVal newProxyLocation = retVal["Value"] - cmd = [VOMS_PROXY_INIT_CMD] - if chain.isLimitedProxy()["Value"]: - cmd.append("-limited") - cmd += ["-cert", proxyLocation] - cmd += ["-key", proxyLocation] - cmd += ["-out", newProxyLocation] - cmd += ["-voms"] - cmd += [f"{vo}:{attribute}" if attribute and attribute != "NoRole" else vo] - cmd += ["-valid", f"{hours}:{mins}"] - cmd += ["-bits", str(bitStrength)] - vomsesPath = self.getVOMSESLocation() - if vomsesPath: - cmd += ["-vomses", vomsesPath] - - if chain.isRFC().get("Value"): - cmd += ["-r"] - cmd += ["-timeout", "12"] + cmd = voms_init_cmd(vo, attribute, chain, proxyLocation, newProxyLocation, self.getVOMSESLocation()) result = shellCall( self._secCmdTimeout,