Merge pull request #308 from PanDAWMS/oracledb

Migrating cx_Oracle to oracledb and some miscelaneous clean up

INSTALL.md

@@ -17,7 +17,7 @@ Installation
    pip install panda-server
    ``` 
 
-1. Install database modules: cx_Oracle for Oracle or MySQLdb for MySQL/MariaDB
+1. Install database modules: oracledb for Oracle or MySQLdb for MySQL/MariaDB
 1. Modify config files.
 
    ```

INSTALL_ATLAS.md

@@ -18,11 +18,11 @@ Installation for ATLAS on CC7 + virtualenv + python3 + atlpan
    chown atlpan:zp /var/log/panda/wsgisocks
    chown atlpan:zp /var/cache/pandaserver
    ```
-1. Install panda-server and cx_Oracle.
+1. Install panda-server and oracledb.
    ```
    source /opt/pandaserver/bin/activate
    pip install panda-server
-   pip install cx_Oracle
+   pip install oracledb
    ``` 
 
 1. Modify config files.

pandaserver/daemons/scripts/panda_activeusers_query.py

@@ -3,12 +3,13 @@
 from pandacommon.pandalogger.LogWrapper import LogWrapper
 from pandacommon.pandalogger.PandaLogger import PandaLogger
 from pandacommon.pandautils.thread_utils import GenericThread
+
 from pandaserver.config import panda_config
 from pandaserver.proxycache import panda_proxy_cache
 from pandaserver.srvcore import CoreUtils
 
 # logger
-_logger = PandaLogger().getLogger("panda_activeusers_query")
+_logger = PandaLogger().getLogger("activeusers_query")
 
 
 # main
@@ -28,7 +29,7 @@ def main(tbuf=None, **kwargs):
             panda_config.dbpasswd,
             nDBConnection=1,
             useTimeout=True,
-            requester=requesterd_id,
+            requester=requester_id,
         )
         tmpLog.debug("Getting new connection - done")
     else:

pandaserver/daemons/utils.py

@@ -16,6 +16,7 @@
 import psutil
 from pandacommon.pandalogger import logger_utils
 from pandacommon.pandautils.thread_utils import GenericThread, LockPool
+
 from pandaserver.config import daemon_config, panda_config
 
 # list of signals accepted to end the main process
@@ -88,7 +89,7 @@ def got_end_sig(sig, frame):
     expiry_ts = start_ts + worker_lifetime
     # create taskBuffer object if not given
     if tbuf is None:
-        # initialize cx_Oracle using dummy connection
+        # initialize oracledb using dummy connection
         try:
             from pandaserver.taskbuffer.Initializer import initializer
 

pandaserver/dataservice/DDM.py

@@ -8,7 +8,6 @@
 import sys
 import traceback
 
-from pandaserver.srvcore import CoreUtils
 from rucio.client import Client as RucioClient
 from rucio.common.exception import (
     DataIdentifierAlreadyExists,
@@ -20,6 +19,8 @@
     UnsupportedOperation,
 )
 
+from pandaserver.srvcore import CoreUtils
+
 
 # rucio
 class RucioAPI:
@@ -337,33 +338,6 @@ def registerZipFiles(self, zipMap):
                 )
                 iFiles += nFiles
 
-    # get disk usage at RSE
-    def getRseUsage(self, rse, src="srm"):
-        retMap = {}
-        try:
-            client = RucioClient()
-            itr = client.get_rse_usage(rse)
-            # look for srm
-            for item in itr:
-                if item["source"] == src:
-                    try:
-                        total = item["total"] / 1024 / 1024 / 1024
-                    except Exception:
-                        total = None
-                    try:
-                        used = item["used"] / 1024 / 1024 / 1024
-                    except Exception:
-                        used = None
-                    try:
-                        free = item["free"] / 1024 / 1024 / 1024
-                    except Exception:
-                        free = None
-                    retMap = {"total": total, "used": used, "free": free}
-                    break
-        except Exception:
-            pass
-        return retMap
-
     # list datasets
     def listDatasets(self, datasetName, old=False):
         result = {}
@@ -463,16 +437,6 @@ def getMetaData(self, dsn):
             errType, errVale = sys.exc_info()[:2]
             return False, f"{errType} {errVale}"
 
-    # check if dataset exists
-    def checkDatasetExist(self, dsn):
-        # register dataset
-        client = RucioClient()
-        try:
-            scope, dsn = self.extract_scope(dsn)
-            return True
-        except DataIdentifierNotFound:
-            return False
-
     # delete dataset
     def eraseDataset(self, dsn, scope=None, grace_period=None):
         presetScope = scope
@@ -632,25 +596,6 @@ def getDatasetSize(self, datasetName):
             errMsg = f"{errtype.__name__} {errvalue}"
             return False, errMsg
 
-    # delete dataset replicas
-    def deleteDatasetReplicas(self, datasetName, locations):
-        # extract scope from dataset
-        scope, dsn = self.extract_scope(datasetName)
-        client = RucioClient()
-        try:
-            for rule in client.list_did_rules(scope, dsn):
-                if rule["account"] != client.account:
-                    continue
-                if rule["rse_expression"] in locations:
-                    client.delete_replication_rule(rule["id"])
-        except DataIdentifierNotFound:
-            pass
-        except Exception:
-            errtype, errvalue = sys.exc_info()[:2]
-            errMsg = f"{errtype.__name__} {errvalue}"
-            return False, errMsg
-        return True, ""
-
     # register files
     def registerFiles(self, files, rse):
         client = RucioClient()

pandaserver/dataservice/DDMHandler.py

@@ -1,7 +1,3 @@
-"""
-master hander for DDM
-"""
-
 import re
 import threading
 
@@ -52,4 +48,4 @@ def run(self):
             else:
                 # finish transferring jobs
                 Finisher(self.taskBuffer, dataset, site=self.site).start()
-        tmpLog.debug("end")
+        tmpLog.debug("end")

pandaserver/dataservice/forkSetupper.py

@@ -2,6 +2,7 @@
 import traceback
 
 from pandacommon.pandautils.thread_utils import GenericThread
+
 from pandaserver.srvcore.CoreUtils import commands_get_status_output
 
 
@@ -23,7 +24,7 @@ def run(inFile, v_onlyTA, v_firstSubmission):
     # password
     from pandaserver.config import panda_config
 
-    # initialize cx_Oracle using dummy connection
+    # initialize oracledb using dummy connection
     from pandaserver.taskbuffer.Initializer import initializer
 
     initializer.init()