Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: PanDAWMS/panda-server
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 0.4.1
Choose a base ref
...
head repository: PanDAWMS/panda-server
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref
Loading
Showing with 3,836 additions and 1,279 deletions.
  1. +2 −2 Dockerfile
  2. +1 −1 PandaPkgInfo.py
  3. 0 pandaserver/api/__init__.py
  4. +240 −0 pandaserver/api/openapi_generator.py
  5. 0 pandaserver/api/v1/__init__.py
  6. +174 −0 pandaserver/api/v1/common.py
  7. +440 −0 pandaserver/api/v1/harvester_api.py
  8. +189 −0 pandaserver/api/v1/harvester_api_tests.py
  9. +144 −0 pandaserver/api/v1/http_client.py
  10. +162 −0 pandaserver/api/v1/idds_api.py
  11. +137 −0 pandaserver/api/v1/secret_management_api.py
  12. +44 −0 pandaserver/api/v1/secret_management_api_tests.py
  13. +162 −0 pandaserver/api/v1/task_api.py
  14. +71 −0 pandaserver/api/v1/task_api_tests.py
  15. +21 −0 pandaserver/api/v1/timed_method.py
  16. +2 −0 pandaserver/api/v1/version.py
  17. +1 −1 pandaserver/brokerage/SiteMapper.py
  18. +13 −3 pandaserver/config/panda_config.py
  19. +4 −2 pandaserver/daemons/scripts/copyArchive.py
  20. +28 −8 pandaserver/daemons/scripts/task_evaluator.py
  21. +10 −4 pandaserver/daemons/utils.py
  22. +21 −17 pandaserver/dataservice/adder_gen.py
  23. +2 −13 pandaserver/dataservice/ddm.py
  24. +20 −20 pandaserver/dataservice/setupper_atlas_plugin.py
  25. +1 −52 pandaserver/jobdispatcher/JobDispatcher.py
  26. +9 −8 pandaserver/jobdispatcher/Watcher.py
  27. +216 −78 pandaserver/server/panda.py
  28. +22 −0 pandaserver/srvcore/CoreUtils.py
  29. +22 −11 pandaserver/srvcore/panda_request.py
  30. +6 −0 pandaserver/taskbuffer/EventServiceUtils.py
  31. +35 −0 pandaserver/taskbuffer/JobSpec.py
  32. +687 −603 pandaserver/taskbuffer/OraDBProxy.py
  33. +1 −1 pandaserver/taskbuffer/PandaDBSchemaInfo.py
  34. +1 −1 pandaserver/taskbuffer/ResourceSpec.py
  35. +60 −24 pandaserver/taskbuffer/TaskBuffer.py
  36. +28 −17 pandaserver/taskbuffer/WrappedCursor.py
  37. 0 pandaserver/taskbuffer/db_proxy_mods/__init__.py
  38. +58 −0 pandaserver/taskbuffer/db_proxy_mods/base_module.py
  39. +373 −0 pandaserver/taskbuffer/db_proxy_mods/metrics_module.py
  40. +133 −0 pandaserver/taskbuffer/db_proxy_mods/task_module.py
  41. +128 −12 pandaserver/taskbuffer/retryModule.py
  42. +27 −10 pandaserver/taskbuffer/task_split_rules.py
  43. +0 −71 pandaserver/test/test_error_classification.py
  44. +21 −11 pandaserver/userinterface/Client.py
  45. +15 −66 pandaserver/userinterface/UserIF.py
  46. +1 −1 pyproject.toml
  47. +0 −181 templates/panda_server-httpd-FastCGI.conf.rpmnew.template
  48. +91 −51 templates/panda_server-httpd.conf.rpmnew.template
  49. +0 −10 templates/panda_server.cfg.rpmnew.template
  50. +9 −0 templates/sysconfig/panda_server.sysconfig.rpmnew.template
  51. +4 −0 templates/sysconfig/panda_server_env.systemd.rpmnew.template
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -67,7 +67,7 @@ RUN mkdir -p /etc/rc.d/init.d
RUN mv /opt/panda/etc/panda/panda_common.cfg.rpmnew /etc/panda/panda_common.cfg
RUN mv /opt/panda/etc/panda/panda_server.cfg.rpmnew /etc/panda/panda_server.cfg
RUN mv /opt/panda/etc/panda/panda_server.sysconfig.rpmnew /etc/sysconfig/panda_server
RUN mv /opt/panda/etc/panda/panda_server-httpd-FastCGI.conf.rpmnew /opt/panda/etc/panda/panda_server-httpd.conf
RUN mv /opt/panda/etc/panda/panda_server-httpd.conf.rpmnew /opt/panda/etc/panda/panda_server-httpd.conf

# make a wrapper script to launch services and periodic jobs in non-root container
RUN echo $'#!/bin/bash \n\
@@ -121,7 +121,7 @@ RUN mkdir /tmp/panda-wnscript && cd /tmp/panda-wnscript && \
git clone https://github.com/PanDAWMS/panda-wnscript.git && \
cp -R panda-wnscript/dist/* /var/trf/user/ && \
cd / && rm -rf /tmp/panda-wnscript

ENV PANDA_LOCK_DIR /var/run/panda
RUN mkdir -p ${PANDA_LOCK_DIR} && chmod 777 ${PANDA_LOCK_DIR}

2 changes: 1 addition & 1 deletion PandaPkgInfo.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
release_version = "0.4.1"
release_version = "0.5.1"
Empty file added pandaserver/api/__init__.py
Empty file.
240 changes: 240 additions & 0 deletions pandaserver/api/openapi_generator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
import ast
import copy
import re
import traceback

import yaml
from docstring_parser import parse

EXCLUDED_FUNCTIONS = ["init_task_buffer"]
EXCLUDED_PARAMS = ["req"]

DEFAULT_RESPONSE_TEMPLATE = {
"200": {
"description": "Method called correctly",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"success": {"type": "boolean", "example": True, "description": "Indicates whether the request was successful (True) or not (False)"},
"message": {
"type": "string",
"description": "Message indicating the nature of the failure. Empty or meaningless if the request was successful.",
},
"response": {
"type": "object",
"description": "The data returned if the operation is successful. Null if it fails or the method does not generate return data.",
},
},
"required": ["success", "message", "response"],
}
}
},
},
"403": {
"description": "Forbidden",
"content": {"text/plain": {"schema": {"type": "string", "example": "You are calling an undefined method is not allowed for the requested URL"}}},
},
"404": {"description": "Not Found", "content": {"text/plain": {"schema": {"type": "string", "example": "Resource not found"}}}},
"500": {
"description": "INTERNAL SERVER ERROR",
"content": {
"text/plain": {
"schema": {
"type": "string",
"example": "INTERNAL SERVER ERROR. The server encountered an internal error and was unable to complete your request.",
}
}
},
},
}


def extract_docstrings(file_path):
"""
Extract all docstrings from a Python file.
Args:
file_path (str): Path to the Python file.
Returns:
dict: A dictionary where keys are function/class/module names and values are docstrings.
"""
with open(file_path, "r") as file:
tree = ast.parse(file.read())

docstrings = {}

# Extract module-level docstring
if ast.get_docstring(tree):
docstrings["__module__"] = ast.get_docstring(tree)

# Walk through all nodes in the file
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name not in EXCLUDED_FUNCTIONS:
# Get the name of the function/class/module
name = node.name if hasattr(node, "name") else "__module__"
# Get the docstring
docstring = ast.get_docstring(node)
if docstring:
docstrings[name] = docstring

return docstrings


def extract_parameters(parsed):
"""
Extract parameters from a docstring and format them as OpenAPI parameters.
"""
parameters = []

# Iterate over the parameters in the docstring
# print(f"Parameters: {parsed.params}")
for param in parsed.params:
# Skip excluded parameters that we don't want to appear in the API documentation
if param.arg_name in EXCLUDED_PARAMS:
continue

# Map each parameter to OpenAPI format
param_type = "string" # Default type (fallback)
if "list" in param.type_name:
param_type = "array"
elif "int" in param.type_name:
param_type = "integer"
elif "float" in param.type_name:
param_type = "number"
elif "bool" in param.type_name:
param_type = "boolean"

print(f"Param: {param.arg_name}, Type: {param_type}, Optional: {param.is_optional} {param.description}")
is_required = not param.is_optional

parameter_schema = {"name": param.arg_name, "in": "query", "required": is_required, "schema": {"type": param_type}, "description": param.description}

if param_type == "array":
parameter_schema["schema"]["items"] = {"type": "string"} # Default array item type

parameters.append(parameter_schema)

return parameters


def extract_parameters_as_json(parsed):
"""
Extract parameters from a docstring and format them as JSON schema for OpenAPI.
"""
properties = {}
required_parameters = []

print(f"Parameters(json): {parsed.params}")
for param in parsed.params:
# Skip excluded parameters that we don't want to appear in the API documentation
if param.arg_name in EXCLUDED_PARAMS:
continue

# Map the type to OpenAPI type
param_type = "string" # Default type
if "list" in param.type_name:
param_type = "array"
elif "int" in param.type_name:
param_type = "integer"
elif "float" in param.type_name:
param_type = "number"
elif "bool" in param.type_name:
param_type = "boolean"

# Build schema for the parameter
param_schema = {"type": param_type, "description": param.description}

if param_type == "array":
param_schema["items"] = {"type": "object"} # Default array item type

properties[param.arg_name] = param_schema

if not param.is_optional:
required_parameters.append(param.arg_name)

return {"type": "object", "properties": properties, "required": required_parameters}


def get_custom_metadata(docstring):
# Parse the docstring manually for the custom section
doc_lines = docstring.splitlines()
custom_section_name = "API details:"
custom_sections = {}
inside_custom_section = False

for line in doc_lines:
stripped_line = line.strip()
if stripped_line == custom_section_name:
inside_custom_section = True
continue # Skip the section header line
if inside_custom_section:
if stripped_line == "" or ":" not in stripped_line: # End of custom section
break
key, value = map(str.strip, stripped_line.split(":", 1))
custom_sections[key] = value

return custom_sections


def convert_docstrings_to_openapi(docstrings):
"""
Convert extracted docstrings to OpenAPI format.
Args:
docstrings (dict): A dictionary where keys are function/class/module names and values are docstrings.
Returns:
dict: A dictionary where keys are function/class/module names and values are OpenAPI descriptions.
"""

# Initialize the OpenAPI dictionary
openapi = {"openapi": "3.0.3", "info": {"title": "PanDA API", "version": "1.0.0"}, "paths": {}}

for name, docstring in docstrings.items():
try:
parsed_docstring = parse(docstring)
summary = parsed_docstring.short_description

# Remove the custom "API details" section from the long description
description = re.sub(r"API details:\n(?:\s+.*\n?)*", "", parsed_docstring.long_description)

custom_metadata = get_custom_metadata(docstring)
method = custom_metadata.get("HTTP Method", "POST").lower()
path = custom_metadata.get("Path", name)

openapi["paths"][path] = {}
openapi["paths"][path][method] = {"summary": summary, "description": description}

# Extract parameters from the docstring
parameters = extract_parameters(parsed_docstring)
if method in ["post", "put"]:
request_body_schema = extract_parameters_as_json(parsed_docstring)
openapi["paths"][path][method]["requestBody"] = {"required": True, "content": {"application/json": {"schema": request_body_schema}}}
elif method in ["get", "delete"]:
openapi["paths"][path][method]["parameters"] = parameters

if parsed_docstring.returns:
return_type = parsed_docstring.returns.type_name
return_desc = parsed_docstring.returns.description

# Add default responses
openapi["paths"][path][method]["responses"] = copy.deepcopy(DEFAULT_RESPONSE_TEMPLATE)

except (AttributeError, TypeError):
print(f"Docstring for {name} could not be parsed. Failed with error:\n{traceback.format_exc()}")

return openapi


if __name__ == "__main__":
# Define the path to the Python file to convert
file_path = "v1/harvester_api.py"
docstrings = extract_docstrings(file_path)

# Convert docstrings to OpenAPI
open_api_doc = convert_docstrings_to_openapi(docstrings)
with open("/tmp/panda_api.yaml", "w") as output_file:
yaml.dump(open_api_doc, output_file, sort_keys=False)
Empty file added pandaserver/api/v1/__init__.py
Empty file.
Loading