diff --git a/docs/gen_ref_pages.py b/docs/gen_ref_pages.py index f547d77..19880bf 100644 --- a/docs/gen_ref_pages.py +++ b/docs/gen_ref_pages.py @@ -105,7 +105,6 @@ def generate_sub_process_result(requirement_file) -> dict: for entry in check_required: entry = entry.lower() if entry not in sub_process_result: - command = ["pip", "show", entry] print("Collect Req. PIP Infos for package:", entry) result = run( @@ -164,7 +163,6 @@ def generate_code_reference_documentation( nav = mkdocs_gen_files.Nav() for path in sorted(Path(source_path).rglob(source_file_type_filter)): - module_path = path.relative_to(source_path).with_suffix("") doc_path = path.relative_to(source_path).with_suffix(md_file_type) full_doc_path = Path(virtual_ref_nav_path, doc_path) @@ -209,7 +207,6 @@ def generate_code_reference_documentation( with open(requirement_file, "r") as req_file: req_txt = req_file.read() if req_txt and len(req_txt) > 0: - with mkdocs_gen_files.open(req_md_file, "w") as fd: fd.write(f"# {source_path.replace('_', ' ')} - Included Libraries\n***\n\n") # Python 3.x only diff --git a/docs/release-notes.md b/docs/release-notes.md index f46e9a4..f6931ef 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,6 +1,12 @@ # msaBase Release Notes ## Possible future features: +# 0.0.110 + +- added supporting pydantic v2 +- update msaDocModels +- added env for sentry + # 0.0.109 - add Input and DTO models for Spellcheck @@ -9,7 +15,7 @@ - added enviroments for sentry -## 0.0.107 +# 0.0.107 - fix algorithm to DocClassifier Input diff --git a/docs/saved_req_package_pip_info.pkl b/docs/saved_req_package_pip_info.pkl index 4ee0e49..6f09b36 100644 Binary files a/docs/saved_req_package_pip_info.pkl and b/docs/saved_req_package_pip_info.pkl differ diff --git a/msaBase/__init__.py b/msaBase/__init__.py index ef46bcd..78973f8 100644 --- a/msaBase/__init__.py +++ b/msaBase/__init__.py @@ -1,7 +1,7 @@ import glob from os.path import basename, dirname, isfile, join -version = "0.0.109" +version = "0.0.110" __author__ = "Stefan Welcker" __copyright__ = "Copyright 2022, U2D.ai" __license__ = "MIT" diff --git a/msaBase/config.py b/msaBase/config.py index c2a9ae7..2b9ac83 100644 --- a/msaBase/config.py +++ b/msaBase/config.py @@ -139,9 +139,9 @@ def save_config(self) -> None: """ Saves config to a JSON file """ - sa = self.copy(deep=True) + sa = self.model_copy(deep=True) with open("config.json", "w") as fp: - json.dump(sa.dict(), fp, sort_keys=True, indent=4) + json.dump(sa.model_dump(), fp, sort_keys=True, indent=4) @staticmethod def load_config(): diff --git a/msaBase/configurate.py b/msaBase/configurate.py index 578cae0..423b3fa 100644 --- a/msaBase/configurate.py +++ b/msaBase/configurate.py @@ -21,6 +21,7 @@ from fastapi.exception_handlers import http_exception_handler from fastapi.exceptions import RequestValidationError from fastapi.responses import ORJSONResponse +from fastapi_restful.timing import add_timing_middleware from fs.base import FS from grpc._channel import _InactiveRpcError from loguru import logger as logger_gruru @@ -105,11 +106,11 @@ async def load_config(url: str) -> None: if resp.status == 200: config = MSAServiceDefinition.parse_obj(await resp.json()) new_config = get_msa_app_settings() - if new_config.dict(exclude={"version"}) == config.dict(exclude={"version"}): + if new_config.model_dump(exclude={"version"}) == config.model_dump(exclude={"version"}): return with open("config.json", "w") as json_file: - json.dump(config.dict(), json_file, sort_keys=True, indent=4) + json.dump(config.model_dump(), json_file, sort_keys=True, indent=4) logger_gruru.info("New config saved to config.json") else: @@ -223,7 +224,7 @@ async def read_config(received_config: ConfigInput) -> None: if reload_needed: self.logger.info("New config needs reload.") with open("config.json", "w") as json_file: - json.dump(received_config.data.dict(), json_file) + json.dump(received_config.data.model_dump(), json_file) self.logger.info("New config saved to config.json") @@ -432,7 +433,6 @@ def get_services_settings(self, request: Request) -> ORJSONResponse: def try_get_json(): try: - return jsonable_encoder(self.settings) except Exception as e: @@ -458,7 +458,6 @@ def get_services_openapi_schema(self, request: Request) -> ORJSONResponse: def try_get_json(): try: - return jsonable_encoder(self.openapi()) except Exception as e: @@ -647,7 +646,6 @@ def update_settings(self, new_config: MSAServiceDefinition, one_time=False) -> b if (current_functionality is not None and new_functionality is not None) and ( current_functionality != new_functionality ): - if reload_needed: return True @@ -905,7 +903,6 @@ def configure_cors_middleware(self) -> None: def configure_timing_middleware(self) -> None: """Add Middleware Timing""" self.logger.info("Add Middleware Timing") - from fastapi_utils.timing import add_timing_middleware add_timing_middleware(self, record=self.logger.info, prefix="app", exclude="untimed") @@ -964,7 +961,7 @@ def init_sentry(self, sentry_dsn: str) -> None: sentry_sdk.init( dsn=sentry_dsn, traces_sample_rate=1.0, - environment=os.getenv("STAGE_ENV","local"), + environment=os.getenv("STAGE_ENV", "local"), ) diff --git a/msaBase/logger.py b/msaBase/logger.py index 284c2be..4ee83e0 100644 --- a/msaBase/logger.py +++ b/msaBase/logger.py @@ -82,17 +82,10 @@ def init_logging(): ``` """ - # disable handlers for specific uvicorn loggers - # to redirect their output to the default uvicorn logger - # works with uvicorn==0.11.6 - # print("Logger", [name for name in logging.root.manager.loggerDict]) - for handler in logging.root.handlers[:]: logging.root.removeHandler(handler) loggers = (logging.getLogger(name) for name in logging.root.manager.loggerDict if name.startswith("uvicorn.")) - - # change handler for default uvicorn logger intercept_handler = InterceptHandler() for uvicorn_logger in loggers: @@ -100,9 +93,7 @@ def init_logging(): uvicorn_logger.handlers = [intercept_handler] except: pass - # logging.getLogger().handlers = [intercept_handler] logging.getLogger("uvicorn").handlers = [] logging.getLogger("rocketry").handlers = [] - # set logs output, level and format logger.configure(handlers=[{"sink": sys.stdout, "level": logging.INFO, "format": format_record}]) diff --git a/msaBase/models/settings.py b/msaBase/models/settings.py index 8a686d2..e317062 100644 --- a/msaBase/models/settings.py +++ b/msaBase/models/settings.py @@ -1,4 +1,4 @@ -from fastapi_utils.api_settings import APISettings +from fastapi_restful.api_settings import APISettings from pydantic.config import Extra diff --git a/msaBase/models/sysinfo.py b/msaBase/models/sysinfo.py index d73e17b..12f3370 100644 --- a/msaBase/models/sysinfo.py +++ b/msaBase/models/sysinfo.py @@ -18,14 +18,14 @@ class MSAGPUInfo(BaseModel): uuid: unique GPU identifier """ - id: Optional[int] - name: Optional[str] - load: Optional[str] - free_memory: Optional[str] - used_memory: Optional[str] - total_memory: Optional[str] - temperature: Optional[str] - uuid: Optional[str] + id: Optional[int] = None + name: Optional[str] = None + load: Optional[str] = None + free_memory: Optional[str] = None + used_memory: Optional[str] = None + total_memory: Optional[str] = None + temperature: Optional[str] = None + uuid: Optional[str] = None class MSADiskIO(BaseModel): @@ -45,15 +45,15 @@ class MSADiskIO(BaseModel): """ - read_count: Optional[int] - write_count: Optional[int] - read_bytes: Optional[int] - write_bytes: Optional[int] - read_time: Optional[int] - write_time: Optional[int] - read_merged_count: Optional[int] - write_merged_count: Optional[int] - busy_time: Optional[int] + read_count: Optional[int] = None + write_count: Optional[int] = None + read_bytes: Optional[int] = None + write_bytes: Optional[int] = None + read_time: Optional[int] = None + write_time: Optional[int] = None + read_merged_count: Optional[int] = None + write_merged_count: Optional[int] = None + busy_time: Optional[int] = None class MSANetworkIO(BaseModel): @@ -71,14 +71,14 @@ class MSANetworkIO(BaseModel): dropout: total number of outgoing packets which were dropped (always 0 on macOS and BSD) """ - bytes_sent: Optional[int] - bytes_recv: Optional[int] - packets_sent: Optional[int] - packets_recv: Optional[int] - errin: Optional[int] - errout: Optional[int] - dropin: Optional[int] - dropout: Optional[int] + bytes_sent: Optional[int] = None + bytes_recv: Optional[int] = None + packets_sent: Optional[int] = None + packets_recv: Optional[int] = None + errin: Optional[int] = None + errout: Optional[int] = None + dropin: Optional[int] = None + dropout: Optional[int] = None class MSANetworkConnection(BaseModel): @@ -102,14 +102,14 @@ class MSANetworkConnection(BaseModel): (e.g. Linux) the availability of this field changes depending on process privileges (root is needed). """ - index: Optional[int] - file_descriptor: Optional[int] - family: Optional[int] - type: Optional[int] - local_addr: Optional[str] - remote_addr: Optional[str] + index: Optional[int] = None + file_descriptor: Optional[int] = None + family: Optional[int] = None + type: Optional[int] = None + local_addr: Optional[str] = None + remote_addr: Optional[str] = None status: str = "" - pid: Optional[int] + pid: Optional[int] = None class MSANetworkAdapter(BaseModel): @@ -125,11 +125,11 @@ class MSANetworkAdapter(BaseModel): (typically a VPN). broadcast and ptp are mutually exclusive. May be None. """ - family: Optional[int] - address: Optional[str] - netmask: Optional[str] - broadcast: Optional[str] - ptp: Optional[int] + family: Optional[int] = None + address: Optional[str] = None + netmask: Optional[str] = None + broadcast: Optional[str] = None + ptp: Optional[int] = None class MSANetworkAdapters(BaseModel): @@ -156,10 +156,10 @@ class MSANetworkStat(BaseModel): mtu: NIC’s maximum transmission unit expressed in bytes. """ - isup: Optional[bool] - duplex: Optional[int] - speed: Optional[int] - mtu: Optional[int] + isup: Optional[bool] = None + duplex: Optional[int] = None + speed: Optional[int] = None + mtu: Optional[int] = None class MSANetworkStats(BaseModel): @@ -186,10 +186,10 @@ class MSATemperature(BaseModel): critical: critical temperature """ - label: Optional[str] - current: Optional[float] - high: Optional[float] - critical: Optional[float] + label: Optional[str] = None + current: Optional[float] = None + high: Optional[float] = None + critical: Optional[float] = None class MSATemperatures(BaseModel): @@ -215,9 +215,9 @@ class MSACPUFrequency(BaseModel): max: maximal frequency """ - current: Optional[float] - min: Optional[int] - max: Optional[int] + current: Optional[float] = None + min: Optional[int] = None + max: Optional[int] = None class MSACPUTimes(BaseModel): @@ -240,16 +240,16 @@ class MSACPUTimes(BaseModel): systems under the control of the Linux kernel) """ - user: Optional[float] - nice: Optional[int] - system: Optional[float] - idle: Optional[float] - iowait: Optional[float] - irq: Optional[int] - softirq: Optional[float] - steal: Optional[int] - guest: Optional[float] - guest_nice: Optional[int] + user: Optional[float] = None + nice: Optional[int] = None + system: Optional[float] = None + idle: Optional[float] = None + iowait: Optional[float] = None + irq: Optional[int] = None + softirq: Optional[float] = None + steal: Optional[int] = None + guest: Optional[float] = None + guest_nice: Optional[int] = None class MSACPUStats(BaseModel): @@ -263,11 +263,10 @@ class MSACPUStats(BaseModel): syscalls: number of system calls since boot. Always set to 0 on Linux. """ - ctx_switches: Optional[int] - interrupts: Optional[int] - soft_interrupts: Optional[int] - syscalls: Optional[int] - """""" + ctx_switches: Optional[int] = None + interrupts: Optional[int] = None + soft_interrupts: Optional[int] = None + syscalls: Optional[int] = None class MSAMemoryUsage(BaseModel): @@ -290,15 +289,15 @@ class MSAMemoryUsage(BaseModel): inactive: (UNIX): memory that is marked as not used. """ - total: Optional[float] - available: Optional[float] - used: Optional[float] - free: Optional[float] - percent: Optional[float] - buffers: Optional[float] - cached: Optional[float] - active: Optional[float] - inactive: Optional[float] + total: Optional[float] = None + available: Optional[float] = None + used: Optional[float] = None + free: Optional[float] = None + percent: Optional[float] = None + buffers: Optional[float] = None + cached: Optional[float] = None + active: Optional[float] = None + inactive: Optional[float] = None """""" @@ -313,10 +312,10 @@ class MSASwap(BaseModel): percent: the percentage usage calculated as (total - available) / total * 100 """ - total: Optional[float] - used: Optional[float] - free: Optional[float] - percent: Optional[float] + total: Optional[float] = None + used: Optional[float] = None + free: Optional[float] = None + percent: Optional[float] = None class MSASystemInfo(BaseModel): @@ -371,32 +370,32 @@ class MSASystemInfo(BaseModel): HW_Identifier: str = "" IP_Address: str = "" MAC_Address: str = "" - CPU_Physical: Optional[int] - CPU_Logical: Optional[int] + CPU_Physical: Optional[int] = None + CPU_Logical: Optional[int] = None Memory_Physical: str = "" Memory_Available: str = "" System_Boot: str = "" Service_Start: str = "" Runtime_Exe: str = "" Runtime_Cmd: List[str] = [] - Disk_IO: Optional[MSADiskIO] - Network_IO: Optional[MSANetworkIO] - Network_Connections: Optional[List[MSANetworkConnection]] - Network_Adapters: Optional[List[MSANetworkAdapters]] - Network_Stats: Optional[List[MSANetworkStats]] - Temperatures: Optional[List[MSATemperatures]] - CPU_Affinity: Optional[int] - CPU_Frequency: Optional[MSACPUFrequency] - CPU_Times: Optional[MSACPUTimes] - CPU_Stats: Optional[MSACPUStats] - PID: Optional[int] - CPU_Current: Optional[int] - CPU_Usage_Total: Optional[float] - CPU_Usage_Process: Optional[float] + Disk_IO: Optional[MSADiskIO] = None + Network_IO: Optional[MSANetworkIO] = None + Network_Connections: Optional[List[MSANetworkConnection]] = None + Network_Adapters: Optional[List[MSANetworkAdapters]] = None + Network_Stats: Optional[List[MSANetworkStats]] = None + Temperatures: Optional[List[MSATemperatures]] = None + CPU_Affinity: Optional[int] = None + CPU_Frequency: Optional[MSACPUFrequency] = None + CPU_Times: Optional[MSACPUTimes] = None + CPU_Stats: Optional[MSACPUStats] = None + PID: Optional[int] = None + CPU_Current: Optional[int] = None + CPU_Usage_Total: Optional[float] = None + CPU_Usage_Process: Optional[float] = None CPU_Usage_Name: str = "" - CPU_LoadAvg: Optional[List[float]] - Memory_Usage: Optional[MSAMemoryUsage] - Swap: Optional[MSASwap] + CPU_LoadAvg: Optional[List[float]] = None + Memory_Usage: Optional[MSAMemoryUsage] = None + Swap: Optional[MSASwap] = None Runtime_Status: str = "" @@ -437,14 +436,14 @@ class MSASystemGPUInfo(BaseModel): HW_Identifier: str = "" IP_Address: str = "" MAC_Address: str = "" - CPU_Physical: Optional[int] - CPU_Logical: Optional[int] + CPU_Physical: Optional[int] = None + CPU_Logical: Optional[int] = None Memory_Physical: str = "" Memory_Available: str = "" System_Boot: str = "" Service_Start: str = "" Runtime_Exe: str = "" Runtime_Cmd: List[str] = [] - PID: Optional[int] - GPUs: Optional[List[MSAGPUInfo]] + PID: Optional[int] = None + GPUs: Optional[List[MSAGPUInfo]] = None Runtime_Status: str = "" diff --git a/requirements.txt b/requirements.txt index 778a6c9..0d5ddd5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,68 +1,68 @@ # MSA Dependencies msaFileSystem==0.0.3 # Agnostic Abstract Filesystem API which allows to use S3, GCS, Azure Datalake, your local FS, Youtube etc Optimized for use with FastAPI/Pydantic. -msaDocModels==0.0.100 # MSA Document Pydantic Models and Schemas, used to store Parser, NLP, NLU and AI results for processed documents +msaDocModels==0.0.101 # MSA Document Pydantic Models and Schemas, used to store Parser, NLP, NLU and AI results for processed documents # FastAPI related Dependencies anyio==3.7.1 # an asynchronous networking and concurrency library that works on top of either asyncio or trio -fastapi[all]==0.86.0 # FastAPI framework, high performance, easy to learn, fast to code, ready for production -fastapi_utils==0.2.1 # Reusable utilities for FastAPI, Repeated Tasks, APIModel, APISettings -pydantic[email,dotenv]==1.9.2 # Data validation and settings management using python type hints -pyinstrument==4.4.0 # pyinstrument to check service performance. +fastapi[all]==0.103.1 # FastAPI framework, high performance, easy to learn, fast to code, ready for production +fastapi-restful==0.5.0 # Reusable utilities for FastAPI, Repeated Tasks, APIModel, APISettings +pydantic[email,dotenv]==2.3.0 # Data validation and settings management using python type hints # General Dependencies -autoflake==2.0.1 # Removes unused imports and unused variables -black==22.6.0 # Code formatter +autoflake==2.2.1 # Removes unused imports and unused variables +black==23.7.0 # Code formatter +typing-inspect==0.9.0 # module defines experimental API for runtime inspection pyproject-flake8==6.0.0 # configure flake8 flake8==6.0.0 # modular source code checker: pep8 pyflakes and co colorama==0.4.6 # Makes ANSI escape character sequence isort==5.12.0 # library to sort Python imports. -sentry-sdk==1.24.0 # library to automatic reporting of errors and exceptions. -loguru==0.6.0 # Python logging made (stupidly) simple -lxml==4.9.2 # Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API. -mypy==1.0.0 # Optional static typing for Python -setuptools==67.6.1 # Easily download, build, install, upgrade, and uninstall Python packages -prometheus_fastapi_instrumentator==5.9.1 # Instrument your FastAPI with Prometheus metrics +sentry-sdk==1.30.0 # library to automatic reporting of errors and exceptions. +loguru==0.7.1 # Python logging made (stupidly) simple +lxml==4.9.3 # Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API. +mypy==1.5.1 # Optional static typing for Python +setuptools==68.2.0 # Easily download, build, install, upgrade, and uninstall Python packages +prometheus_fastapi_instrumentator==6.1.0 # Instrument your FastAPI with Prometheus metrics Jinja2==3.1.2 # A very fast and expressive template engine. -orjson==3.8.3 # Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy -pyinstrument==4.4.0 # pyinstrument to check service performance. +orjson==3.9.6 # Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy +pyinstrument==4.5.3 # pyinstrument to check service performance. msgpack-asgi==1.1.0 # Drop-in MessagePack support for ASGI applications and frameworks -slowapi==0.1.7 # A rate limiting extension for Starlette and Fastapi +slowapi==0.1.8 # A rate limiting extension for Starlette and Fastapi addict==2.4.0 # A dictionary whose items can be set using both attribute and item syntax. -pymongo==4.3.3 # A Python distribution containing tools for working with MongoDB +pymongo==4.5.0 # A Python distribution containing tools for working with MongoDB # Dapr Dependencies -dapr==1.9.0 # Dapr is a portable, serverless, event-driven runtime that makes it easy for developers to build resilient, stateless and stateful microservices that run on the cloud and edge -dapr-ext-grpc==1.9.0 # gRPC extension for Dapr. -dapr-ext-fastapi==1.9.0 # Dapr is a portable, serverless, event-driven runtime +dapr==1.10.0 # Dapr is a portable, serverless, event-driven runtime that makes it easy for developers to build resilient, stateless and stateful microservices that run on the cloud and edge +dapr-ext-grpc==1.10.0 # gRPC extension for Dapr. +dapr-ext-fastapi==1.10.0 # Dapr is a portable, serverless, event-driven runtime # File Management related libs -aiofiles==23.1.0 # handling local disk files in asyncio applications +aiofiles==23.2.1 # handling local disk files in asyncio applications fs==2.4.16 # Python's filesystem abstraction layer # Starlette related Dependencies -starlette==0.20.4 # Starlette is a lightweight ASGI framework/toolkit, which is ideal for building async web services in Python. -starlette-context==0.3.5 # Access context in Starlette -starception==0.4.1 # Beautiful debugging page for Starlette apps. +starlette==0.27.0 # Starlette is a lightweight ASGI framework/toolkit, which is ideal for building async web services in Python. +starlette-context==0.3.6 # Access context in Starlette +starception==1.0.1 # Beautiful debugging page for Starlette apps. Starlette-WTF==0.4.3 # Simple integration of Starlette and WTForms. # Other Dependencies -httpx==0.23.3 # The next generation HTTP client. -aiohttp==3.8.4 # The next generation HTTP client. +httpx==0.24.1 # The next generation HTTP client. +aiohttp==3.8.5 # The next generation HTTP client. hjson==3.1.0 # Hjson, a user interface for JSON. # Uvicorn related Dependencies -uvicorn==0.18.3 # The lightning-fast ASGI server. +uvicorn==0.23.2 # The lightning-fast ASGI server. uvloop==0.17.0 # Fast implementation of asyncio event loop on top of libuv # Sysinfo related libs gputil==1.4.0 # Python module for getting the GPU status from NVIDA GPUs using nvidia-smi. -psutil==5.9.4 # Cross-platform lib for process and system monitoring in Python. +psutil==5.9.5 # Cross-platform lib for process and system monitoring in Python. # Scheduler -apscheduler==3.10.0 # APScheduler is a Python library that lets you schedule your Python code to be executed later, either just once or periodically +apscheduler==3.10.4 # APScheduler is a Python library that lets you schedule your Python code to be executed later, either just once or periodically # Testing -pytest==7.2.0 # Framework makes it easy to write small tests, yet scales to support complex functional testing for applications and libraries. -pytest-asyncio==0.20.2 # Library for testing asyncio code with pytest. -pytest-mock==3.10.0 # Library for mock data with pytest. \ No newline at end of file +pytest==7.4.2 # Framework makes it easy to write small tests, yet scales to support complex functional testing for applications and libraries. +pytest-asyncio==0.21.1 # Library for testing asyncio code with pytest. +pytest-mock==3.11.1 # Library for mock data with pytest. \ No newline at end of file