From 93a500fd6a4f487c46fdd56aa3dde12c4786f5ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Mon, 9 Dec 2024 10:28:28 -0800 Subject: [PATCH 1/3] Prerun input query in `append_pipes` (#63) * Prerun input query to append_pipes and add limit option * Add an ANALYZE after creating the temporary table * Tweak carton/program query with specific settings --- python/valis/db/queries.py | 62 +++++++++++++++++++++++++++++------- python/valis/routes/query.py | 19 +++++++++-- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index 1209a33..0590682 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -4,8 +4,10 @@ # all resuable queries go here +from contextlib import contextmanager import itertools import packaging +import uuid from typing import Sequence, Union, Generator import astropy.units as u @@ -57,17 +59,20 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked', if table not in {'stacked', 'flat'}: raise ValueError('table must be either "stacked" or "flat"') - model = vizdb.SDSSidStacked if table == 'stacked' else vizdb.SDSSidFlat - qq = query.select_extend(vizdb.SDSSidToPipes.in_boss, - vizdb.SDSSidToPipes.in_apogee, - vizdb.SDSSidToPipes.in_bvs, - vizdb.SDSSidToPipes.in_astra, - vizdb.SDSSidToPipes.has_been_observed, - vizdb.SDSSidToPipes.release, - vizdb.SDSSidToPipes.obs, - vizdb.SDSSidToPipes.mjd).\ - join(vizdb.SDSSidToPipes, on=(model.sdss_id == vizdb.SDSSidToPipes.sdss_id), - attr='pipes').distinct(vizdb.SDSSidToPipes.sdss_id) + # Run initial query as a temporary table. + temp = create_temporary_table(query, indices=['sdss_id']) + + qq = temp.select(temp.__star__, + vizdb.SDSSidToPipes.in_boss, + vizdb.SDSSidToPipes.in_apogee, + vizdb.SDSSidToPipes.in_bvs, + vizdb.SDSSidToPipes.in_astra, + vizdb.SDSSidToPipes.has_been_observed, + vizdb.SDSSidToPipes.release, + vizdb.SDSSidToPipes.obs, + vizdb.SDSSidToPipes.mjd).\ + join(vizdb.SDSSidToPipes, on=(temp.c.sdss_id == vizdb.SDSSidToPipes.sdss_id)).\ + distinct(temp.c.sdss_id) if observed: qq = qq.where(vizdb.SDSSidToPipes.has_been_observed == observed) @@ -264,7 +269,8 @@ def carton_program_map(key: str = 'program') -> dict: def carton_program_search(name: str, name_type: str, - query: peewee.ModelSelect | None = None) -> peewee.ModelSelect: + query: peewee.ModelSelect | None = None, + limit: int | None = None) -> peewee.ModelSelect: """ Perform a search on either carton or program Parameters @@ -276,6 +282,8 @@ def carton_program_search(name: str, query : ModelSelect An initial query to extend. If ``None``, a new query with all the unique ``sdss_id``s is created. + limit : int + Limit the number of results returned. Returns ------- @@ -286,6 +294,13 @@ def carton_program_search(name: str, if query is None: query = vizdb.SDSSidStacked.select(vizdb.SDSSidStacked).distinct() + # NOTE: These setting seem to help when querying some cartons or programs, mainly + # those with small number of targets, and in some cases with these the query + # actually applies the LIMIT more efficiently, but it's not a perfect solution. + vizdb.database.execute_sql('SET enable_gathermerge = off;') + vizdb.database.execute_sql('SET parallel_tuple_cost = 100;') + vizdb.database.execute_sql('SET enable_bitmapscan = off;') + query = (query.join( vizdb.SDSSidFlat, on=(vizdb.SDSSidFlat.sdss_id == vizdb.SDSSidStacked.sdss_id)) @@ -295,6 +310,9 @@ def carton_program_search(name: str, .join(targetdb.Carton) .where(getattr(targetdb.Carton, name_type) == name)) + if limit: + query = query.limit(limit) + return query def get_targets_obs(release: str, obs: str, spectrograph: str) -> peewee.ModelSelect: @@ -931,3 +949,23 @@ def get_target_by_altid(id: str | int, idtype: str = None) -> peewee.ModelSelect # get the sdss_id metadata info return get_targets_by_sdss_id(res.sdss_id) + + +def create_temporary_table(query: peewee.ModelSelect, + indices: list[str] | None = None) -> Generator[None, None, peewee.Table]: + """Create a temporary table from a query.""" + + table_name = uuid.uuid4().hex[0:8] + + table = peewee.Table(table_name) + table.bind(vizdb.database) + + query.create_table(table_name, temporary=True) + + if indices: + for index in indices: + vizdb.database.execute_sql(f'CREATE INDEX ON "{table_name}" ({index})') + + vizdb.database.execute_sql(f'ANALYZE "{table_name}"') + + return table diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index 5a7d25c..866430c 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -41,6 +41,7 @@ class SearchModel(BaseModel): program: Optional[str] = Field(None, description='The program name', example='bhm_rm') carton: Optional[str] = Field(None, description='The carton name', example='bhm_rm_core') observed: Optional[bool] = Field(True, description='Flag to only include targets that have been observed', example=True) + limit: Optional[int] = Field(None, description='Limit the number of returned targets', example=100) class MainResponse(SDSSModel): """ Combined model from all individual query models """ @@ -105,6 +106,13 @@ async def main_search(self, body: SearchModel): query = carton_program_search(body.program or body.carton, 'program' if body.program else 'carton', query=query) + + # DANGER!!! This limit applies *before* the append_pipes call. If the + # append_pipes call includes observed=True we may have limited things in + # such a way that only unobserved or very few targets are returned. + if body.limit: + query = query.limit(body.limit) + # append query to pipes if query: query = append_pipes(query, observed=body.observed) @@ -203,12 +211,17 @@ async def carton_program(self, Query(enum=['carton', 'program'], description='Specify search on carton or program', example='carton')] = 'carton', - observed: Annotated[bool, Query(description='Flag to only include targets that have been observed', example=True)] = True): + observed: Annotated[bool, Query(description='Flag to only include targets that have been observed', example=True)] = True, + limit: Annotated[int | None, Query(description='Limit the number of returned targets', example=100)] = None): """ Perform a search on carton or program """ with database.atomic(): - database.execute_sql('SET LOCAL enable_seqscan=false;') - query = carton_program_search(name, name_type) + if limit is False: + # This tweak seems to do more harm than good when limit is passed. + database.execute_sql('SET LOCAL enable_seqscan=false;') + + query = carton_program_search(name, name_type, limit=limit) query = append_pipes(query, observed=observed) + return query.dicts().iterator() @router.get('/obs', summary='Return targets with spectrum at observatory', From dbf2ad5384a914c3c54981248ee3e171301b46b2 Mon Sep 17 00:00:00 2001 From: Brian Cherinka Date: Mon, 9 Dec 2024 10:39:30 -0800 Subject: [PATCH 2/3] adding release filter for queries using the mjd cutoffs (#72) --- python/valis/db/queries.py | 24 +++++++++++++++++++++++- python/valis/routes/query.py | 4 ++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index 0590682..2275d06 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -13,6 +13,7 @@ import astropy.units as u import deepmerge import peewee +from peewee import Case from astropy.coordinates import SkyCoord from sdssdb.peewee.sdss5db import apogee_drpdb as apo from sdssdb.peewee.sdss5db import boss_drp as boss @@ -28,7 +29,7 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked', - observed: bool = True) -> peewee.ModelSelect: + observed: bool = True, release: str = None) -> peewee.ModelSelect: """ Joins a query to the SDSSidToPipes table Joines an existing query to the SDSSidToPipes table and returns @@ -77,6 +78,27 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked', if observed: qq = qq.where(vizdb.SDSSidToPipes.has_been_observed == observed) + if release: + # get the release + rel = vizdb.Releases.select().where(vizdb.Releases.release==release).first() + + # if a release has no cutoff info, then force the cutoff to 0, query will return nothing + # to fix this we want mjd cutoffs by survey for all older releases + if not rel.mjd_cutoff_apo and not rel.mjd_cutoff_lco: + rel.mjd_cutoff_apo = 0 + rel.mjd_cutoff_lco = 0 + + # create the mjd cutoff condition + qq = qq.where(vizdb.SDSSidToPipes.mjd <= Case( + vizdb.SDSSidToPipes.obs, + ( + ('apo', rel.mjd_cutoff_apo), + ('lco', rel.mjd_cutoff_lco) + ), + None + ) + ) + return qq diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index 866430c..cdb5171 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -115,7 +115,7 @@ async def main_search(self, body: SearchModel): # append query to pipes if query: - query = append_pipes(query, observed=body.observed) + query = append_pipes(query, observed=body.observed, release=self.release) # query iterator res = query.dicts().iterator() if query else [] @@ -133,7 +133,7 @@ async def cone_search(self, """ Perform a cone search """ res = cone_search(ra, dec, radius, units=units) - r = append_pipes(res, observed=observed) + r = append_pipes(res, observed=observed, release=self.release) # return sorted by distance # doing this here due to the append_pipes distinct return sorted(r.dicts().iterator(), key=lambda x: x['distance']) From 6aa05309bd3206d5d3f0e4120283d4e4b200f249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 19 Dec 2024 07:51:01 -0800 Subject: [PATCH 3/3] Implement caching system (#73) * Implement caching system * memcache -> memcached * Add instructions for local development using memcached * Add note on Redis requirement for deployment * Use custom version of fastapi-cache decorator to support POST requests * Change CACHE_TTL to 6 months * Replace memcached with in-memory * Add a null cache backend * Remove memcache extra from fastapi-cache2 * Make cache_ttl a setting option * Add comment about in-memory backend * Improve valis_cache docstring * Set custom cache namespaces for target and query routes * Add function to clear namespaces in the Redis cache * Remove import added by autoimport * adding custom orjson encoder to cache, custom encoding memoryview objects * fixing missing import --------- Co-authored-by: havok2063 --- README.md | 7 +- poetry.lock | 181 +++++++++++++++-- pyproject.toml | 1 + python/valis/cache.py | 363 ++++++++++++++++++++++++++++++++++ python/valis/db/queries.py | 1 - python/valis/main.py | 29 +-- python/valis/routes/query.py | 19 +- python/valis/routes/target.py | 12 +- python/valis/settings.py | 27 ++- 9 files changed, 597 insertions(+), 43 deletions(-) create mode 100644 python/valis/cache.py diff --git a/README.md b/README.md index 3860f3e..dcdea4c 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ uvicorn valis.wsgi:app --reload ``` This will start a local web server at `http://localhost:8000/valis/`. The API documentation will be located at `http://localhost:8000/valis/docs`. Or to see the alternate documentation, go to `http://localhost:8000/valis/redoc/` +By default, the app will try to cache some route responses to a Redis database in localhost. If you don't have a Redis instance running you can use `in-memory` for testing (this caches the response directly in RAM). To do so, edit `~/.config/sdss/valis.yaml` and add `cache_backend: in-memory` (this should only be used in development or it could quickly use all available memory; the memory is freed when the app is stopped). Caching can be completely disabled by setting `cache_backend: null`. The time the cache is kept can be set with the `cache_ttl` (time to live) setting option. + ### Database Connection Valis uses the `sdssdb` package for all connections to databases. The most relevant database for the API is the `sdss5db` on `pipelines.sdss.org`. The easiest way to connect is through a local SSH tunnel. To set up a tunnel, @@ -91,8 +93,9 @@ Additionally, you can set the environment variable `VALIS_DB_RESET=false` or add ## Deployment This section describes a variety of deployment methods. Valis uses gunicorn as its -wsgi http server. It binds the app both to port 8000, and a unix socket. The defaut mode -is to start valis with an awsgi uvicorn server, with 4 workers. +wsgi http server. It binds the app both to port 8000, and a unix socket. The default mode is to start valis with an awsgi uvicorn server, with 4 workers. + +Valis requires a Redis database running at the default location in `localhost:6379`. If this is not possible, caching can be done in memory by modifying `~/.config/sdss/valis.yaml` to use `cache_backend: in-memory`. ### Deploying Zora + Valis together See the SDSS [Zora+Valis Docker](https://github.com/sdss/zora_valis_dockers) repo page. diff --git a/poetry.lock b/poetry.lock index 6100a7a..dd9a731 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiofiles" @@ -415,6 +415,17 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} +[[package]] +name = "async-timeout" +version = "5.0.1" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, +] + [[package]] name = "attrs" version = "23.2.0" @@ -1412,6 +1423,30 @@ uvicorn = {version = ">=0.12.0", extras = ["standard"], optional = true, markers [package.extras] all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.5)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] +[[package]] +name = "fastapi-cache2" +version = "0.2.2" +description = "Cache for FastAPI" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "fastapi_cache2-0.2.2-py3-none-any.whl", hash = "sha256:e1fae86d8eaaa6c8501dfe08407f71d69e87cc6748042d59d51994000532846c"}, + {file = "fastapi_cache2-0.2.2.tar.gz", hash = "sha256:71bf4450117dc24224ec120be489dbe09e331143c9f74e75eb6f576b78926026"}, +] + +[package.dependencies] +fastapi = "*" +pendulum = ">=3.0.0,<4.0.0" +redis = {version = ">=4.2.0rc1,<5.0.0", optional = true, markers = "extra == \"redis\" or extra == \"all\""} +typing-extensions = ">=4.1.0" +uvicorn = "*" + +[package.extras] +all = ["aiobotocore (>=2.13.1,<3.0.0)", "aiomcache (>=0.8.2,<0.9.0)", "redis (>=4.2.0rc1,<5.0.0)"] +dynamodb = ["aiobotocore (>=2.13.1,<3.0.0)"] +memcache = ["aiomcache (>=0.8.2,<0.9.0)"] +redis = ["redis (>=4.2.0rc1,<5.0.0)"] + [[package]] name = "fastapi-restful" version = "0.5.0" @@ -3967,6 +4002,105 @@ files = [ {file = "peewee-3.17.6.tar.gz", hash = "sha256:cea5592c6f4da1592b7cff8eaf655be6648a1f5857469e30037bf920c03fb8fb"}, ] +[[package]] +name = "pendulum" +version = "3.0.0" +description = "Python datetimes made easy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + +[package.dependencies] +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] + [[package]] name = "pexpect" version = "4.9.0" @@ -4262,7 +4396,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, @@ -4271,8 +4404,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, @@ -5011,7 +5142,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -5019,16 +5149,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -5045,7 +5167,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -5053,7 +5174,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -5322,6 +5442,24 @@ typing-extensions = ">=4.1.1" dev = ["black", "bqplot", "bump2version", "coverage", "flake8", "ipykernel", "ipyvuetify", "jinja2", "mypy", "numpy", "pandas", "pre-commit", "pytest", "pytest-cov"] generate = ["black", "bqplot", "jinja2", "mypy"] +[[package]] +name = "redis" +version = "4.6.0" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.7" +files = [ + {file = "redis-4.6.0-py3-none-any.whl", hash = "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c"}, + {file = "redis-4.6.0.tar.gz", hash = "sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""} + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + [[package]] name = "referencing" version = "0.35.1" @@ -6700,6 +6838,17 @@ files = [ mypy-extensions = ">=0.3.0" typing-extensions = ">=3.7.4" +[[package]] +name = "tzdata" +version = "2024.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, +] + [[package]] name = "ujson" version = "5.10.0" @@ -7577,4 +7726,4 @@ solara = ["sdss-solara"] [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "5a2ad6f3768c18f548cd59796a0e6d049bb02c54cfa5bff02fc6210a9bcea88a" +content-hash = "0a193f0210136effbb8253249007095c88f59d71f9831e1dc809d03a11605b91" diff --git a/pyproject.toml b/pyproject.toml index 6ea5476..ad36a3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ deepmerge = "^1.1.1" fuzzy-types = "^0.1.3" sdss-solara = {git = "https://github.com/sdss/sdss_solara.git", rev = "main", optional = true} markdown = "^3.7" +fastapi-cache2 = { version = "^0.2.2", extras = ["redis"] } [tool.poetry.dev-dependencies] ipython = ">=7.11.0" diff --git a/python/valis/cache.py b/python/valis/cache.py new file mode 100644 index 0000000..786c509 --- /dev/null +++ b/python/valis/cache.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Filename: main.py +# Project: app +# Author: José Sánchez-Gallego +# Created: Monday, 9th December 2024 +# License: BSD 3-clause "New" or "Revised" License +# Copyright (c) 2020 José Sánchez-Gallego +# Last Modified: Monday, 9th December 2024 +# Modified By: José Sánchez-Gallego + +from __future__ import annotations + +import base64 +import hashlib +import json +import logging +import re +import orjson +from contextlib import asynccontextmanager +from functools import wraps +from inspect import Parameter, isawaitable, iscoroutinefunction +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + List, + Optional, + ParamSpec, + Tuple, + Type, + TypeVar, + Union, + cast +) + +from fastapi.concurrency import run_in_threadpool +from fastapi.dependencies.utils import ( + get_typed_return_annotation, + get_typed_signature +) +from fastapi_cache import Backend, Coder, FastAPICache +from fastapi_cache.backends.inmemory import InMemoryBackend +from fastapi_cache.backends.redis import RedisBackend +from fastapi_cache.decorator import _augment_signature, _locate_param +from redis.asyncio.client import Redis as RedisAsync +from redis.client import Redis +from starlette.requests import Request +from starlette.responses import Response +from starlette.status import HTTP_304_NOT_MODIFIED + +from valis.settings import settings + + +if TYPE_CHECKING: + from typing import AsyncIterator + + from fastapi import FastAPI + from fastapi_cache.coder import Coder + from fastapi_cache.types import KeyBuilder + + +__all__ = ['valis_cache', 'lifespan', 'valis_cache_key_builder'] + + +P = ParamSpec("P") +R = TypeVar("R") + + +logger = logging.getLogger("uvicorn.error") + + +def bdefault(obj): + """ Custom encoder for orjson """ + # handle python memoryview objects + if isinstance(obj, memoryview): + return base64.b64encode(obj.tobytes()).decode() + raise TypeError + + +class ORJsonCoder(Coder): + """ Custom encoder class for the cache that uses orjson """ + + @classmethod + def encode(cls, value: Any) -> bytes: + """ serialization """ + return orjson.dumps( + value, + default=bdefault, + option=orjson.OPT_SERIALIZE_NUMPY, + ) + + @classmethod + def decode(cls, value: bytes) -> Any: + """ deserialization """ + return orjson.loads(value) + + +@asynccontextmanager +async def lifespan(_: FastAPI) -> AsyncIterator[None]: + backend = settings.cache_backend + if backend == 'in-memory': + logger.info('Using in-memory backend for caching') + FastAPICache.init(InMemoryBackend(), + prefix="fastapi-cache", + key_builder=valis_cache_key_builder) + elif backend == 'redis': + logger.info('Using Redis backend for caching') + redis = RedisAsync.from_url("redis://localhost") + FastAPICache.init(RedisBackend(redis), + prefix="fastapi-cache", + key_builder=valis_cache_key_builder) + elif backend == 'null' or not backend: + logger.info('Using null backend for caching') + FastAPICache.init(NullCacheBackend(), + prefix="fastapi-cache", + key_builder=valis_cache_key_builder) + else: + raise ValueError(f'Invalid cache backend {backend}') + + yield + + +async def valis_cache_key_builder( + func, + namespace: str = "", + request: Request | None = None, + _: Response | None = None, + *args, + **kwargs, +): + query_params = request.query_params.items() if request else [] + + try: + body_json = await request.json() + body = sorted(body_json.items()) if body_json else [] + except json.JSONDecodeError: + body = [] + + hash = hashlib.new('md5') + for param,value in list(query_params) + body: + hash.update(param.encode()) + hash.update(str(value).encode()) + + params_hash = hash.hexdigest()[0:8] + + url = request.url.path.replace('/', '_') if request else "" + if url.startswith('_'): + url = url[1:] + + chunks = [ + namespace, + request.method.lower() if request else "", + url, + params_hash, + ] + + return ":".join(chunks) + + +def valis_cache( + expire: Optional[int] = settings.cache_ttl, + coder: Optional[Type[Coder]] = ORJsonCoder, + key_builder: Optional[KeyBuilder] = None, + namespace: str = "valis-cache", + injected_dependency_namespace: str = "__fastapi_cache", +) -> Callable[[Callable[P, Awaitable[R]]], Callable[P, Awaitable[Union[R, Response]]]]: + """Caches an API route. + + This is a copy of the ``cache`` decorator from ``fastapi_cache`` with some + modifications to allow using it with POST requests. This version should be used + with a key builder that hashes the body of the request in addition to the function + arguments. + + The main change is that the call to ``fastapi_cache.decorator._uncacheable`` has + been removed and we accept all route types. `.valis_cache_key_builder` looks + at the body of the request and hashes its body so that POST requests to the same + route with different parameters are cached separately. It also defaults to + ``settings.cache_ttl`` for the expiration time of the cached value. + + """ + + injected_request = Parameter( + name=f"{injected_dependency_namespace}_request", + annotation=Request, + kind=Parameter.KEYWORD_ONLY, + ) + injected_response = Parameter( + name=f"{injected_dependency_namespace}_response", + annotation=Response, + kind=Parameter.KEYWORD_ONLY, + ) + + def wrapper( + func: Callable[P, Awaitable[R]] + ) -> Callable[P, Awaitable[Union[R, Response]]]: + # get_typed_signature ensures that any forward references are resolved first + wrapped_signature = get_typed_signature(func) + to_inject: List[Parameter] = [] + request_param = _locate_param(wrapped_signature, injected_request, to_inject) + response_param = _locate_param(wrapped_signature, injected_response, to_inject) + return_type = get_typed_return_annotation(func) + + @wraps(func) + async def inner(*args: P.args, **kwargs: P.kwargs) -> Union[R, Response]: + nonlocal coder + nonlocal expire + nonlocal key_builder + + async def ensure_async_func(*args: P.args, **kwargs: P.kwargs) -> R: + """Run cached sync functions in thread pool just like FastAPI.""" + # if the wrapped function does NOT have request or response in + # its function signature, make sure we don't pass them in as + # keyword arguments + kwargs.pop(injected_request.name, None) + kwargs.pop(injected_response.name, None) + + if iscoroutinefunction(func): + # async, return as is. + # unintuitively, we have to await once here, so that caller + # does not have to await twice. See + # https://stackoverflow.com/a/59268198/532513 + return await func(*args, **kwargs) + else: + # sync, wrap in thread and return async + # see above why we have to await even although caller also awaits. + return await run_in_threadpool(func, *args, **kwargs) # type: ignore[arg-type] + + copy_kwargs = kwargs.copy() + request: Optional[Request] = copy_kwargs.pop(request_param.name, None) # type: ignore[assignment] + response: Optional[Response] = copy_kwargs.pop(response_param.name, None) # type: ignore[assignment] + + prefix = FastAPICache.get_prefix() + coder = coder or FastAPICache.get_coder() + expire = expire or FastAPICache.get_expire() + key_builder = key_builder or FastAPICache.get_key_builder() + backend = FastAPICache.get_backend() + cache_status_header = FastAPICache.get_cache_status_header() + + cache_key = key_builder( + func, + f"{prefix}:{namespace}", + request=request, + response=response, + args=args, + kwargs=copy_kwargs, + ) + if isawaitable(cache_key): + cache_key = await cache_key + assert isinstance(cache_key, str) # noqa: S101 # assertion is a type guard + + try: + ttl, cached = await backend.get_with_ttl(cache_key) + except Exception: + logger.warning( + f"Error retrieving cache key '{cache_key}' from backend:", + exc_info=True, + ) + ttl, cached = 0, None + + if cached is None or (request is not None and request.headers.get("Cache-Control") == "no-cache"): # cache miss + result = await ensure_async_func(*args, **kwargs) + to_cache = coder.encode(result) + + try: + await backend.set(cache_key, to_cache, expire) + except Exception: + logger.warning( + f"Error setting cache key '{cache_key}' in backend:", + exc_info=True, + ) + + if response: + response.headers.update( + { + "Cache-Control": f"max-age={expire}", + "ETag": f"W/{hash(to_cache)}", + cache_status_header: "MISS", + } + ) + + else: # cache hit + if response: + etag = f"W/{hash(cached)}" + response.headers.update( + { + "Cache-Control": f"max-age={ttl}", + "ETag": etag, + cache_status_header: "HIT", + } + ) + + if_none_match = request and request.headers.get("if-none-match") + if if_none_match == etag: + response.status_code = HTTP_304_NOT_MODIFIED + return response + + result = cast(R, coder.decode_as_type(cached, type_=return_type)) + + return result + + inner.__signature__ = _augment_signature(wrapped_signature, *to_inject) # type: ignore[attr-defined] + + return inner + + return wrapper + + +class NullCacheBackend(Backend): + """A null cache backend that does no caching and always runs the route.""" + + async def get_with_ttl(self, key: str) -> Tuple[int, Optional[bytes]]: + return 0, None + + async def get(self, key: str) -> Optional[bytes]: + return None + + async def set(self, key: str, value: bytes, expire: Optional[int] = None) -> None: + pass + + async def clear(self, namespace: Optional[str] = None, key: Optional[str] = None) -> int: + pass + + +def clear_redis_cache(namespace: Optional[str] = None, + host: str = 'localhost', + port: int = 6379) -> None: + """Clears the Redis cache. + + Parameters + ---------- + namespace + The namespace to clear, e.g., ``valis-target``. If ``None``, all the + ``valis-*`` keys under the ``fastapi-cache`` namespace will be cleared. + host + The Redis host. + port + The Redis port. + + """ + + redis = Redis.from_url(f"redis://{host}:{port}") + + if namespace is None: + # There is no good way in Redis to delete an entire namespace so we need + # to get all the keys and delete them one by one. + keys = redis.keys("fastapi-cache:valis-*") + namespaces: set[str] = set() + for key in keys: + valis_namespace = re.match(rb"fastapi-cache:(valis-\w+):", key) + if valis_namespace: + namespaces.add(valis_namespace.group(1).decode()) + + else: + namespaces = {namespace} + + for namespace in namespaces: + # Same here. For each namespace we get its keys and delete them. + namespace_keys = redis.keys(f"fastapi-cache:{namespace}:*") + for key in namespace_keys: + redis.delete(key) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index 2275d06..9da315d 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -21,7 +21,6 @@ from sdssdb.peewee.sdss5db import catalogdb as cat from sdssdb.peewee.sdss5db import astradb as astra - from valis.db.models import MapperName from valis.io.spectra import extract_data, get_product_model from valis.utils.paths import build_boss_path, build_apogee_path, build_astra_path diff --git a/python/valis/main.py b/python/valis/main.py index 7fda5cd..692ed76 100644 --- a/python/valis/main.py +++ b/python/valis/main.py @@ -16,7 +16,6 @@ import os import pathlib from typing import Dict -from functools import lru_cache from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -24,10 +23,22 @@ from fastapi.staticfiles import StaticFiles import valis -from valis.routes import access, auth, envs, files, info, maskbits, mocs, target, query +from valis.cache import lifespan +from valis.routes import ( + access, + auth, + envs, + files, + info, + maskbits, + mocs, + query, + target +) from valis.routes.auth import set_auth from valis.routes.base import release -from valis.settings import Settings, read_valis_config +from valis.settings import settings + # set up the solara server try: @@ -91,19 +102,9 @@ }, ] - -@lru_cache -def get_settings(): - """ Get the valis settings """ - cfg = read_valis_config() - return Settings(**cfg) - - -settings = get_settings() - # create the application app = FastAPI(title='Valis', description='The SDSS API', version=valis.__version__, - openapi_tags=tags_metadata, dependencies=[]) + openapi_tags=tags_metadata, lifespan=lifespan, dependencies=[]) # submount app to allow for production /valis location app.mount("/valis", app) diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index cdb5171..4459245 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -8,6 +8,7 @@ from fastapi_restful.cbv import cbv from pydantic import BaseModel, Field, BeforeValidator +from valis.cache import valis_cache from valis.routes.base import Base from valis.db.db import get_pw_db from valis.db.models import SDSSidStackedBase, SDSSidPipesBase, MapperName, SDSSModel @@ -83,6 +84,7 @@ class QueryRoutes(Base): dependencies=[Depends(get_pw_db)], response_model=MainSearchResponse, response_model_exclude_unset=True, response_model_exclude_none=True) + @valis_cache(namespace='valis-query') async def main_search(self, body: SearchModel): """ Main query for UI and for combining queries together """ @@ -117,13 +119,15 @@ async def main_search(self, body: SearchModel): if query: query = append_pipes(query, observed=body.observed, release=self.release) - # query iterator - res = query.dicts().iterator() if query else [] + # Results. Note that we cannot return an iterator in a cached route or the + # initial query (when it does not hit the cache) will return an empty list. + res = list(query.dicts()) if query else [] return {'status': 'success', 'data': res, 'msg': 'data successfully retrieved'} @router.get('/cone', summary='Perform a cone search for SDSS targets with sdss_ids', response_model=List[SDSSModel], dependencies=[Depends(get_pw_db)]) + @valis_cache(namespace='valis-query') async def cone_search(self, ra: Annotated[Union[float, str], Query(description='Right Ascension in degrees or hmsdms', example=315.78)], dec: Annotated[Union[float, str], Query(description='Declination in degrees or hmsdms', example=-3.2)], @@ -136,10 +140,11 @@ async def cone_search(self, r = append_pipes(res, observed=observed, release=self.release) # return sorted by distance # doing this here due to the append_pipes distinct - return sorted(r.dicts().iterator(), key=lambda x: x['distance']) + return sorted(r.dicts(), key=lambda x: x['distance']) @router.get('/sdssid', summary='Perform a search for an SDSS target based on the sdss_id', response_model=Union[SDSSidStackedBase, dict], dependencies=[Depends(get_pw_db)]) + @valis_cache(namespace='valis-query') async def sdss_id_search(self, sdss_id: Annotated[int, Query(description='Value of sdss_id', example=47510284)]): """ Perform an sdss_id search. @@ -171,6 +176,7 @@ async def catalog_id_search(self, catalog_id: Annotated[int, Query(description=' @router.get('/list/cartons', summary='Return a list of all cartons', response_model=list, dependencies=[Depends(get_pw_db)]) + @valis_cache(namespace='valis-query') async def cartons(self): """ Return a list of all carton or programs """ @@ -178,6 +184,7 @@ async def cartons(self): @router.get('/list/programs', summary='Return a list of all programs', response_model=list, dependencies=[Depends(get_pw_db)]) + @valis_cache(namespace='valis-query') async def programs(self): """ Return a list of all carton or programs """ @@ -185,6 +192,7 @@ async def programs(self): @router.get('/list/program-map', summary='Return a mapping of cartons in all programs', response_model=Dict[str, List[str]], dependencies=[Depends(get_pw_db)]) + @valis_cache(namespace='valis-query') async def program_map(self): """ Return a mapping of cartons in all programs """ @@ -192,6 +200,7 @@ async def program_map(self): @router.get('/list/parents', summary='Return a list of available parent catalog tables', response_model=List[str]) + @valis_cache(namespace='valis-query') async def parent_catalogs(self): """Return a list of available parent catalog tables.""" @@ -205,6 +214,7 @@ async def parent_catalogs(self): @router.get('/carton-program', summary='Search for all SDSS targets within a carton or program', response_model=List[SDSSModel], dependencies=[Depends(get_pw_db)]) + @valis_cache(namespace='valis-query') async def carton_program(self, name: Annotated[str, Query(description='Carton or program name', example='manual_mwm_tess_ob')], name_type: Annotated[str, @@ -222,7 +232,8 @@ async def carton_program(self, query = carton_program_search(name, name_type, limit=limit) query = append_pipes(query, observed=observed) - return query.dicts().iterator() + # The list() is necessary here to not return a generator in the cached route. + return list(query.dicts()) @router.get('/obs', summary='Return targets with spectrum at observatory', response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) diff --git a/python/valis/routes/target.py b/python/valis/routes/target.py index 91745a4..c5f39d8 100644 --- a/python/valis/routes/target.py +++ b/python/valis/routes/target.py @@ -11,14 +11,15 @@ import astropy.units as u from astropy.coordinates import SkyCoord from astroquery.simbad import Simbad - from valis.routes.base import Base +from valis.cache import valis_cache from valis.db.queries import (get_target_meta, get_a_spectrum, get_catalog_sources, get_parent_catalog_data, get_target_cartons, get_target_pipeline, get_target_by_altid, append_pipes) from valis.db.db import get_pw_db from valis.db.models import CatalogResponse, CartonModel, ParentCatalogModel, PipesModel, SDSSModel + router = APIRouter() Simbad.add_votable_fields('distance_result') @@ -182,16 +183,18 @@ async def get_target_altid(self, @router.get('/spectra/{sdss_id}', summary='Retrieve a spectrum for a target sdss_id', dependencies=[Depends(get_pw_db)], response_model=List[SpectrumModel]) + @valis_cache(namespace='valis-target') async def get_spectrum(self, sdss_id: Annotated[int, Path(title="The sdss_id of the target to get", example=23326)], product: Annotated[str, Query(description='The file species or data product name', example='specLite')], ext: Annotated[str, Query(description='For multi-extension spectra, e.g. mwmStar, the name of the spectral extension', example='BOSS/APO')] = None, ): - return get_a_spectrum(sdss_id, product, self.release, ext=ext) + return list(get_a_spectrum(sdss_id, product, self.release, ext=ext)) @router.get('/catalogs/{sdss_id}', summary='Retrieve catalog information for a target sdss_id', dependencies=[Depends(get_pw_db)], response_model=List[CatalogResponse], response_model_exclude_unset=True, response_model_exclude_none=True) + @valis_cache(namespace='valis-target') async def get_catalogs(self, sdss_id: int = Path(title="The sdss_id of the target to get", example=23326)): """ Return catalog information for a given sdss_id """ @@ -213,6 +216,7 @@ async def get_catalogs(self, sdss_id: int = Path(title="The sdss_id of the targe response_model=list[ParentCatalogModel], responses={400: {'description': 'Invalid input sdss_id or catalog'}}, summary='Retrieve parent catalog information for a taget by sdss_id') + @valis_cache(namespace='valis-target') async def get_parents(self, catalog: Annotated[str, Path(description='The parent catalog to search', example='gaia_dr3_source')], @@ -240,14 +244,16 @@ async def get_parents(self, dependencies=[Depends(get_pw_db)], response_model=List[CartonModel], response_model_exclude_unset=True, response_model_exclude_none=True) + @valis_cache(namespace='valis-target') async def get_cartons(self, sdss_id: int = Path(title="The sdss_id of the target to get", example=23326)): """ Return carton information for a given sdss_id """ - return get_target_cartons(sdss_id).dicts().iterator() + return list(get_target_cartons(sdss_id).dicts()) @router.get('/pipelines/{sdss_id}', summary='Retrieve pipeline data for a target sdss_id', dependencies=[Depends(get_pw_db)], response_model=PipesModel, response_model_exclude_unset=True) + @valis_cache(namespace='valis-target') async def get_pipeline(self, sdss_id: int = Path(title="The sdss_id of the target to get", example=23326), pipe: Annotated[str, Query(enum=['all', 'boss', 'apogee', 'astra'], diff --git a/python/valis/settings.py b/python/valis/settings.py index 6a7bc5f..64b1cfd 100644 --- a/python/valis/settings.py +++ b/python/valis/settings.py @@ -3,10 +3,13 @@ # from enum import Enum -from typing import List, Union, Optional -from valis import config +from functools import lru_cache +from typing import List, Literal, Optional, Union + +from pydantic import AnyHttpUrl, Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict -from pydantic import field_validator, Field, AnyHttpUrl + +from valis import config def read_valis_config() -> dict: @@ -30,6 +33,12 @@ class EnvEnum(str, Enum): prod = 'production' +class CacheBackendEnum(str, Enum): + inmemory = 'in-memory' + redis = 'redis' + null = 'null' + + class Settings(BaseSettings): valis_env: EnvEnum = EnvEnum.dev allow_origin: Union[str, List[AnyHttpUrl]] = Field([]) @@ -40,6 +49,8 @@ class Settings(BaseSettings): db_host: Optional[str] = 'localhost' db_pass: Optional[str] = None db_reset: bool = True + cache_backend: CacheBackendEnum | None = CacheBackendEnum.inmemory + cache_ttl: int = 15552000 # 6 months model_config = SettingsConfigDict(env_prefix="valis_") @field_validator('allow_origin') @@ -53,3 +64,13 @@ def must_be_list(cls, v): @classmethod def strip_slash(cls, v): return [i.rstrip('/') for i in v] + + +@lru_cache +def get_settings(): + """ Get the valis settings """ + cfg = read_valis_config() + return Settings(**cfg) + + +settings = get_settings()