From 26a32e125a7105ef97e9af1b72543795fb4de1e0 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Tue, 6 Aug 2024 10:53:47 -0400 Subject: [PATCH 1/2] small tweaks to coordinates and main query --- python/valis/db/queries.py | 6 +++--- python/valis/routes/query.py | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index ac0c526..35b8bb7 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -113,8 +113,8 @@ def convert_coords(ra: Union[str, float], dec: Union[str, float]) -> tuple: """ is_hms = set('hms: ') & set(str(ra)) if is_hms: - ra = str(ra).replace(' ', ':') - dec = str(dec).replace(' ', ':') + ra = str(ra).strip().replace(' ', ':') + dec = str(dec).strip().replace(' ', ':') unit = ('hourangle', 'degree') if is_hms else ('degree', 'degree') coord = SkyCoord(f'{ra} {dec}', unit=unit) ra = round(coord.ra.value, 5) @@ -182,7 +182,7 @@ def get_targets_by_sdss_id(sdss_id: Union[int, list[int]] = []) -> peewee.ModelS peewee.ModelSelect the ORM query """ - if type(sdss_id) is int: + if type(sdss_id) in (int, str): sdss_id = [sdss_id] return vizdb.SDSSidStacked.select().where(vizdb.SDSSidStacked.sdss_id.in_(sdss_id)) diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index e034af4..5e790af 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -82,6 +82,7 @@ async def main_search(self, body: SearchModel): """ Main query for UI and for combining queries together """ print('form data', body) + query = None # build the coordinate query if body.ra and body.dec: @@ -97,9 +98,13 @@ async def main_search(self, body: SearchModel): 'program' if body.program else 'carton', query=query) # append query to pipes - query = append_pipes(query, observed=body.observed) + if query: + query = append_pipes(query, observed=body.observed) - return {'status': 'success', 'data': query.dicts().iterator(), 'msg': 'data successfully retrieved'} + # query iterator + res = query.dicts().iterator() if query else [] + + return {'status': 'success', 'data': res, 'msg': 'data successfully retrieved'} @router.get('/cone', summary='Perform a cone search for SDSS targets with sdss_ids', response_model=List[SDSSModel], dependencies=[Depends(get_pw_db)]) From b30d03db351bf564a8303a6ddb7f6e488fb993b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez-Gallego?= Date: Thu, 8 Aug 2024 07:54:06 -0700 Subject: [PATCH 2/2] Include parent catalog associations in /target/catalogs endpoint (#37) * Include parent catalog associations in /target/catalogs endpoint * Exclude parent catalogues with None values when serialising * Exclude nones in parent_catalogs only if exclude_none=True Co-authored-by: Brian Cherinka * Missing FieldSerializationInfo import * Simplify code by allowing get_parent_catalogs to accept a list * Trim parent catalog strings * Use single query and rearrange the response data * Update changelog * Remove unused function get_parent_catalogs --------- Co-authored-by: Brian Cherinka --- CHANGELOG.rst | 3 ++- python/valis/db/models.py | 16 +++++++++++++--- python/valis/db/queries.py | 8 +++++--- python/valis/routes/target.py | 16 ++++++++++++++-- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2102651..fe1f4a2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,7 +13,8 @@ Change Log * Adds endpoints for target search by carton and program * Adds ``db_reset`` option to avoid closing and resetting the database connection * Updates main, cone, and carton/program search to use new ``has_been_observed`` column; default is True. -* Updates the ``append_pipes`` query to return the new ``has_been_observed`` column +* Updates the ``append_pipes`` query to return the new ``has_been_observed`` column. +* Include parent catalog associations in the ``/target/catalogs`` endpoint (#37). 0.1.0 (10-24-2023) ------------------ diff --git a/python/valis/db/models.py b/python/valis/db/models.py index d5ff842..7276961 100644 --- a/python/valis/db/models.py +++ b/python/valis/db/models.py @@ -7,7 +7,7 @@ import datetime import math from typing import Optional, Annotated, Any, TypeVar -from pydantic import ConfigDict, BaseModel, Field, BeforeValidator, field_validator, FieldValidationInfo +from pydantic import ConfigDict, BaseModel, Field, BeforeValidator, FieldSerializationInfo, field_serializer, field_validator, FieldValidationInfo from enum import Enum @@ -194,7 +194,17 @@ class CatalogModel(PeeweeBase): class CatalogResponse(CatalogModel, SDSSidFlatBase): """ Response model for source catalog and sdss_id information """ - pass + + parent_catalogs: dict[str, Any] = Field(..., description='The parent catalog associations for a given catalogid') + + @field_serializer('parent_catalogs') + def serialize_parent_catalogs(v: dict[str, Any], info: FieldSerializationInfo) -> dict[str, Any]: + """ Serialize the parent catalogs, excluding None values and trimming strings.""" + + if info.exclude_none: + return {k: v.strip() if isinstance(v, str) else v for k, v in v.items() if v is not None} + else: + return v class CartonModel(PeeweeBase): @@ -243,4 +253,4 @@ class MapperName(str, Enum): """Mapper names""" MWM: str = 'MWM' BHM: str = 'BHM' - LVM: str = 'LVM' \ No newline at end of file + LVM: str = 'LVM' diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index 35b8bb7..553af34 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -6,7 +6,7 @@ import itertools import packaging -from typing import Union, Generator +from typing import Sequence, Union, Generator import astropy.units as u import deepmerge @@ -690,8 +690,10 @@ def get_catalog_sources(sdss_id: int) -> peewee.ModelSelect: """ s = vizdb.SDSSidFlat.select(vizdb.SDSSidFlat).where(vizdb.SDSSidFlat.sdss_id == sdss_id).alias('s') - return cat.Catalog.select(cat.Catalog, starfields(s)).\ - join(s, on=(s.c.catalogid == cat.Catalog.catalogid)).order_by(cat.Catalog.version.desc()) + return cat.Catalog.select(cat.Catalog, cat.SDSS_ID_To_Catalog, starfields(s)).\ + join(s, on=(s.c.catalogid == cat.Catalog.catalogid)).\ + join(cat.SDSS_ID_To_Catalog, on=(s.c.catalogid == cat.SDSS_ID_To_Catalog.catalogid)).\ + order_by(cat.Catalog.version.desc()) def get_target_cartons(sdss_id: int) -> peewee.ModelSelect: diff --git a/python/valis/routes/target.py b/python/valis/routes/target.py index c56e491..9baf94a 100644 --- a/python/valis/routes/target.py +++ b/python/valis/routes/target.py @@ -4,7 +4,7 @@ import re import httpx import orjson -from typing import Tuple, List, Union, Optional, Annotated +from typing import Any, Tuple, List, Union, Optional, Annotated from pydantic import field_validator, model_validator, BaseModel, Field, model_serializer from fastapi import APIRouter, HTTPException, Query, Path, Depends from fastapi_restful.cbv import cbv @@ -181,7 +181,19 @@ async def get_spectrum(self, sdss_id: Annotated[int, Path(title="The sdss_id of response_model_exclude_unset=True, response_model_exclude_none=True) async def get_catalogs(self, sdss_id: int = Path(title="The sdss_id of the target to get", example=23326)): """ Return catalog information for a given sdss_id """ - return get_catalog_sources(sdss_id).dicts().iterator() + + sdss_id_data = get_catalog_sources(sdss_id).dicts() + + # The response has the parent catalogs at the same level as the other + # columns. For the response we want to nest them under a parent_catalogs key. + # This takes advantage that all the parent catalog columns have '__' in the name. + response_data: list[dict[str, Any]] = [] + for row in sdss_id_data: + s_data = {k: v for k, v in row.items() if '__' not in k} + cat_data = {k.split('__')[0]: v for k, v in row.items() if '__' in k} + response_data.append({**s_data, 'parent_catalogs': cat_data}) + + return response_data @router.get('/cartons/{sdss_id}', summary='Retrieve carton information for a target sdss_id', dependencies=[Depends(get_pw_db)],