diff --git a/poetry.lock b/poetry.lock index 4f08d50..6100a7a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1953,7 +1953,7 @@ files = [ name = "h5py" version = "3.11.0" description = "Read and write HDF5 files from Python" -optional = false +optional = true python-versions = ">=3.8" files = [ {file = "h5py-3.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1625fd24ad6cfc9c1ccd44a66dac2396e7ee74940776792772819fc69f3a3731"}, @@ -3959,12 +3959,12 @@ files = [ [[package]] name = "peewee" -version = "3.17.5" +version = "3.17.6" description = "a little orm" optional = false python-versions = "*" files = [ - {file = "peewee-3.17.5.tar.gz", hash = "sha256:e1b6a64192207fd3ddb4e1188054820f42aef0aadfa749e3981af3c119a76420"}, + {file = "peewee-3.17.6.tar.gz", hash = "sha256:cea5592c6f4da1592b7cff8eaf655be6648a1f5857469e30037bf920c03fb8fb"}, ] [[package]] @@ -5741,7 +5741,7 @@ test = ["pytest", "pytest-doctestplus"] type = "git" url = "https://github.com/sdss/sdss_solara.git" reference = "main" -resolved_reference = "22d46277cdd42a9426b984bbea4cca8f5a243c6e" +resolved_reference = "2223f2496221fdfa6d44baa3dc9839c7781c6422" [[package]] name = "sdss-tree" @@ -5765,30 +5765,26 @@ docs = ["Sphinx (>=7.2.0)", "importlib-metadata (>=1.6.0)", "jinja2 (<3.1)", "re [[package]] name = "sdssdb" -version = "0.12.4" +version = "0.13.1" description = "SDSS product for database management" optional = false python-versions = ">=3.6" files = [ - {file = "sdssdb-0.12.4-py3-none-any.whl", hash = "sha256:cba5393621d6a7a455954c8a0ca695f91846e109043435669545b01804b6bd11"}, - {file = "sdssdb-0.12.4.tar.gz", hash = "sha256:ac4c1e021cc3454c1fed319cf6e646041af7378d322b0bb89f852366e6471e1a"}, + {file = "sdssdb-0.13.1-py3-none-any.whl", hash = "sha256:d37df0af9b8966c3507625e6f9bbce438cc1fd3f10d3be37911312b334adfc1c"}, + {file = "sdssdb-0.13.1.tar.gz", hash = "sha256:76512a7a5b8eead5603d7e08616cd4a3b701f489f5cbeb2aba28663cd0ea03c3"}, ] [package.dependencies] -h5py = ">=3.8.0" numpy = ">=1.18.2" -peewee = ">=3.17.3" +peewee = ">=3.17.6" pgpasslib = ">=1.1.0" psycopg2-binary = ">=2.7.7" -pygments = "*" -pyyaml = ">=5.1" sdsstools = {version = ">=1.7.0", markers = "python_version >= \"3.8\""} -six = ">=1.12.0" -sqlalchemy = ">=1.3.6" +sqlalchemy = ">=1.3.6,<2" [package.extras] -all = ["astropy (>=4.0.0)", "inflect (>=4.1.0)", "pandas (>=1.0.0)", "progressbar2 (>=3.46.1)", "pydot (>=1.4.1)"] -dev = ["astropy (>=4.0.0)", "factory-boy (>=2.12.0)", "ipdb (>=0.13.2)", "ipython (>=7.13.0)", "pydot (>=1.4.2)", "pytest (>=5.2)", "pytest-cov (>=2.4.0)", "pytest-factoryboy (>=2.0.3)", "pytest-postgresql (>=2.2.1)", "pytest-sugar (>=0.8.0)"] +all = ["astropy (>=4.0.0)", "h5py (>=3.8.0)", "inflect (>=4.1.0)", "pandas (>=1.0.0)", "progressbar2 (>=3.46.1)", "psycopg[binary]", "pydot (>=1.4.1)", "sdssdb[dev]", "sdssdb[docs]"] +dev = ["astropy (>=4.0.0)", "factory-boy (>=2.12.0)", "flake8 (>=7.1.1)", "flake8-pyproject (>=1.2.3)", "ipdb (>=0.13.2)", "ipython (>=7.13.0)", "pydot (>=1.4.2)", "pytest (>=5.2)", "pytest-cov (>=2.4.0)", "pytest-factoryboy (>=2.0.3)", "pytest-postgresql (>=2.2.1,<6)", "pytest-sugar (>=0.8.0)", "pyyaml (>=5.1)", "ruff (>=0.6.3)"] docs = ["Sphinx (>=7.0.0)", "releases (>=2.0.0)", "sphinx-bootstrap-theme (>=0.4.12)"] [[package]] @@ -7581,4 +7577,4 @@ solara = ["sdss-solara"] [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "9a16bfa5f5c433e7e4286f22829972f472a0ca88ef7f97789862dd236472c0f9" +content-hash = "5a2ad6f3768c18f548cd59796a0e6d049bb02c54cfa5bff02fc6210a9bcea88a" diff --git a/pyproject.toml b/pyproject.toml index b5f2315..6ea5476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ httpx = "^0.24.0" astroquery = "^0.4.6" pandas = "^1.5.3" SQLAlchemy = "^1.4.35" -sdssdb = "^0.12.4" +sdssdb = "^0.13.1" deepmerge = "^1.1.1" fuzzy-types = "^0.1.3" sdss-solara = {git = "https://github.com/sdss/sdss_solara.git", rev = "main", optional = true} diff --git a/python/valis/db/models.py b/python/valis/db/models.py index d7bc61c..0f12760 100644 --- a/python/valis/db/models.py +++ b/python/valis/db/models.py @@ -54,6 +54,11 @@ class SDSSidStackedBase(PeeweeBase): catalogid21: Optional[int] = Field(None, description='the version 21 catalog id') catalogid25: Optional[int] = Field(None, description='the version 25 catalog id') catalogid31: Optional[int] = Field(None, description='the version 31 catalog id') + last_updated: datetime.date = Field(None, description='the date the sdss_id row was last updated', exclude=True) + + @field_serializer('last_updated') + def serialize_dt(self, date: datetime.date) -> str: + return date.isoformat() class SDSSidFlatBase(PeeweeBase): @@ -67,7 +72,7 @@ class SDSSidFlatBase(PeeweeBase): n_associated: int = Field(..., description='The total number of sdss_ids associated with that catalogid.') ra_catalogid: Optional[float] = Field(None, description='Right Ascension, in degrees, specific to the catalogid') dec_catalogid: Optional[float] = Field(None, description='Declination, in degrees, specific to the catalogid') - + rank: int = Field(..., description='Ranking when catalogid paired to multiple sdss_id, with rank 1 as priority.') class SDSSidPipesBase(PeeweeBase): """ Pydantic response model for the Peewee vizdb.SDSSidToPipes ORM """ @@ -75,8 +80,12 @@ class SDSSidPipesBase(PeeweeBase): sdss_id: int = Field(..., description='the SDSS identifier') in_boss: bool = Field(..., description='Flag if target is in the BHM reductions', examples=[False]) in_apogee: bool = Field(..., description='Flag if target is in the MWM reductions', examples=[False]) + in_bvs: bool = Field(..., description='Flag if target is in the boss component of the Astra reductions', examples=[False], exclude=True) in_astra: bool = Field(..., description='Flag if the target is in the Astra reductions', examples=[False]) has_been_observed: Optional[bool] = Field(False, validate_default=True, description='Flag if target has been observed or not', examples=[False]) + release: Optional[str] = Field(None, description='the Astra release field, either sdss5 or dr17') + obs: Optional[str] = Field(None, description='the observatory the observation is from') + mjd: Optional[int] = Field(None, description='the MJD of the data reduction') @field_validator('has_been_observed') @classmethod diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index b026922..7f19c9b 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -60,8 +60,12 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked', model = vizdb.SDSSidStacked if table == 'stacked' else vizdb.SDSSidFlat qq = query.select_extend(vizdb.SDSSidToPipes.in_boss, vizdb.SDSSidToPipes.in_apogee, + vizdb.SDSSidToPipes.in_bvs, vizdb.SDSSidToPipes.in_astra, - vizdb.SDSSidToPipes.has_been_observed).\ + vizdb.SDSSidToPipes.has_been_observed, + vizdb.SDSSidToPipes.release, + vizdb.SDSSidToPipes.obs, + vizdb.SDSSidToPipes.mjd).\ join(vizdb.SDSSidToPipes, on=(model.sdss_id == vizdb.SDSSidToPipes.sdss_id), attr='pipes').distinct(vizdb.SDSSidToPipes.sdss_id) @@ -356,7 +360,7 @@ def get_targets_obs(release: str, obs: str, spectrograph: str) -> peewee.ModelSe # 10 - all false # 57651832 - my file on disk # 57832526 - all true, in both astra snow_white, apogee_net (source_pk=912174,star_pk=2954029) - +# 61731453 - in astra, false all else; dr17 release def get_boss_target(sdss_id: int, release: str, fields: list = None, primary: bool = True) -> peewee.ModelSelect: @@ -408,24 +412,30 @@ def get_apogee_target(sdss_id: int, release: str, fields: list = None): # get the relevant software tag apred = get_software_tag(release, 'apred_vers') + # create apogee version conditions if isinstance(apred, list): vercond = apo.Star.apred_vers.in_(apred) + avsver = astra.ApogeeVisitSpectrum.apred.in_(apred) else: vercond = apo.Star.apred_vers == apred + avsver = astra.ApogeeVisitSpectrum.apred == apred # check fields fields = fields or [apo.Star] if fields and isinstance(fields[0], str): fields = (getattr(apo.Star, i) for i in fields) + # get the astra source for the sdss_id s = get_astra_target(sdss_id, release) if not s: return - a = s.first().apogee_visit_spectrum.first() + # get the astra apogee visit spectrum + a = s.first().apogee_visit_spectrum.where(avsver).first() if not a: return + # get the apogee star data return apo.Star.select(*fields).where(apo.Star.pk == a.star_pk, vercond) @@ -433,8 +443,8 @@ def get_astra_target(sdss_id: int, release: str, fields: list = None): """ temporary placeholder for astra """ vastra = get_software_tag(release, 'v_astra') - if not vastra or vastra != "0.5.0": - print('astra only supports DR19 / IPL3 = version 0.5.0') + if not vastra or vastra not in ("0.5.0", "0.6.0"): + print('astra only supports DR19 / IPL3 = version 0.5.0, 0.6.0') return None # check fields @@ -469,7 +479,7 @@ def get_target_meta(sdss_id: int, release: str) -> dict: def get_pipe_meta(sdss_id: int, release: str, pipeline: str) -> dict: - """ Get the pipeline metadata for a pipeline + """ Get the pipeline reduction data for a pipeline Parameters ---------- @@ -553,6 +563,12 @@ def get_target_pipeline(sdss_id: int, release: str, pipeline: str = 'all') -> di if pipes['in_astra'] and (res := get_pipe_meta(sdss_id, release, 'astra')): deepmerge.always_merger.merge(data, res) + if pipes['release'] == 'dr17': + s = get_astra_target(sdss_id, release) + v = s.first().apogee_visit_spectrum.where(astra.ApogeeVisitSpectrum.apred == 'dr17').dicts().first() + path = build_apogee_path(v, 'DR17') + deepmerge.always_merger.merge(data, {'files': {'apogee': path}}) + return data diff --git a/python/valis/routes/info.py b/python/valis/routes/info.py index 8c3aff9..e15a3e2 100644 --- a/python/valis/routes/info.py +++ b/python/valis/routes/info.py @@ -52,7 +52,7 @@ def convert_metadata(data) -> dict: """ mm = defaultdict(dict) for i in itertools.chain(data, gen_misc_models()): - mm[i['schema']].update({i['column_name']: i}) + mm[i['schema']].update({f"{i['table_name']}.{i['column_name']}": i}) return mm diff --git a/python/valis/utils/paths.py b/python/valis/utils/paths.py index e7d228d..164ef0b 100644 --- a/python/valis/utils/paths.py +++ b/python/valis/utils/paths.py @@ -152,6 +152,7 @@ def build_apogee_path(values: dict, release: str, ignore_existence: bool = False """ return build_file_path(values, 'apStar', release, remap={'obj': 'apogee_id', 'apred': 'apred_vers'}, + defaults={'apstar': 'stars'}, ignore_existence=ignore_existence)