Support for astra dr17 spectra

pyproject.toml

@@ -45,7 +45,7 @@ httpx = "^0.24.0"
 astroquery = "^0.4.6"
 pandas = "^1.5.3"
 SQLAlchemy = "^1.4.35"
-sdssdb = "^0.12.4"
+sdssdb = "^0.13.1"
 deepmerge = "^1.1.1"
 fuzzy-types = "^0.1.3"
 sdss-solara = {git = "https://github.com/sdss/sdss_solara.git", rev = "main", optional = true}

python/valis/db/models.py

@@ -54,6 +54,11 @@ class SDSSidStackedBase(PeeweeBase):
     catalogid21: Optional[int] = Field(None, description='the version 21 catalog id')
     catalogid25: Optional[int] = Field(None, description='the version 25 catalog id')
     catalogid31: Optional[int] = Field(None, description='the version 31 catalog id')
+    last_updated: datetime.date = Field(None, description='the date the sdss_id row was last updated', exclude=True)
+
+    @field_serializer('last_updated')
+    def serialize_dt(self, date: datetime.date) -> str:
+        return date.isoformat()
 
 
 class SDSSidFlatBase(PeeweeBase):
@@ -67,16 +72,20 @@ class SDSSidFlatBase(PeeweeBase):
     n_associated: int = Field(..., description='The total number of sdss_ids associated with that catalogid.')
     ra_catalogid: Optional[float] = Field(None, description='Right Ascension, in degrees, specific to the catalogid')
     dec_catalogid: Optional[float] = Field(None, description='Declination, in degrees, specific to the catalogid')
-
+    rank: int = Field(..., description='Ranking when catalogid paired to multiple sdss_id, with rank 1 as priority.')
 
 class SDSSidPipesBase(PeeweeBase):
     """ Pydantic response model for the Peewee vizdb.SDSSidToPipes ORM """
 
     sdss_id: int = Field(..., description='the SDSS identifier')
     in_boss: bool = Field(..., description='Flag if target is in the BHM reductions', examples=[False])
     in_apogee: bool = Field(..., description='Flag if target is in the MWM reductions', examples=[False])
+    in_bvs: bool = Field(..., description='Flag if target is in the boss component of the Astra reductions', examples=[False], exclude=True)
     in_astra: bool = Field(..., description='Flag if the target is in the Astra reductions', examples=[False])
     has_been_observed: Optional[bool] = Field(False, validate_default=True, description='Flag if target has been observed or not', examples=[False])
+    release: Optional[str] = Field(None, description='the Astra release field, either sdss5 or dr17')
+    obs: Optional[str] = Field(None, description='the observatory the observation is from')
+    mjd: Optional[int] = Field(None, description='the MJD of the data reduction')
 
     @field_validator('has_been_observed')
     @classmethod

python/valis/db/queries.py

@@ -60,8 +60,12 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked',
     model = vizdb.SDSSidStacked if table == 'stacked' else vizdb.SDSSidFlat
     qq = query.select_extend(vizdb.SDSSidToPipes.in_boss,
                                vizdb.SDSSidToPipes.in_apogee,
+                               vizdb.SDSSidToPipes.in_bvs,
                                vizdb.SDSSidToPipes.in_astra,
-                               vizdb.SDSSidToPipes.has_been_observed).\
+                               vizdb.SDSSidToPipes.has_been_observed,
+                               vizdb.SDSSidToPipes.release,
+                               vizdb.SDSSidToPipes.obs,
+                               vizdb.SDSSidToPipes.mjd).\
         join(vizdb.SDSSidToPipes, on=(model.sdss_id == vizdb.SDSSidToPipes.sdss_id),
              attr='pipes').distinct(vizdb.SDSSidToPipes.sdss_id)
 
@@ -356,7 +360,7 @@ def get_targets_obs(release: str, obs: str, spectrograph: str) -> peewee.ModelSe
 # 10 - all false
 # 57651832 - my file on disk
 # 57832526 - all true, in both astra snow_white, apogee_net (source_pk=912174,star_pk=2954029)
-
+# 61731453 - in astra, false all else; dr17 release
 
 def get_boss_target(sdss_id: int, release: str, fields: list = None,
                     primary: bool = True) -> peewee.ModelSelect:
@@ -408,33 +412,39 @@ def get_apogee_target(sdss_id: int, release: str, fields: list = None):
     # get the relevant software tag
     apred = get_software_tag(release, 'apred_vers')
 
+    # create apogee version conditions
     if isinstance(apred, list):
         vercond = apo.Star.apred_vers.in_(apred)
+        avsver = astra.ApogeeVisitSpectrum.apred.in_(apred)
     else:
         vercond = apo.Star.apred_vers == apred
+        avsver = astra.ApogeeVisitSpectrum.apred == apred
 
     # check fields
     fields = fields or [apo.Star]
     if fields and isinstance(fields[0], str):
         fields = (getattr(apo.Star, i) for i in fields)
 
+    # get the astra source for the sdss_id
     s = get_astra_target(sdss_id, release)
     if not s:
         return
 
-    a = s.first().apogee_visit_spectrum.first()
+    # get the astra apogee visit spectrum
+    a = s.first().apogee_visit_spectrum.where(avsver).first()
     if not a:
         return
 
+    # get the apogee star data
     return apo.Star.select(*fields).where(apo.Star.pk == a.star_pk, vercond)
 
 
 def get_astra_target(sdss_id: int, release: str, fields: list = None):
     """ temporary placeholder for astra """
 
     vastra = get_software_tag(release, 'v_astra')
-    if not vastra or vastra != "0.5.0":
-        print('astra only supports DR19 / IPL3 = version 0.5.0')
+    if not vastra or vastra not in ("0.5.0", "0.6.0"):
+        print('astra only supports DR19 / IPL3 = version 0.5.0, 0.6.0')
         return None
 
     # check fields
@@ -469,7 +479,7 @@ def get_target_meta(sdss_id: int, release: str) -> dict:
 
 
 def get_pipe_meta(sdss_id: int, release: str, pipeline: str) -> dict:
-    """ Get the pipeline metadata for a pipeline
+    """ Get the pipeline reduction data for a pipeline
 
     Parameters
     ----------
@@ -553,6 +563,12 @@ def get_target_pipeline(sdss_id: int, release: str, pipeline: str = 'all') -> di
         if pipes['in_astra'] and  (res := get_pipe_meta(sdss_id, release, 'astra')):
             deepmerge.always_merger.merge(data, res)
 
+            if pipes['release'] == 'dr17':
+                s = get_astra_target(sdss_id, release)
+                v = s.first().apogee_visit_spectrum.where(astra.ApogeeVisitSpectrum.apred == 'dr17').dicts().first()
+                path = build_apogee_path(v, 'DR17')
+                deepmerge.always_merger.merge(data, {'files': {'apogee': path}})
+
     return data
 
 

python/valis/routes/info.py

@@ -52,7 +52,7 @@ def convert_metadata(data) -> dict:
     """
     mm = defaultdict(dict)
     for i in itertools.chain(data, gen_misc_models()):
-        mm[i['schema']].update({i['column_name']: i})
+        mm[i['schema']].update({f"{i['table_name']}.{i['column_name']}": i})
     return mm
 
 

python/valis/utils/paths.py

@@ -152,6 +152,7 @@ def build_apogee_path(values: dict, release: str, ignore_existence: bool = False
     """
     return build_file_path(values, 'apStar', release,
                            remap={'obj': 'apogee_id', 'apred': 'apred_vers'},
+                           defaults={'apstar': 'stars'},
                            ignore_existence=ignore_existence)