From ba8867a9b873fa37b89d842cdfa4ee5b1d7e94da Mon Sep 17 00:00:00 2001 From: David Wang Date: Fri, 1 Apr 2022 16:36:13 -0700 Subject: [PATCH] pull in ND data if last_obs > 30 days --- alert_stream_crossmatch/db_caching.py | 13 +++++++++++-- alert_stream_crossmatch/simple_crossmatch.py | 6 +++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/alert_stream_crossmatch/db_caching.py b/alert_stream_crossmatch/db_caching.py index 0591f8b..ad2538f 100644 --- a/alert_stream_crossmatch/db_caching.py +++ b/alert_stream_crossmatch/db_caching.py @@ -128,12 +128,21 @@ def get_cached_ids(conn, condition=None): cur.close() return ids +def last_obs_gt_30(conn, ztf_object_id, jd, thres=30): + """Return true if last obs of ztf_object_id was more than 30 days ago + """ + cur = conn.cursor() + cur.execute("SELECT last_obs FROM ZTF_objects WHERE ZTF_object_id=?", (ztf_object_id,)) + last_obs = cur.fetchone()[0] + if pd.isna(last_obs): + return True + return (jd - last_obs) > thres def clear_ZTF_table(conn): """Delete all rows in ZTF_objects """ cur = conn.cursor() - cur.execute("DELETE FROM ZTF_objects") + cur.execute("DELETE FROM ZTF_objects") cur.close() @@ -185,7 +194,7 @@ def main(): magnr float, sigmagnr float, field int, - rcid int + rcid int );""" # create a database connection diff --git a/alert_stream_crossmatch/simple_crossmatch.py b/alert_stream_crossmatch/simple_crossmatch.py index 645e3e4..6c834b6 100644 --- a/alert_stream_crossmatch/simple_crossmatch.py +++ b/alert_stream_crossmatch/simple_crossmatch.py @@ -260,7 +260,11 @@ def save_to_db(packet, otype, sources_saved, database, interest): conn = create_connection(database) if ztf_object_id in sources_saved: logging.info(f"{ztf_object_id} already saved in time between simbad check and now") - dflc = make_dataframe(packet, repeat_obs=True) + if last_obs_gt_30(conn, ztf_object_id, packet['candidate']['jd']): # pull in ND data + dflc = make_dataframe(packet, repeat_obs=False) + else: + dflc = make_dataframe(packet, repeat_obs=True) + else: update_value(conn, data_to_update, f'ZTF_object_id = "{ztf_object_id}"') dflc = make_dataframe(packet, repeat_obs=False)