Merge pull request #405 from GEMScienceTools/global_parser

removing deprecated columns from global flatfile parser
GEMScienceTools · Apr 4, 2024 · 4cbfcf0 · 4cbfcf0
2 parents 9d28ed5 + 9c59c5e
commit 4cbfcf0
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 32 deletions.
diff --git a/openquake/smt/parsers/esm_flatfile_parser.py b/openquake/smt/parsers/esm_flatfile_parser.py
@@ -166,7 +166,6 @@ def _parse_record(self, metadata):
                                   xcomp, ycomp,
                                   vertical=vertical)
 
-
     def _parse_event_data(self, metadata):
         """
         Parses the event metadata

diff --git a/openquake/smt/parsers/esm_url_flatfile_parser.py b/openquake/smt/parsers/esm_url_flatfile_parser.py
@@ -478,7 +478,6 @@ def _parse_ground_motion(self, location, row, record, headers):
         record.datafile = filename
         return record
 
-
     def _retreive_ground_motion_from_row(self, row, header_list):
         """
         Get the ground-motion data from a row (record) in the database

diff --git a/openquake/smt/parsers/esm_ws_flatfile_parser.py b/openquake/smt/parsers/esm_ws_flatfile_parser.py
@@ -159,7 +159,6 @@ def _parse_record(self, metadata):
                                   xcomp, ycomp,
                                   vertical=vertical)
 
-
     def _parse_event_data(self, metadata):
         """
         Parses the event metadata
@@ -473,7 +472,6 @@ def _parse_ground_motion(self, location, row, record, headers):
         record.datafile = filename
         return record
 
-
     def _retreive_ground_motion_from_row(self, row, header_list):
         """
         Get the ground-motion data from a row (record) in the database

diff --git a/openquake/smt/parsers/gem_flatfile_parser.py b/openquake/smt/parsers/gem_flatfile_parser.py
@@ -50,14 +50,13 @@
 
 SCALAR_LIST = ["PGA", "PGV", "PGD", "CAV", "CAV5", "Ia", "D5-95"]
 
-HEADER_STR = "event_id;event_time;ISC_ev_id;USGS_ev_id;INGV_ev_id;"\
-             "EMSC_ev_id;ev_nation_code;ev_latitude;ev_longitude;"\
+HEADER_STR = "event_id;event_time;ISC_ev_id;ev_latitude;ev_longitude;"\
              "ev_depth_km;fm_type_code;ML;Mw;Ms;event_source_id;"\
-             "es_strike;es_dip;es_rake;es_strike_dip_rake_ref;es_z_top;"\
-             "es_length;es_width;network_code;station_code;location_code;"\
-             "instrument_code;sensor_depth_m;housing_code;installation_code;"\
-             "st_nation_code;st_latitude;st_longitude;st_elevation;vs30_m_sec;"\
-             "slope_deg;vs30_meas_type;epi_dist;epi_az;JB_dist;rup_dist;Rx_dist;"\
+             "es_strike;es_dip;es_rake;es_z_top;es_length;es_width;"\
+             "network_code;station_code;location_code;"\
+             "sensor_depth_m;"\
+             "st_latitude;st_longitude;st_elevation;vs30_m_sec;slope_deg;"\
+             "vs30_meas_type;epi_dist;epi_az;JB_dist;rup_dist;Rx_dist;"\
              "Ry0_dist;late_triggered_flag_01;U_channel_code;U_azimuth_deg;"\
              "V_channel_code;V_azimuth_deg;W_channel_code;U_hp;V_hp;W_hp;U_lp;"\
              "V_lp;W_lp"
@@ -168,7 +167,6 @@ def _parse_event_data(self, metadata):
         eq_id = metadata["event_id"]
         eq_name = metadata["event_id"]
         # Country
-        cntry_code = metadata["ev_nation_code"].strip()
         # Date and time
         eq_datetime = valid.date_time(metadata["event_time"],
                                      "%Y-%m-%d %H:%M:%S")
@@ -337,13 +335,9 @@ def _parse_site_data(self, metadata):
         site.slope = valid.vfloat(metadata["slope_deg"], "slope_deg")
         site.sensor_depth = valid.vfloat(metadata["sensor_depth_m"],
                                          "sensor_depth_m")
-        site.instrument_type = metadata["instrument_code"].strip()
         if site.vs30:
             site.z1pt0 = vs30_to_z1pt0_cy14(vs30)
             site.z2pt5 = vs30_to_z2pt5_cb14(vs30)
-        housing_code = metadata["housing_code"].strip()
-        if housing_code and (housing_code in HOUSING):
-            site.building_structure = HOUSING[housing_code]
         return site
 
     def _parse_waveform_data(self, metadata, wfid):
@@ -353,15 +347,14 @@ def _parse_waveform_data(self, metadata, wfid):
         late_trigger = valid.vint(metadata["late_triggered_flag_01"],
                                   "late_triggered_flag_01")
         # U channel - usually east
-        xorientation = metadata["U_channel_code"].strip()
         xazimuth = valid.vfloat(metadata["U_azimuth_deg"], "U_azimuth_deg")
         xfilter = {"Low-Cut": valid.vfloat(metadata["U_hp"], "U_hp"),
                    "High-Cut": valid.vfloat(metadata["U_lp"], "U_lp")}
         xcomp = Component(wfid, xazimuth, waveform_filter=xfilter,
                           units="cm/s/s")
         xcomp.late_trigger = late_trigger
+
         # V channel - usually North
-        vorientation = metadata["V_channel_code"].strip()
         vazimuth = valid.vfloat(metadata["V_azimuth_deg"], "V_azimuth_deg")
         vfilter = {"Low-Cut": valid.vfloat(metadata["V_hp"], "V_hp"),
                    "High-Cut": valid.vfloat(metadata["V_lp"], "V_lp")}
@@ -523,10 +516,9 @@ def _retreive_ground_motion_from_row(self, row, header_list):
 
 def _prioritise_rotd50(df, proxy=None, removal=None):
     """
-    Assign RotD50 values to accelerations for computation of residuals. RotD50
-    is available for the vast majority of the records in the GEM flatfile for
-    PGA to 10 s (KikNet subset of records is limited to up to 5 s because we
-    use Beyes and Bommer 2006 to convert from horizontals to RotD50).
+    Assign RotD50 values to horizontal accelerations for computation of
+    residuals. RotD50 is available for the vast majority of the records in the
+    GEM flatfile for PGA to 10 s.
     
     If no RotD50 use the geometric mean if available (if specified) as a proxy
     for RotD50.