Skip to content

Commit

Permalink
Fix bias adjustment calcs for GDPS and GFS (bcgov#3308)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dgboss
dgboss authored Jan 9, 2024
1 parent b7fafaa commit 703caa7
Showing 1 changed file with 88 additions and 27 deletions.
115 changes: 88 additions & 27 deletions api/app/jobs/common_model_fetchers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
refresh_morecast2_materialized_view,
delete_model_run_predictions)
from app.weather_models.machine_learning import StationMachineLearning
from app.weather_models import SCALAR_MODEL_VALUE_KEYS, ModelEnum, construct_interpolated_noon_prediction
from app.weather_models import SCALAR_MODEL_VALUE_KEYS, ModelEnum, construct_interpolated_noon_prediction, interpolate_between_two_points
from app.schemas.stations import WeatherStation
from app import config, configure_logging
import app.utils.time as time_utils
Expand Down Expand Up @@ -207,11 +207,86 @@ def _process_model_run(self, model_run: PredictionModelRunTimestamp, model_type:
self.session.commit()
logger.info('done commit.')

def _add_interpolated_bias_adjustments_to_prediction(self, station_prediction: WeatherStationModelPrediction, machine: StationMachineLearning):
# We need to interpolate prediction for 2000 using predictions for 1800 and 2100
# Predict the temperature
temp_at_1800 = machine.predict_temperature(station_prediction.tmp_tgl_2,
station_prediction.prediction_timestamp.replace(hour=18))
temp_at_2100 = machine.predict_temperature(station_prediction.tmp_tgl_2,
station_prediction.prediction_timestamp.replace(hour=21))
station_prediction.bias_adjusted_temperature = interpolate_between_two_points(18, 21, temp_at_1800,
temp_at_2100, 20)
# Predict the rh
rh_at_1800 = machine.predict_rh(station_prediction.rh_tgl_2,
station_prediction.prediction_timestamp.replace(hour=18))
rh_at_2100 = machine.predict_rh(station_prediction.rh_tgl_2,
station_prediction.prediction_timestamp.replace(hour=21))
station_prediction.bias_adjusted_rh = interpolate_between_two_points(18, 21, rh_at_1800, rh_at_2100, 20)

# Predict the wind speed
wind_speed_at_1800 = machine.predict_wind_speed(station_prediction.wind_tgl_10,
station_prediction.prediction_timestamp.replace(hour=18))
wind_speed_at_2100 = machine.predict_wind_speed(station_prediction.wind_tgl_10,
station_prediction.prediction_timestamp.replace(hour=21))
station_prediction.bias_adjusted_wind_speed = interpolate_between_two_points(18, 21, wind_speed_at_1800,
wind_speed_at_2100, 20)

# Predict the wind direction
wind_direction_at_1800 = station_prediction.bias_adjusted_wdir = machine.predict_wind_direction(
station_prediction.wind_tgl_10,
station_prediction.wdir_tgl_10,
station_prediction.prediction_timestamp.replace(hour=18)
)
wind_direction_at_2100 = station_prediction.bias_adjusted_wdir = machine.predict_wind_direction(
station_prediction.wind_tgl_10,
station_prediction.wdir_tgl_10,
station_prediction.prediction_timestamp.replace(hour=21)
)
station_prediction.bias_adjusted_wdir = interpolate_between_two_points(18, 21, wind_direction_at_1800,
wind_direction_at_2100, 20)

# Predict the 24h precipitation. No interpolation necessary due to the underlying model training.
station_prediction.bias_adjusted_precip_24h = machine.predict_precipitation(
station_prediction.precip_24h,
station_prediction.prediction_timestamp
)

def _add_bias_adjustments_to_prediction(self, station_prediction: WeatherStationModelPrediction,
machine: StationMachineLearning):
# Predict the temperature
station_prediction.bias_adjusted_temperature = machine.predict_temperature(
station_prediction.tmp_tgl_2,
station_prediction.prediction_timestamp)

# Predict the rh
station_prediction.bias_adjusted_rh = machine.predict_rh(station_prediction.rh_tgl_2,
station_prediction.prediction_timestamp)

# Predict the wind speed
station_prediction.bias_adjusted_wind_speed = machine.predict_wind_speed(
station_prediction.wind_tgl_10,
station_prediction.prediction_timestamp
)

# Predict the wind direction
station_prediction.bias_adjusted_wdir = machine.predict_wind_direction(
station_prediction.wind_tgl_10,
station_prediction.wdir_tgl_10,
station_prediction.prediction_timestamp
)

# Predict the 24h precipitation
station_prediction.bias_adjusted_precip_24h = machine.predict_precipitation(
station_prediction.precip_24h,
station_prediction.prediction_timestamp
)

def _process_prediction(self,
prediction: ModelRunPrediction,
station: WeatherStation,
model_run: PredictionModelRunTimestamp,
machine: StationMachineLearning):
machine: StationMachineLearning,
prediction_is_interpolated: bool):
""" Create a WeatherStationModelPrediction from the ModelRunPrediction data.
"""
# If there's already a prediction, we want to update it
Expand Down Expand Up @@ -262,29 +337,14 @@ def _process_prediction(self,
if prediction.wdir_tgl_10 is not None:
station_prediction.wdir_tgl_10 = prediction.wdir_tgl_10

# Predict the temperature
station_prediction.bias_adjusted_temperature = machine.predict_temperature(
station_prediction.tmp_tgl_2,
station_prediction.prediction_timestamp)
# Predict the rh
station_prediction.bias_adjusted_rh = machine.predict_rh(
station_prediction.rh_tgl_2, station_prediction.prediction_timestamp)
# Predict the wind speed
station_prediction.bias_adjusted_wind_speed = machine.predict_wind_speed(
station_prediction.wind_tgl_10,
station_prediction.prediction_timestamp
)
# Predict the wind direction
station_prediction.bias_adjusted_wdir = machine.predict_wind_direction(
station_prediction.wind_tgl_10,
station_prediction.wdir_tgl_10,
station_prediction.prediction_timestamp
)
# Predict the 24 hour precipitation
station_prediction.bias_adjusted_precip_24h = machine.predict_precipitation(
station_prediction.precip_24h,
station_prediction.prediction_timestamp
)
if prediction_is_interpolated:
# Dealing with a numerical weather model that only has predictions at 3 hour intervals,
# so no 20:00 UTC prediction available in the trained linear regression
self._add_interpolated_bias_adjustments_to_prediction(station_prediction, machine)

else:
# No interpolation required
self._add_bias_adjustments_to_prediction(station_prediction, machine)

# Update the update time (this might be an update)
station_prediction.update_date = time_utils.get_utc_now()
Expand Down Expand Up @@ -373,9 +433,10 @@ def _process_model_run_for_station(self,
noon_prediction = construct_interpolated_noon_prediction(
prev_prediction, prediction, SCALAR_MODEL_VALUE_KEYS)
self._process_prediction(
noon_prediction, station, model_run, machine)
noon_prediction, station, model_run, machine, True)
self._process_prediction(
prediction, station, model_run, machine)
prediction, station, model_run, machine, False)

prev_prediction = prediction

def _mark_model_run_interpolated(self, model_run: PredictionModelRunTimestamp):
Expand Down

0 comments on commit 703caa7

Please sign in to comment.