Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Task/use backend morecast token #3824

Merged
merged 7 commits into from
Aug 7, 2024
Merged

Task/use backend morecast token #3824

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
daf5b73
Backend changes to use API token
conbrad Aug 7, 2024
b3b1ff7
Remove frontend cruft
conbrad Aug 7, 2024
c17da1c
Maintain original forecast creator
conbrad Aug 7, 2024
fb85f0c
Fix test
conbrad Aug 7, 2024
f69c923
more cleanup
conbrad Aug 7, 2024
e370fba
Fix test
conbrad Aug 7, 2024
8fad4ca
Fix test
conbrad Aug 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .vscode/settings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,28 +55,36 @@
"typescript.preferences.importModuleSpecifier": "non-relative",
"cSpell.words": [
"aiobotocore",
"aiohttp",
"Albers",
"allclose",
"anyio",
"APCP",
"Behaviour",
"botocore",
"cffdrs",
"determinates",
"excinfo",
"fastapi",
"fireweather",
"firezone",
"GDPS",
"GEOGCS",
"geospatial",
"geotiff",
"grib",
"gribs",
"HAINES",
"hourlies",
"HRDPS",
"luxon",
"maxx",
"maxy",
"miny",
"morecast",
"ndarray",
"numba",
"ORJSON",
"osgeo",
"PMTILES",
"polygonize",
Expand All @@ -96,6 +104,7 @@
"vsimem",
"vsis",
"WDIR",
"wfwx",
"Whitebox"
],
"autoDocstring.docstringFormat": "sphinx-notypes",
Expand Down
124 changes: 55 additions & 69 deletions api/app/morecast_v2/forecasts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

from datetime import datetime, time, timedelta, timezone
from urllib.parse import urljoin
from app import config
Expand All @@ -22,38 +21,45 @@ def get_forecasts(db_session: Session, start_time: Optional[datetime], end_time:

result = get_forecasts_in_range(db_session, start_time, end_time, station_codes)

forecasts: List[MoreCastForecastOutput] = [MoreCastForecastOutput(station_code=forecast.station_code,
for_date=forecast.for_date.timestamp() * 1000,
temp=forecast.temp,
rh=forecast.rh,
precip=forecast.precip,
wind_speed=forecast.wind_speed,
wind_direction=forecast.wind_direction,
grass_curing=forecast.grass_curing,
update_timestamp=int(forecast.update_timestamp.timestamp())) for forecast in result]
forecasts: List[MoreCastForecastOutput] = [
MoreCastForecastOutput(
station_code=forecast.station_code,
for_date=forecast.for_date.timestamp() * 1000,
temp=forecast.temp,
rh=forecast.rh,
precip=forecast.precip,
wind_speed=forecast.wind_speed,
wind_direction=forecast.wind_direction,
grass_curing=forecast.grass_curing,
update_timestamp=int(forecast.update_timestamp.timestamp()),
)
for forecast in result
]
return forecasts


def construct_wf1_forecast(forecast: MoreCastForecastInput, stations: List[WFWXWeatherStation], forecast_id: Optional[str], created_by: Optional[str]) -> WF1PostForecast:
station = next(filter(lambda obj: obj.code == forecast.station_code, stations))
station_id = station.wfwx_id
station_url = urljoin(config.get('WFWX_BASE_URL'), f'wfwx-fireweather-api/v1/stations/{station_id}')
wf1_post_forecast = WF1PostForecast(createdBy=created_by,
id=forecast_id,
stationId=station_id,
station=station_url,
temperature=forecast.temp,
relativeHumidity=forecast.rh,
precipitation=forecast.precip,
windSpeed=forecast.wind_speed,
windDirection=forecast.wind_direction,
weatherTimestamp=forecast.for_date,
recordType=WF1ForecastRecordType(),
grasslandCuring=forecast.grass_curing)
station_url = urljoin(config.get("WFWX_BASE_URL"), f"wfwx-fireweather-api/v1/stations/{station_id}")
wf1_post_forecast = WF1PostForecast(
createdBy=created_by,
id=forecast_id,
stationId=station_id,
station=station_url,
temperature=forecast.temp,
relativeHumidity=forecast.rh,
precipitation=forecast.precip,
windSpeed=forecast.wind_speed,
windDirection=forecast.wind_direction,
weatherTimestamp=forecast.for_date,
recordType=WF1ForecastRecordType(),
grasslandCuring=forecast.grass_curing,
)
return wf1_post_forecast


async def construct_wf1_forecasts(session: ClientSession, forecast_records: List[MoreCastForecastInput], stations: List[WFWXWeatherStation]) -> List[WF1PostForecast]:
async def construct_wf1_forecasts(session: ClientSession, forecast_records: List[MoreCastForecastInput], stations: List[WFWXWeatherStation], username: str) -> List[WF1PostForecast]:
# Fetch existing forecasts from WF1 for the stations and date range in the forecast records
header = await get_auth_header(session)
forecast_dates = [datetime.fromtimestamp(f.for_date / 1000, timezone.utc) for f in forecast_records]
Expand All @@ -62,8 +68,7 @@ async def construct_wf1_forecasts(session: ClientSession, forecast_records: List
start_time = vancouver_tz.localize(datetime.combine(min_forecast_date, time.min))
end_time = vancouver_tz.localize(datetime.combine(max_forecast_date, time.max))
unique_station_codes = list(set([f.station_code for f in forecast_records]))
dailies = await get_forecasts_for_stations_by_date_range(session, header, start_time,
end_time, unique_station_codes)
dailies = await get_forecasts_for_stations_by_date_range(session, header, start_time, end_time, unique_station_codes)

# Shape the WF1 dailies into a dictionary keyed by station codes for easier consumption
grouped_dailies = defaultdict(list[StationDailyFromWF1])
Expand All @@ -75,36 +80,33 @@ async def construct_wf1_forecasts(session: ClientSession, forecast_records: List
for forecast in forecast_records:
forecast_timestamp = datetime.fromtimestamp(forecast.for_date / 1000, timezone.utc)
# Check if an existing daily was retrieved from WF1 and use id and createdBy attributes if present
observed_daily = next(
(daily for daily in grouped_dailies[forecast.station_code] if daily.utcTimestamp == forecast_timestamp), None)
observed_daily = next((daily for daily in grouped_dailies[forecast.station_code] if daily.utcTimestamp == forecast_timestamp), None)
forecast_id = observed_daily.forecast_id if observed_daily is not None else None
created_by = observed_daily.created_by if observed_daily is not None else None
created_by = username
wf1_forecasts.append(construct_wf1_forecast(forecast, stations, forecast_id, created_by))
conbrad marked this conversation as resolved.
Show resolved Hide resolved
return wf1_forecasts


async def format_as_wf1_post_forecasts(session: ClientSession, forecast_records: List[MoreCastForecastInput]) -> List[WF1PostForecast]:
""" Returns list of forecast records re-formatted in the data structure WF1 API expects """
async def format_as_wf1_post_forecasts(session: ClientSession, forecast_records: List[MoreCastForecastInput], username: str) -> List[WF1PostForecast]:
"""Returns list of forecast records re-formatted in the data structure WF1 API expects"""
header = await get_auth_header(session)
station_codes = [record.station_code for record in forecast_records]
stations = await get_wfwx_stations_from_station_codes(session, header, station_codes)
unique_stations = list(set(stations))
wf1_post_forecasts = await construct_wf1_forecasts(session, forecast_records, unique_stations)
wf1_post_forecasts = await construct_wf1_forecasts(session, forecast_records, unique_stations, username)
return wf1_post_forecasts


def actual_exists(forecast: WeatherIndeterminate, actuals: List[WeatherIndeterminate]):
""" Returns True if the actuals contain a WeatherIndeterminate with station_code and utc_timestamp that
"""Returns True if the actuals contain a WeatherIndeterminate with station_code and utc_timestamp that
matches those of the forecast; otherwise, returns False."""
station_code_matches = [actual for actual in actuals if actual.station_code ==
forecast.station_code]
utc_timestamp_matches = [station_code_match for station_code_match in station_code_matches
if station_code_match.utc_timestamp == forecast.utc_timestamp]
station_code_matches = [actual for actual in actuals if actual.station_code == forecast.station_code]
utc_timestamp_matches = [station_code_match for station_code_match in station_code_matches if station_code_match.utc_timestamp == forecast.utc_timestamp]
return len(utc_timestamp_matches) > 0


def filter_for_api_forecasts(forecasts: List[WeatherIndeterminate], actuals: List[WeatherIndeterminate]):
""" Returns a list of forecasts where each forecast has a corresponding WeatherIndeterminate in the
"""Returns a list of forecasts where each forecast has a corresponding WeatherIndeterminate in the
actuals with a matching station_code and utc_timestamp."""
filtered_forecasts = []
for forecast in forecasts:
Expand Down Expand Up @@ -132,23 +134,20 @@ def get_fwi_values(actuals: List[WeatherIndeterminate], forecasts: List[WeatherI
indeterminates_dict[indeterminate.station_code][indeterminate.utc_timestamp.date()] = indeterminate

for idx, indeterminate in enumerate(all_indeterminates):
last_indeterminate = indeterminates_dict[indeterminate.station_code].get(
indeterminate.utc_timestamp.date() - timedelta(days=1), None)
last_indeterminate = indeterminates_dict[indeterminate.station_code].get(indeterminate.utc_timestamp.date() - timedelta(days=1), None)
if last_indeterminate is not None:
updated_forecast = calculate_fwi_values(last_indeterminate, indeterminate)
all_indeterminates[idx] = updated_forecast

updated_forecasts = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate ==
WeatherDeterminate.FORECAST]
updated_actuals = [
indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL]
updated_forecasts = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == WeatherDeterminate.FORECAST]
updated_actuals = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL]

return updated_actuals, updated_forecasts


def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterminate) -> WeatherIndeterminate:
"""
Uses CFFDRS library to calculate Fire Weather Index System values
Uses CFFDRS library to calculate Fire Weather Index System values

:param yesterday: The WeatherIndeterminate from the day before the date to calculate
:type yesterday: WeatherIndeterminate
Expand All @@ -159,40 +158,27 @@ def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterm
"""

# weather params for calculation date
month_to_calculate_for = int(today.utc_timestamp.strftime('%m'))
month_to_calculate_for = int(today.utc_timestamp.strftime("%m"))
latitude = today.latitude
temp = today.temperature
rh = today.relative_humidity
precip = today.precipitation
wind_spd = today.wind_speed

if yesterday.fine_fuel_moisture_code is not None:
today.fine_fuel_moisture_code = cffdrs.fine_fuel_moisture_code(ffmc=yesterday.fine_fuel_moisture_code,
temperature=temp,
relative_humidity=rh,
precipitation=precip,
wind_speed=wind_spd)
today.fine_fuel_moisture_code = cffdrs.fine_fuel_moisture_code(
ffmc=yesterday.fine_fuel_moisture_code, temperature=temp, relative_humidity=rh, precipitation=precip, wind_speed=wind_spd
)
if yesterday.duff_moisture_code is not None:
today.duff_moisture_code = cffdrs.duff_moisture_code(dmc=yesterday.duff_moisture_code,
temperature=temp,
relative_humidity=rh,
precipitation=precip,
latitude=latitude,
month=month_to_calculate_for,
latitude_adjust=True
)
today.duff_moisture_code = cffdrs.duff_moisture_code(
dmc=yesterday.duff_moisture_code, temperature=temp, relative_humidity=rh, precipitation=precip, latitude=latitude, month=month_to_calculate_for, latitude_adjust=True
)
if yesterday.drought_code is not None:
today.drought_code = cffdrs.drought_code(dc=yesterday.drought_code,
temperature=temp,
relative_humidity=rh,
precipitation=precip,
latitude=latitude,
month=month_to_calculate_for,
latitude_adjust=True
)
today.drought_code = cffdrs.drought_code(
dc=yesterday.drought_code, temperature=temp, relative_humidity=rh, precipitation=precip, latitude=latitude, month=month_to_calculate_for, latitude_adjust=True
)
if today.fine_fuel_moisture_code is not None:
today.initial_spread_index = cffdrs.initial_spread_index(ffmc=today.fine_fuel_moisture_code,
wind_speed=today.wind_speed)
today.initial_spread_index = cffdrs.initial_spread_index(ffmc=today.fine_fuel_moisture_code, wind_speed=today.wind_speed)
if today.duff_moisture_code is not None and today.drought_code is not None:
today.build_up_index = cffdrs.bui_calc(dmc=today.duff_moisture_code, dc=today.drought_code)
if today.initial_spread_index is not None and today.build_up_index is not None:
Expand Down
Loading
Loading