diff --git a/power_systems_data_api_demonstrator/data/example/fuel_source_metadata.csv b/power_systems_data_api_demonstrator/data/example/fuel_source_metadata.csv new file mode 100644 index 0000000..6a99ef0 --- /dev/null +++ b/power_systems_data_api_demonstrator/data/example/fuel_source_metadata.csv @@ -0,0 +1,5 @@ +name,external_reference,external_id +Solar - Photovoltaic - Unspecified,EECS Rules Fact Sheet 5 TYPES OF ENERGY INPUTS AND TECHNOLOGIES,T010100 +Fossil - Solid - Hard Coal - Unspecified,EECS Rules Fact Sheet 5 TYPES OF ENERGY INPUTS AND TECHNOLOGIES,F02010100 +Mechanical source or other - Wind - Unspecified, EECS Rules Fact Sheet 5 TYPES OF ENERGY INPUTS AND TECHNOLOGIES,F01050100 +Thermal - Steam engine - Unspecified,EECS Rules Fact Sheet 5 TYPES OF ENERGY INPUTS AND TECHNOLOGIES,T050900 diff --git a/power_systems_data_api_demonstrator/data/example/psr_metadata.csv b/power_systems_data_api_demonstrator/data/example/psr_metadata.csv new file mode 100644 index 0000000..af6c5cc --- /dev/null +++ b/power_systems_data_api_demonstrator/data/example/psr_metadata.csv @@ -0,0 +1,5 @@ +Grid Node,Topology Level,Unit,Connected Node,Capacity +US-WECC-CISO,1,MW,US-WECC-PACW,800 +US-WECC-CISO,1,MW,US-WECC-BANC,100 +US-WECC-PACW,0,MW,US-WECC-CISO,800 +US-WECC-BANC,1,MW,US-WECC-CISO,100 diff --git a/power_systems_data_api_demonstrator/data/example/psr_metadata.json b/power_systems_data_api_demonstrator/data/example/psr_metadata.json new file mode 100644 index 0000000..c1fa7d0 --- /dev/null +++ b/power_systems_data_api_demonstrator/data/example/psr_metadata.json @@ -0,0 +1,159 @@ +[ + { + "id": "US-WECC-BANC", + "topology_level": 1, + "transmission_capacity": [ + { + "connectedPSR": "US-WECC-CISO", + "unit": "MW", + "value": 700 + } + ], + "fuelSource_capacity": [ + { + "type": "Renewable-Heat-Solar-Unespecified", + "technology": "Solar-Photovoltaic-Classic silicon", + "unit": "MW", + "capacity": [ + { + "value": 500, + "startDatetime": "2017-10-10" + } + ] + }, + { + "type": "Renewable-Mechanical source or other-Wind-Unespecified", + "technology": "Wind-Offshore-Unespecified", + "unit": "MW", + "capacity": [ + { + "value": 500, + "startDatetime": "2016-02-28" + } + ] + } + ] + }, + { + "id": "US-WECC-PACW", + "topology_level": 1, + "transmission_capacity": [ + { + "connectedPSR": "US-WECC-CISO", + "unit": "MW", + "value": 1000 + } + ], + "fuelSource_capacity": [ + { + "type": "Renewable-Mechanical source or other-Wind-Unespecified", + "technology": "Wind-Onshore-Unespecified", + "unit": "MW", + "capacity": [ + { + "value": 1000, + "startDatetime": "2017-01-01" + } + ] + }, + { + "type": "Nuclear-Solid-Radiactive fuel-UOX", + "technology": "Thermal-Steam Turbine with condensation turbine-Unespecified", + "unit": "MW", + "capacity": [ + { + "value": 800, + "startDatetime": "1975-10-15", + "endDatetime": "2016-12-31" + } + ] + } + ] + }, + { + "id": "US-WECC-CISO", + "topology_level": 1, + "transmission_capacity": [ + { + "connectedPSR": "US-WECC-BANC", + "unit": "MW", + "value": 700 + }, + { + "connectedPSR": "US-WECC-PACW", + "unit": "MW", + "value": 1000 + } + ], + "fuelSource_capacity": [ + { + "type": "Renewable-Heat-Solar-Unespecified", + "technology": "Solar-Photovoltaic-Classic silicon", + "unit": "MW", + "capacity": [ + { + "value": 500, + "startDatetime": "2020-10-15" + } + ] + }, + { + "type": "Fossil-Solid-Hard Coal-Antracite", + "technology": "Thermal-Steam Turbine with condensation turbine-Unespecified", + "unit": "MW", + "capacity": [ + { + "value": 1000, + "startDatetime": "1999-12-13" + }, + { + "value": 500, + "startDatetime": "1980-10-15", + "endDatetime": "1999-12-12" + } + ] + } + ] + }, + { + "id": "US-WECC-AZPS", + "topology_level": 2, + "parent": "US-WECC-CISO", + "fuelSource_capacity": [ + { + "type": "Renewable-Heat-Solar-Unespecified", + "technology": "Solar-Photovoltaic-Classic silicon", + "unit": "MW", + "capacity": [ + { + "value": 500, + "startDatetime": "2020-10-15" + } + ] + } + ] + }, + { + "id": "US-WECC-CEN", + "topology_level": 2, + "parent": "US-WECC-CISO", + "fuelSource_capacity": [ + { + "type": "Fossil-Solid-Hard Coal-Antracite", + "technology": "Thermal-Steam Turbine with condensation turbine-Unespecified", + "unit": "MW", + "capacity": [ + { + "value": 1000, + "startDatetime": "1999-12-13" + }, + { + "value": 500, + "startDatetime": "1980-10-15", + "endDatetime": "1999-12-12" + } + ] + } + ] + } +] diff --git a/power_systems_data_api_demonstrator/data/example/topology_metadata.csv b/power_systems_data_api_demonstrator/data/example/topology_metadata.csv new file mode 100644 index 0000000..ff96d9f --- /dev/null +++ b/power_systems_data_api_demonstrator/data/example/topology_metadata.csv @@ -0,0 +1,6 @@ +level,id +0,Interconnection +1,Balancing Area +2,Generating Plant +3,Generating Unit +4,Consumption Unit diff --git a/power_systems_data_api_demonstrator/src/api/metadata/views.py b/power_systems_data_api_demonstrator/src/api/metadata/views.py index b443f45..d95785d 100644 --- a/power_systems_data_api_demonstrator/src/api/metadata/views.py +++ b/power_systems_data_api_demonstrator/src/api/metadata/views.py @@ -1,17 +1,12 @@ -from pydantic import BaseModel -from typing import Sequence -from sqlmodel import select - -from sqlmodel import Session -from enum import Enum -from sqlmodel import Field -from datetime import datetime -from sqlmodel import SQLModel +from typing import Optional, Sequence -from fastapi import APIRouter, Request +import pandas as pd +from fastapi import APIRouter from fastapi.param_functions import Depends -from power_systems_data_api_demonstrator.src.api.db import get_session +from pydantic import BaseModel +from sqlmodel import Field, Session, SQLModel, select +from power_systems_data_api_demonstrator.src.api.db import get_session router = APIRouter() @@ -20,7 +15,9 @@ class TopologyLevel(SQLModel, table=True): id: str = Field(primary_key=True) level: int | None = Field( ge=0, - description="A number representing the hierarchy of this resource topology in relation to the other resource types. These levels **shall** include a sequential set of positive integers starting at 0.", + description="""A number representing the hierarchy of this resource topology in + relation to the other resource types. These levels **shall** include a + sequential set of positive integers starting at 0.""", ) @@ -40,22 +37,87 @@ async def get_topology_levels( return TopologyLevelsResponse(topology_levels=topology_levels) -class FuelTypeDescription(BaseModel): - name: str = Field( - description="A common name to use for the fuel type. If using AIB codes, it should be a concatenation of the three code descriptions with a dash between (i.e. 'Solar - Photovoltaic - Unspecified')." - ) - external_id: str = Field( - description="A unique code (such as the AIB code) referencing the type of fuel." - ) +class FuelSourceType(SQLModel, table=True): + name: str = Field(primary_key=True) + external_reference: Optional[str] + external_id: Optional[str] -class FuelTypesResponse(BaseModel): - external_reference: str = Field( - default="AIB EECS Rule Fact Sheet 5", - description="A reference that provides context for this specific fuel type.", - ) - external_reference_url: str = Field( - default="https://www.aib-net.org/sites/default/files/assets/eecs/facts-sheets/AIB-2019-EECSFS-05%20EECS%20Rules%20Fact%20Sheet%2005%20-%20Types%20of%20Energy%20Inputs%20and%20Technologies%20-%20Release%207.7%20v5.pdf", - description="A unique code (such as the AIB code) referencing the type of fuel.", - ) - types: list[FuelTypeDescription] +class FuelSourceTypesResponse(SQLModel): + types: list[FuelSourceType] + + +@router.get( + "/fuel-source/types", + summary="FUEL SOURCE TYPES", +) +async def get_fuel_source_types( + session: Session = Depends(get_session), +) -> FuelSourceTypesResponse: + result = session.execute(select(FuelSourceType)) + types = result.scalars().all() + return FuelSourceTypesResponse(types=types) + + +class FuelSourceTechnologyReference(SQLModel): + aibCode: str + source_document: str + + +class FuelSourceTechnologyReferenceTable(FuelSourceTechnologyReference, table=True): + name: str = Field(primary_key=True) + + +class FuelSourceTechnology(SQLModel): + name: str + externalReference: FuelSourceTechnologyReference + + +class FuelSourceTechnologyResponse(SQLModel): + technologies: Sequence[FuelSourceTechnology] + + +@router.get( + "/fuel-source/technologies", + summary="FUEL SOURCE TECHNOLOGIES", +) +async def get_fuel_source_technologies( + session: Session = Depends(get_session), +) -> FuelSourceTechnologyResponse: + result = session.execute(select(FuelSourceTechnologyReferenceTable)) + technologies = result.scalars().all() + df = pd.DataFrame(g.dict() for g in technologies) + technologies = [] + for index, row in df.iterrows(): + technologies.append( + # FuelSourceTechnologyReferenceTable( + FuelSourceTechnology( + name=row["name"], + externalReference=FuelSourceTechnologyReference( + aibCode=row["aibCode"], + source_document=row["source_document"], + ), + ) + ) + return FuelSourceTechnologyResponse(technologies=technologies) + + +# class FuelTypeDescription(BaseModel): +# name: str = Field( +# description="A common name to use for the fuel type. If using AIB codes, it should be a concatenation of the three code descriptions with a dash between (i.e. 'Solar - Photovoltaic - Unspecified')."# noqa: E501 +# ) +# external_id: str = Field( +# description="A unique code (such as the AIB code) referencing the type of fuel. # noqa: E501" +# ) + + +# class FuelTypesResponse(BaseModel): +# external_reference: str = Field( +# default="AIB EECS Rule Fact Sheet 5", +# description="A reference that provides context for this specific fuel type.", +# ) +# external_reference_url: str = Field( +# default="https://www.aib-net.org/sites/default/files/assets/eecs/facts-sheets/AIB-2019-EECSFS-05%20EECS%20Rules%20Fact%20Sheet%2005%20-%20Types%20of%20Energy%20Inputs%20and%20Technologies%20-%20Release%207.7%20v5.pdf",# noqa: E501 +# description="A unique code (such as the AIB code) referencing the type of fuel # noqa: E501.", +# ) +# types: list[FuelTypeDescription] diff --git a/power_systems_data_api_demonstrator/src/api/psr_metadata/views.py b/power_systems_data_api_demonstrator/src/api/psr_metadata/views.py index 6b56a92..7bfccef 100644 --- a/power_systems_data_api_demonstrator/src/api/psr_metadata/views.py +++ b/power_systems_data_api_demonstrator/src/api/psr_metadata/views.py @@ -1,30 +1,201 @@ -from pydantic import BaseModel -from sqlmodel import select - -from sqlmodel import Session from enum import Enum -from sqlmodel import Field -from datetime import datetime -from sqlmodel import SQLModel +from typing import Annotated, Optional, Sequence -from fastapi import APIRouter, Request +import pandas as pd +from fastapi import APIRouter, Path, Query from fastapi.param_functions import Depends -from power_systems_data_api_demonstrator.src.api.db import get_session +from sqlmodel import Field, Session, SQLModel, select +from power_systems_data_api_demonstrator.src.api.db import get_session router = APIRouter() -class DescribeResponse(BaseModel): - pass +class PowerUnit(str, Enum): + mw = "MW" + kw = "kW" + w = "W" + + +class PSRList(SQLModel, table=True): + id: str = Field(primary_key=True) + level: int + name: Optional[str] + + +class PSRListResponse(SQLModel): + psr_list: list[PSRList] + + +class PSRListReqest(SQLModel): + level: int @router.get( - "/{id}/describe", - summary="TOPOLOGY DESCRIPTION", + "/power-system-resource", + summary="PSR LIST", ) -async def get_topology_levels( - id: int, +async def get_psr_list( + level: Annotated[Optional[int], Path(description="Filter by level")] | None = None, session: Session = Depends(get_session), -) -> DescribeResponse: - return {} +) -> PSRListResponse: + if level is None: + result = session.execute(select(PSRList)) + else: + result = session.execute(select(PSRList).filter_by(level=level)) + psr_list = result.scalars().all() + return PSRListResponse(psr_list=psr_list) + + +class PSR(SQLModel): + id: str + + +class PSRInterconnection(SQLModel): + # connectedPSR: PSR + connectedPSR: str = Field(primary_key=True) + value: float + + +class PSRInterconnectionTable(PSRInterconnection, table=True): + id: str = Field(primary_key=True) + unit: PowerUnit + + +class PSRCapacity(SQLModel): + id: str = Field(primary_key=True) + unit: PowerUnit + transmissionCapacity: list[PSRInterconnection] + + +class PSRCapacityResponse(SQLModel): + capacity: list[PSRCapacity] + + +class Capacity(SQLModel): + value: float + startDatetime: str = Field(primary_key=True) + endDatetime: Optional[str] = None + + +class CapacityTable(Capacity, table=True): + id: str = Field(primary_key=True) + type: str = Field(primary_key=True) + unit: PowerUnit + technology: str + + +class FuelSource(SQLModel): + technology: str + type: str + unit: PowerUnit + capacity: list[Capacity] + + +class FuelSourceCapacity(SQLModel): + id: str = Field(primary_key=True) + fuelSource: list[FuelSource] + + +class FuelSourceCapacityResponse(SQLModel): + capacity: Sequence[FuelSourceCapacity] + + +@router.get( + "/power-system-resource/capacity", + summary="PSR CAPACITY", +) +async def get_psr_capacity( + id: Annotated[str, Query(description="Filter by PSR")] = "US-WECC-CISO", + session: Session = Depends(get_session), +) -> FuelSourceCapacityResponse: + result = session.execute(select(CapacityTable).filter_by(id=id)) + psr_capacity = result.scalars().all() + df = pd.DataFrame(g.dict() for g in psr_capacity) + print("This is the basic df") + print(df) + print(df.columns) + + # colapse the data frame to a single row per id and unit + df_psr_capacity = ( + df.groupby(["id", "unit", "technology", "type"]) + .agg( + { + "value": lambda x: list(x), + "startDatetime": lambda x: list(x), + "endDatetime": lambda x: list(x), + } + ) + .copy() + .reset_index() + ) + + print("This is after grouby") + print(df_psr_capacity.to_string()) + print(df_psr_capacity.columns) + + # print("this is psr capacity") + # print(df_psr_capacity) + + psr_capacity = [] + psr_capacity.append( + FuelSourceCapacity( + id=df_psr_capacity.loc[0, "id"], + fuelSource=[ + FuelSource( + technology=row["technology"], + type=row["type"], + unit=row["unit"], + capacity=[ + Capacity( + value=row["value"][i], + startDatetime=row["startDatetime"][i], + endDatetime=row["endDatetime"][i], + ) + for i in range(len(row["startDatetime"])) + ], + ) + for index, row in df_psr_capacity.iterrows() + ], + ) + ) + + return FuelSourceCapacityResponse(capacity=psr_capacity) + + +@router.get( + "/power-system-resource/transmission-capacity", + summary="TRANSMISSION CAPACITY", +) +async def get_psr_transmission_capacity( + id: Annotated[str, Query(description="Filter by PSR")] = "US-WECC-CISO", + session: Session = Depends(get_session), +) -> PSRCapacityResponse: + result = session.execute(select(PSRInterconnectionTable).filter_by(id=id)) + psr_capacity = result.scalars().all() + df = pd.DataFrame(g.dict() for g in psr_capacity) + # colapse the data frame to a single row per id and unit + df_psr_interconnections = ( + df.groupby(["id", "unit"]) + .agg({"connectedPSR": lambda x: list(x), "value": lambda x: list(x)}) + .reset_index() + ) + + number_of_interconnections = df_psr_interconnections["connectedPSR"].apply(len)[0] + + psr_capacity = [] + psr_capacity.append( + PSRCapacity( + id=df_psr_interconnections.loc[0, "id"], + unit=df_psr_interconnections.loc[0, "unit"], + transmissionCapacity=[ + PSRInterconnection( + connectedPSR=df_psr_interconnections["connectedPSR"].str[i][0], + value=df_psr_interconnections["value"].str[i][0], + ) + for i in range(number_of_interconnections) + ], + ) + ) + + return PSRCapacityResponse(capacity=psr_capacity) diff --git a/power_systems_data_api_demonstrator/src/api/seed.py b/power_systems_data_api_demonstrator/src/api/seed.py index bbb0153..73f19c5 100644 --- a/power_systems_data_api_demonstrator/src/api/seed.py +++ b/power_systems_data_api_demonstrator/src/api/seed.py @@ -1,37 +1,180 @@ -from datetime import datetime import os + import pandas as pd from sqlmodel import Session -from sqlmodel import SQLModel -from power_systems_data_api_demonstrator.src.api.metadata.views import TopologyLevel + +import power_systems_data_api_demonstrator.data +from power_systems_data_api_demonstrator.src.api.db import init_db +from power_systems_data_api_demonstrator.src.api.metadata.views import ( + FuelSourceTechnologyReferenceTable, + FuelSourceType, + TopologyLevel, +) +from power_systems_data_api_demonstrator.src.api.psr_metadata.views import ( + CapacityTable, + PSRInterconnectionTable, + PSRList, +) from power_systems_data_api_demonstrator.src.api.psr_timeseries.views import ( - GenerationByFuelSourceTable, FuelType, - FuelTechnology, + GenerationByFuelSourceTable, ) -from power_systems_data_api_demonstrator.src.api.db import init_db -import power_systems_data_api_demonstrator.data DATA_DIR = os.path.dirname(power_systems_data_api_demonstrator.data.__file__) def seed() -> None: engine = init_db() - topology_levels = [ - TopologyLevel(id="Level 1", level=1), - TopologyLevel(id="Level 2", level=2), - ] + + topology_levels = [] + + for grid_source in ["example"]: + df = pd.read_csv(os.path.join(DATA_DIR, grid_source, "topology_metadata.csv")) + + levels = df + + for index, row in levels.iterrows(): + topology_levels.extend([TopologyLevel(id=row["id"], level=row["level"])]) with Session(engine) as session: seed_generation(session) session.add_all(topology_levels) + seed_fuelsource(session) + seed_psr(session) session.commit() +def seed_fuelsource(session: Session) -> None: + for grid_source in ["example"]: + df = pd.read_csv( + os.path.join(DATA_DIR, grid_source, "fuel_source_metadata.csv") + ) + + fuel_source_types = [] + fuel_source_technologies = [] + + types = df[df["external_id"].str.contains("F")].reset_index() + technologies = df[~df["external_id"].str.contains("F")].reset_index() + for index, row in types.iterrows(): + fuel_source_types.extend( + [ + FuelSourceType( + name=row["name"], + external_id=row["external_id"], + external_reference=row["external_reference"], + ) + ] + ) + + for index, row in technologies.iterrows(): + fuel_source_technologies.extend( + [ + FuelSourceTechnologyReferenceTable( + name=row["name"], + aibCode=row["external_id"], + source_document=row["external_reference"], + ) + ] + ) + + # session.add_all(fuelsource_types) + # session.add_all(fuelsource_technologies) + session.add_all(fuel_source_types) + session.add_all(fuel_source_technologies) + session.commit() + + +def extract_info(df: pd.DataFrame, column_name: str): + """ + Extracts the information from a column with a list of dicts and + adds them as columns to the dataframe. + + Input: + ---------- + df : pd.DataFrame + column_name : str + + Returns + ------- + df : pd.DataFrame + """ + df = df.explode(column_name).reset_index(drop=True) + extracted_info = df[column_name].apply(pd.Series).reset_index(drop=True) + df = df.join(extracted_info) + df = df.drop(column_name, axis=1) + + return df + + +def seed_psr(session: Session) -> None: + psr_data = [] + generation_capacity = [] + transmission_capacity = [] + + for grid_source in ["example"]: + df = pd.read_json(os.path.join(DATA_DIR, grid_source, "psr_metadata.json")) + # PSR List + psr_data = [] + + for index, row in df.iterrows(): + psr_data.extend([PSRList(id=row["id"], level=row["topology_level"])]) + + # PSR GENERATION CAPACITY + + generation_capacity = [] + # select only the rows with generation capacity data + df_generation = df.dropna(subset=["fuelSource_capacity"]).copy() + df_generation = df_generation[["id", "fuelSource_capacity"]] + # extract fuelSource_capacity info + df_generation = extract_info(df_generation, "fuelSource_capacity") + df_generation = extract_info(df_generation, "capacity") + + for index, row in df_generation.iterrows(): + print("datetime") + print(pd.to_datetime(row["startDatetime"])) + capacity_table = CapacityTable( + id=row["id"], + unit=row["unit"], + type=row["type"], + technology=row["technology"], + value=row["value"], + startDatetime=row["startDatetime"], + ) + + if not pd.isna(row["endDatetime"]): + capacity_table.endDatetime = row["endDatetime"] + + generation_capacity.extend([capacity_table]) + + # PSR TRANSMISSION CAPACITY + transmission_capacity = [] + # select only the rows with transmission capacity data + df_transmission = df.dropna(subset=["transmission_capacity"]).copy() + # extract transmission_capacity info + df_transmission = extract_info(df_transmission, "transmission_capacity") + + for index, row in df_transmission.iterrows(): + transmission_capacity.extend( + [ + PSRInterconnectionTable( + id=row["id"], + unit=row["unit"], + connectedPSR=row["connectedPSR"], + value=row["value"], + ) + ] + ) + + session.add_all(generation_capacity) + session.add_all(psr_data) + session.add_all(transmission_capacity) + session.commit() + + def seed_generation(session: Session) -> None: # Overall generation fuel_types = [] - fuel_technologies = [] + # fuel_technologies = [] generation_interconnection_by_fuel_source = [] generation_balancing_area_by_fuel_source = []