From 745be17d9d8dc15b926116de64c0d43be7c7c2d7 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 10:22:22 +0200 Subject: [PATCH 01/26] chore: Moved osm_utils --- ra2ce/graph/{ => osm_network_wrapper}/osm_utils.py | 0 tests/graph/{ => osm_network_wrapper}/test_osm_utils.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename ra2ce/graph/{ => osm_network_wrapper}/osm_utils.py (100%) rename tests/graph/{ => osm_network_wrapper}/test_osm_utils.py (96%) diff --git a/ra2ce/graph/osm_utils.py b/ra2ce/graph/osm_network_wrapper/osm_utils.py similarity index 100% rename from ra2ce/graph/osm_utils.py rename to ra2ce/graph/osm_network_wrapper/osm_utils.py diff --git a/tests/graph/test_osm_utils.py b/tests/graph/osm_network_wrapper/test_osm_utils.py similarity index 96% rename from tests/graph/test_osm_utils.py rename to tests/graph/osm_network_wrapper/test_osm_utils.py index 67303be27..e4963b6d6 100644 --- a/tests/graph/test_osm_utils.py +++ b/tests/graph/osm_network_wrapper/test_osm_utils.py @@ -2,7 +2,7 @@ import pytest -from ra2ce.graph.osm_utils import from_shapefile_to_poly +from ra2ce.graph.osm_network_wrapper.osm_utils import from_shapefile_to_poly from tests import test_data, test_results From e0c3d07164a9010285f72a3fe3bc3134001252a9 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 10:35:05 +0200 Subject: [PATCH 02/26] chore: Moved vector newtork wrapper for further rework --- ra2ce/graph/networks.py | 17 ++++++++++------- ra2ce/graph/shp_network_wrapper/__init__.py | 0 .../vector_network_wrapper.py | 0 tests/graph/test_vector_network_wrapper.py | 4 +--- tests/test_acceptance.py | 3 +-- 5 files changed, 12 insertions(+), 12 deletions(-) create mode 100644 ra2ce/graph/shp_network_wrapper/__init__.py rename ra2ce/graph/{ => shp_network_wrapper}/vector_network_wrapper.py (100%) diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index e97dfc5b7..d69e0cea5 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -39,7 +39,7 @@ from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper from ra2ce.graph.segmentation import Segmentation -from ra2ce.graph.vector_network_wrapper import VectorNetworkWrapper +from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper class Network: @@ -241,6 +241,14 @@ def network_cleanshp(self) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: return graph_complex, edges_complex + def _create_network_from_shp( + self, + ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + logging.info("Start creating a network from the submitted shapefile.") + if self._any_cleanup_enabled(): + return self.network_shp() + return self.network_cleanshp() + def _export_linking_tables(self, linking_tables: List[Any]) -> None: _exporter = JsonExporter() _exporter.export( @@ -554,12 +562,7 @@ def create(self) -> dict: if not (self.files["base_graph"] or self.files["base_network"]): # Create the network from the network source if self._network_config.source == "shapefile": - logging.info("Start creating a network from the submitted shapefile.") - if self._any_cleanup_enabled(): - base_graph, network_gdf = self.network_shp() - else: - base_graph, network_gdf = self.network_cleanshp() - + base_graph, network_gdf = self._create_network_from_shp() elif self._network_config.source == "OSM PBF": logging.info( """The original OSM PBF import is no longer supported. diff --git a/ra2ce/graph/shp_network_wrapper/__init__.py b/ra2ce/graph/shp_network_wrapper/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ra2ce/graph/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py similarity index 100% rename from ra2ce/graph/vector_network_wrapper.py rename to ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py diff --git a/tests/graph/test_vector_network_wrapper.py b/tests/graph/test_vector_network_wrapper.py index e34a1a879..95fdc880f 100644 --- a/tests/graph/test_vector_network_wrapper.py +++ b/tests/graph/test_vector_network_wrapper.py @@ -1,13 +1,11 @@ import pytest -from pathlib import Path - import geopandas as gpd import networkx as nx from shapely.geometry import LineString, Point, MultiLineString from tests import test_data -from ra2ce.graph.vector_network_wrapper import VectorNetworkWrapper +from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper _test_dir = test_data / "vector_network_wrapper" diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index ea4116380..d81cd7d1b 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -14,14 +14,13 @@ _analysis_ini_name = "analyses.ini" _base_graph_p_filename = "base_graph.p" _base_network_feather_filename = "base_network.feather" +_skip_cases = [] def get_external_test_cases() -> list[pytest.param]: if not test_external_data.exists(): return [] - _skip_cases = ["bolivia"] - def get_pytest_param(test_dir: Path) -> pytest.param: _marks = [external_test] if test_dir.stem.lower() in _skip_cases: From bbb3bbd6c9b3921c954572892d495c0a1ea53be7 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 11:55:38 +0200 Subject: [PATCH 03/26] chore: Adapted VectorNetworkWrapper --- ra2ce/graph/networks.py | 21 +- .../vector_network_wrapper.py | 246 ++++++------------ tests/graph/test_vector_network_wrapper.py | 4 +- 3 files changed, 96 insertions(+), 175 deletions(-) diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index d69e0cea5..2c9358476 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -29,7 +29,6 @@ import networkx as nx import pandas as pd import pyproj -import momepy from shapely.geometry import MultiLineString from ra2ce.common.io.readers import GraphPickleReader @@ -227,13 +226,20 @@ def network_cleanshp(self) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: edges_complex (GeoDataFrame): The resulting network. """ # initialise vector network wrapper - vector_network_wrapper = VectorNetworkWrapper(config=self.config) + vector_network_wrapper = VectorNetworkWrapper( + list( + map(self._get_shp_paths, self._network_config.primary_file.split(",")) + ), + self.region, + "", + self._network_config.directed, + ) # setup network using the wrapper ( graph_complex, edges_complex, - ) = vector_network_wrapper.setup_network_from_vector() + ) = vector_network_wrapper.get_network_from_vector() # Set the CRS of the graph and network to wrapper crs self.base_graph_crs = vector_network_wrapper.crs @@ -434,6 +440,9 @@ def generate_origins_from_raster(self): return out_fn + def _get_shp_paths(self, shp_str: str) -> Path: + return self._network_dir.joinpath(shp_str) + def read_merge_shp(self, crs_: pyproj.CRS) -> gpd.GeoDataFrame: """Imports shapefile(s) and saves attributes in a pandas dataframe. @@ -445,14 +454,12 @@ def read_merge_shp(self, crs_: pyproj.CRS) -> gpd.GeoDataFrame: """ # read shapefiles and add to list with path - def get_shp_paths(shp_str: str) -> Path: - return self._network_dir.joinpath(shp_str) shapefiles_analysis = list( - map(get_shp_paths, self._network_config.primary_file.split(",")) + map(self._get_shp_paths, self._network_config.primary_file.split(",")) ) shapefiles_diversion = list( - map(get_shp_paths, self._network_config.diversion_file.split(",")) + map(self._get_shp_paths, self._network_config.diversion_file.split(",")) ) # concatenate all shapefile into one geodataframe and set analysis to 1 or 0 for diversions diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py index e6e3b2035..e183c7412 100644 --- a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py @@ -9,10 +9,12 @@ from shapely.geometry import Point from pyproj import CRS +from ra2ce.graph.network_config_data.network_config_data import ( + NetworkConfigData, + NetworkSection, +) import ra2ce.graph.networks_utils as nut -logger = logging.getLogger() - class VectorNetworkWrapper: """A class for handling and manipulating vector files. @@ -21,14 +23,18 @@ class VectorNetworkWrapper: network. """ - name: str - region: gpd.GeoDataFrame + primary_files: list[Path] + region_path: Path crs: CRS - input_path: Path - output_path: Path - network_dict: dict + directed: bool - def __init__(self, config: dict) -> None: + def __init__( + self, + primary_files: list[Path], + region_path: Path, + crs_value: str, + is_directed: bool, + ) -> None: """Initializes the VectorNetworkWrapper object. Args: @@ -38,99 +44,93 @@ def __init__(self, config: dict) -> None: ValueError: If the config is None or doesn't contain a network dictionary, or if config['network'] is not a dictionary. """ + self.primary_files = primary_files + self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") + self.region_path = region_path + self.directed = is_directed - if not config: - raise ValueError("Config cannot be None") - if not config.get("network", {}): - raise ValueError( - "A network dictionary is required for creating a " - + f"{self.__class__.__name__} object." - ) - if not isinstance(config.get("network"), dict): - raise ValueError('Config["network"] should be a dictionary') - - self._setup_global(config) - self.network_dict = self._get_network_opt(config["network"]) - - def _parse_ini_stringlist(self, value: str) -> Union[str, list, None]: - """Parses a string with "," into a list from an ini file. - - Args: - value (str): Value to parse. + def get_network_from_vector( + self, + ) -> tuple[nx.MultiGraph, gpd.GeoDataFrame]: + """Gets a network built from vector files. Returns: - list of str If the value contains a comma, it is split and returned - as a list, otherwise the original value is returned. - None if the value is an empty string. + nx.MultiGraph: MultiGraph representing the graph. + gpd.GeoDataFrame: GeoDataFrame representing the network. """ - if not value: - return None - elif isinstance(value, str) and "," in value: - return value.split(",") + gdf = self._read_vector_to_project_region_and_crs( + vector_filenames=self.primary_files, crs=self.crs + ) + gdf = self.clean_vector(gdf) + if self.directed: + graph = self.setup_digraph_from_vector(gdf) else: - return value - - def _setup_global(self, config: dict) -> None: - """Sets up project properties based on provided configuration. - - Args: - config (dict): Project configuration dictionary. - """ - self.input_path = config.get("static").joinpath("network") - self.output_path = config.get("output") + graph = self.setup_graph_from_vector(gdf) + edges, nodes = self.setup_network_edges_and_nodes_from_graph(graph) + graph_complex = nut.graph_from_gdf(edges, nodes, node_id="node_fid") + return graph_complex, edges - project_config = config.get("project") - name = project_config.get("name", "project_name") - region = project_config.get("region", None) - crs = project_config.get("crs", 4326) - self.name = name - self.crs = CRS.from_user_input(crs) - self.region = self._read_files([Path(region)]) if region else region + def _read_vector_to_project_region_and_crs( + self, vector_filenames: list[Path], crs: CRS + ) -> gpd.GeoDataFrame: + """Reads a vector file or a list of vector files. - def _parse_ini_filenamelist(self, filename: str) -> list[Path]: - """Makes a list of file paths by joining with input path and checks validity of files. + Clips for project region and reproject to project crs if available. + Explodes multi geometry into single geometry. Args: - filename (str): String of file names separated by comma (","). + vector_filenames (list[Path]): List of Path to the vector files. + crs (CRS): Coordinate reference system for the files. Allow only one crs for all `vector_filenames`. Returns: - List[Path]: List of file paths. + gpd.GeoDataFrame: GeoDataFrame representing the vector data. """ - if not isinstance(filename, str): - logger.error("file names are not valid.") + gdf = self._read_files(vector_filenames) + if gdf is None: + logging.info("no file is read.") + return None - file_paths = [self.input_path.joinpath(f.strip()) for f in filename.split(",")] + # set crs and reproject if needed + if not gdf.crs and crs: + gdf = gdf.set_crs(crs) + logging.info("setting crs as default EPSG:4326. specify crs if incorrect") - for f in file_paths: - if not f.resolve().is_file(): - logger.error(f"vector file {f} is not found.") + if self.crs: + gdf = gdf.to_crs(self.crs) + logging.info("reproject vector file to project crs") - return file_paths + # clip for region + if self.region_path: + _region_path = self._read_files([self.region_path]) + gdf = gpd.overlay( + gdf, _region_path, how="intersection", keep_geom_type=True + ) + logging.info("clip vector file to project region") - def _get_network_opt(self, network_config: dict) -> dict: - """Retrieves network options used in this wrapper from provided configuration. + # validate + if not any(gdf): + logging.warning("No vector features found within project region") + return None + + return gdf + + def _read_files(self, file_list: list[Path]) -> gpd.GeoDataFrame: + """Reads a list of files into a GeoDataFrame. Args: - network_config (dict): Network configuration dictionary. + file_list (list[Path]): List of file paths. Returns: - dict: Dictionary of network options. + gpd.GeoDataFrame: GeoDataFrame representing the data. """ - - files = self._parse_ini_filenamelist(network_config.get("primary_file", "")) - file_id = self._parse_ini_stringlist( - network_config.get("file_id", "") - ) # only needed when cleanup based on fid - file_filter = self._parse_ini_stringlist(network_config.get("filter", "")) - file_crs = CRS.from_user_input(network_config.get("crs", self.crs)) - is_directed = network_config.get("directed", False) - return dict( - files=files, - file_id=file_id, - file_filter=file_filter, - file_crs=file_crs, - is_directed=is_directed, + # read file + gdf = gpd.GeoDataFrame(pd.concat(list(map(gpd.read_file, file_list)))) + logging.info( + "Read files {} into a 'GeoDataFrame'.".format( + ", ".join(map(str, file_list)) + ) ) + return gdf @staticmethod def setup_digraph_from_vector(gdf: gpd.GeoDataFrame) -> nx.DiGraph: @@ -149,7 +149,7 @@ def setup_digraph_from_vector(gdf: gpd.GeoDataFrame) -> nx.DiGraph: # to graph digraph = nx.DiGraph(crs=gdf.crs, approach="primal") - for index, row in gdf.iterrows(): + for _, row in gdf.iterrows(): from_node = row.geometry.coords[0] to_node = row.geometry.coords[-1] digraph.add_node(from_node, geometry=Point(from_node)) @@ -207,92 +207,6 @@ def setup_network_edges_and_nodes_from_graph( edges.crs = graph.graph["crs"] return edges, nodes - def setup_network_from_vector( - self, - ) -> tuple[nx.MultiGraph, gpd.GeoDataFrame]: - """Sets up a network from vector files. - - Returns: - nx.MultiGraph: MultiGraph representing the graph. - gpd.GeoDataFrame: GeoDataFrame representing the network. - """ - files = self.network_dict["files"] - file_crs = self.network_dict["file_crs"] - is_directed = self.network_dict["is_directed"] - - gdf = self._read_vector_to_project_region_and_crs( - vector_filenames=files, crs=file_crs - ) - gdf = VectorNetworkWrapper.clean_vector(gdf) - if is_directed: - graph = VectorNetworkWrapper.setup_digraph_from_vector(gdf) - else: - graph = VectorNetworkWrapper.setup_graph_from_vector(gdf) - edges, nodes = VectorNetworkWrapper.setup_network_edges_and_nodes_from_graph( - graph - ) - graph_complex = nut.graph_from_gdf(edges, nodes, node_id="node_fid") - return graph_complex, edges - - def _read_vector_to_project_region_and_crs( - self, vector_filenames: list[Path], crs: CRS - ) -> gpd.GeoDataFrame: - """Reads a vector file or a list of vector files. - - Clips for project region and reproject to project crs if available. - Explodes multi geometry into single geometry. - - Args: - vector_filenames (list[Path]): List of Path to the vector files. - crs (CRS): Coordinate reference system for the files. Allow only one crs for all `vector_filenames`. - - Returns: - gpd.GeoDataFrame: GeoDataFrame representing the vector data. - """ - gdf = self._read_files(vector_filenames) - if gdf is None: - logger.info("no file is read.") - return None - - # set crs and reproject if needed - if not gdf.crs and crs: - gdf = gdf.set_crs(crs) - logger.info("setting crs as default EPSG:4326. specify crs if incorrect") - - if self.crs: - gdf = gdf.to_crs(self.crs) - logger.info("reproject vector file to project crs") - - # clip for region - if self.region is not None: - gdf = gpd.overlay(gdf, self.region, how="intersection", keep_geom_type=True) - logger.info("clip vector file to project region") - - # validate - if not any(gdf): - logger.warning("No vector features found within project region") - return None - - return gdf - - def _read_files(self, file_list: list[Path]) -> gpd.GeoDataFrame: - """Reads a list of files into a GeoDataFrame. - - Args: - file_list (list[Path]): List of file paths. - - Returns: - gpd.GeoDataFrame: GeoDataFrame representing the data. - """ - # read file - if isinstance(file_list, list): - gdf = gpd.GeoDataFrame(pd.concat([gpd.read_file(_fn) for _fn in file_list])) - logger.info("read vector files.") - else: - gdf = None - logger.info("no file is read.") - return gdf - @staticmethod def clean_vector(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: """Cleans a GeoDataFrame. diff --git a/tests/graph/test_vector_network_wrapper.py b/tests/graph/test_vector_network_wrapper.py index 95fdc880f..cbb8f73be 100644 --- a/tests/graph/test_vector_network_wrapper.py +++ b/tests/graph/test_vector_network_wrapper.py @@ -169,7 +169,7 @@ def test_setup_network_from_vector(self, _config_fixture): test_wrapper = VectorNetworkWrapper(_config_fixture) # When - graph, edges = test_wrapper.setup_network_from_vector() + graph, edges = test_wrapper.get_network_from_vector() # Then assert isinstance(graph, nx.MultiGraph) @@ -181,7 +181,7 @@ def test_setup_network_from_vector_with_region(self, _config_fixture): test_wrapper = VectorNetworkWrapper(_config_fixture) # When - graph, edges = test_wrapper.setup_network_from_vector() + graph, edges = test_wrapper.get_network_from_vector() # Then assert isinstance(graph, nx.MultiGraph) From 4a4f16b8a3e56d9288968555a0cd039cc59feda8 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 12:38:00 +0200 Subject: [PATCH 04/26] chore: renames of methods to fit the current get/set approach. Updated tests --- .../vector_network_wrapper.py | 50 ++--- tests/graph/test_vector_network_wrapper.py | 191 ++++++------------ 2 files changed, 71 insertions(+), 170 deletions(-) diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py index e183c7412..7a751658f 100644 --- a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py @@ -9,10 +9,6 @@ from shapely.geometry import Point from pyproj import CRS -from ra2ce.graph.network_config_data.network_config_data import ( - NetworkConfigData, - NetworkSection, -) import ra2ce.graph.networks_utils as nut @@ -58,41 +54,25 @@ def get_network_from_vector( nx.MultiGraph: MultiGraph representing the graph. gpd.GeoDataFrame: GeoDataFrame representing the network. """ - gdf = self._read_vector_to_project_region_and_crs( - vector_filenames=self.primary_files, crs=self.crs - ) + gdf = self._read_vector_to_project_region_and_crs() gdf = self.clean_vector(gdf) if self.directed: - graph = self.setup_digraph_from_vector(gdf) + graph = self.get_direct_graph_from_vector(gdf) else: - graph = self.setup_graph_from_vector(gdf) - edges, nodes = self.setup_network_edges_and_nodes_from_graph(graph) + graph = self.get_indirect_graph_from_vector(gdf) + edges, nodes = self.get_network_edges_and_nodes_from_graph(graph) graph_complex = nut.graph_from_gdf(edges, nodes, node_id="node_fid") return graph_complex, edges - def _read_vector_to_project_region_and_crs( - self, vector_filenames: list[Path], crs: CRS - ) -> gpd.GeoDataFrame: - """Reads a vector file or a list of vector files. - - Clips for project region and reproject to project crs if available. - Explodes multi geometry into single geometry. - - Args: - vector_filenames (list[Path]): List of Path to the vector files. - crs (CRS): Coordinate reference system for the files. Allow only one crs for all `vector_filenames`. - - Returns: - gpd.GeoDataFrame: GeoDataFrame representing the vector data. - """ - gdf = self._read_files(vector_filenames) + def _read_vector_to_project_region_and_crs(self) -> gpd.GeoDataFrame: + gdf = self._read_files(self.primary_files) if gdf is None: logging.info("no file is read.") return None # set crs and reproject if needed - if not gdf.crs and crs: - gdf = gdf.set_crs(crs) + if not gdf.crs and self.crs: + gdf = gdf.set_crs(self.crs) logging.info("setting crs as default EPSG:4326. specify crs if incorrect") if self.crs: @@ -101,10 +81,8 @@ def _read_vector_to_project_region_and_crs( # clip for region if self.region_path: - _region_path = self._read_files([self.region_path]) - gdf = gpd.overlay( - gdf, _region_path, how="intersection", keep_geom_type=True - ) + _region_gpd = self._read_files([self.region_path]) + gdf = gpd.overlay(gdf, _region_gpd, how="intersection", keep_geom_type=True) logging.info("clip vector file to project region") # validate @@ -133,7 +111,7 @@ def _read_files(self, file_list: list[Path]) -> gpd.GeoDataFrame: return gdf @staticmethod - def setup_digraph_from_vector(gdf: gpd.GeoDataFrame) -> nx.DiGraph: + def get_direct_graph_from_vector(gdf: gpd.GeoDataFrame) -> nx.DiGraph: """Creates a simple directed graph with node and edge geometries based on a given GeoDataFrame. Args: @@ -165,7 +143,7 @@ def setup_digraph_from_vector(gdf: gpd.GeoDataFrame) -> nx.DiGraph: return digraph @staticmethod - def setup_graph_from_vector(gdf: gpd.GeoDataFrame) -> nx.Graph: + def get_indirect_graph_from_vector(gdf: gpd.GeoDataFrame) -> nx.Graph: """Creates a simple undirected graph with node and edge geometries based on a given GeoDataFrame. Args: @@ -175,11 +153,11 @@ def setup_graph_from_vector(gdf: gpd.GeoDataFrame) -> nx.Graph: Returns: nx.Graph: NetworkX graph object with "crs", "approach" as graph properties. """ - digraph = VectorNetworkWrapper.setup_digraph_from_vector(gdf) + digraph = VectorNetworkWrapper.get_direct_graph_from_vector(gdf) return digraph.to_undirected() @staticmethod - def setup_network_edges_and_nodes_from_graph( + def get_network_edges_and_nodes_from_graph( graph: nx.Graph, ) -> tuple[gpd.GeoDataFrame, gpd.GeoDataFrame]: """Sets up network nodes and edges from a given graph. diff --git a/tests/graph/test_vector_network_wrapper.py b/tests/graph/test_vector_network_wrapper.py index cbb8f73be..51775280c 100644 --- a/tests/graph/test_vector_network_wrapper.py +++ b/tests/graph/test_vector_network_wrapper.py @@ -1,3 +1,4 @@ +from pathlib import Path import pytest import geopandas as gpd @@ -12,34 +13,12 @@ class TestVectorNetworkWrapper: @pytest.fixture - def _config_fixture(self) -> dict: - yield { - "project": { - "name": "test", - "crs": 4326, - }, - "network": { - "directed": False, - "source": "shapefile", - "primary_file": "_test_lines.geojson", - "diversion_file": None, - "file_id": "fid", - "polygon": None, - "network_type": None, - "road_types": None, - "save_shp": False, - }, - "static": _test_dir / "static", - "output": _test_dir / "output", - } - - @pytest.fixture - def points_gdf(self): + def points_gdf(self) -> gpd.GeoDataFrame: points = [Point(-122.3, 47.6), Point(-122.2, 47.5), Point(-122.1, 47.6)] return gpd.GeoDataFrame(geometry=points, crs=4326) @pytest.fixture - def lines_gdf(self): + def lines_gdf(self) -> gpd.GeoDataFrame: points = [Point(-122.3, 47.6), Point(-122.2, 47.5), Point(-122.1, 47.6)] lines = [LineString([points[0], points[1]]), LineString([points[1], points[2]])] return gpd.GeoDataFrame(geometry=lines, crs=4326) @@ -59,135 +38,79 @@ def mock_graph(self): return graph - @pytest.mark.parametrize( - "config", - [ - pytest.param(None, id="NONE as dictionary"), - pytest.param({}, id="Empty dictionary"), - pytest.param({"network": {}}, id='Empty "network" in Config'), - pytest.param({"network": "string"}, id='Invalid "network" type in Config'), - ], - ) - def test_init(self, config: dict): - with pytest.raises(ValueError) as exc_err: - VectorNetworkWrapper(config=config) - assert str(exc_err.value) in [ - "Config cannot be None", - "A network dictionary is required for creating a VectorNetworkWrapper object.", - 'Config["network"] should be a dictionary', - ] - - def test_parse_ini_stringlist_with_comma_separated_string(self, _config_fixture): - # Given - test_wrapper = VectorNetworkWrapper(_config_fixture) - ini_value = "a,b,c" - - # When - result = test_wrapper._parse_ini_stringlist(ini_value) - - # Then - assert result == ["a", "b", "c"] - - def test_parse_ini_stringlist_with_single_string(self, _config_fixture): - # Given - test_wrapper = VectorNetworkWrapper(_config_fixture) - ini_value = "abc" - - # When - result = test_wrapper._parse_ini_stringlist(ini_value) - - # Then - assert result == "abc" - - def test_parse_ini_stringlist_with_empty_string(self, _config_fixture): - # Given - test_wrapper = VectorNetworkWrapper(_config_fixture) - ini_value = "" - - # When - result = test_wrapper._parse_ini_stringlist(ini_value) + def test_init_without_crs_sts_default(self): + # 1. Define test data. + _primary_files = [Path("dummy_primary")] + _region = Path("dummy_region") + _crs_value = "" - # Then - assert result is None + # 2. Run test. + _wrapper = VectorNetworkWrapper(_primary_files, _region, _crs_value, False) - def test_parse_ini_filenamelist(self, _config_fixture): - # Given - test_wrapper = VectorNetworkWrapper(_config_fixture) - ini_value = "_test_lines.geojson, dummy.geojson" + # 3. Verify expectations. + assert isinstance(_wrapper, VectorNetworkWrapper) + assert _wrapper.primary_files == _primary_files + assert _wrapper.region_path == _region + assert str(_wrapper.crs) == "epsg:4326" - # When - file_paths = test_wrapper._parse_ini_filenamelist(ini_value) + @pytest.fixture + def _valid_wrapper(self) -> VectorNetworkWrapper: + _network_dir = _test_dir.joinpath("static", "network") + yield VectorNetworkWrapper( + primary_files=[_network_dir.joinpath("_test_lines.geojson")], + region_path=None, + crs_value=4326, + is_directed=False, + ) - # Then - assert file_paths[0].is_file() - - def test_setup_global(self, _config_fixture): - test_wrapper = VectorNetworkWrapper(_config_fixture) - test_wrapper._setup_global(_config_fixture) - assert test_wrapper.name == "test" - assert test_wrapper.region is None - assert test_wrapper.crs.to_epsg() == 4326 - assert test_wrapper.input_path == _test_dir / "static/network" - assert test_wrapper.output_path == _test_dir / "output" - - def test_get_network_opt(self, _config_fixture): - test_wrapper = VectorNetworkWrapper(_config_fixture) - network_dict = test_wrapper._get_network_opt(_config_fixture["network"]) - assert network_dict["files"][0].is_file() - assert network_dict["file_id"] == "fid" - assert network_dict["file_crs"].to_epsg() == 4326 - assert network_dict["is_directed"] is False - - def test_read_vector_to_project_region_and_crs(self, _config_fixture): + def test_read_vector_to_project_region_and_crs( + self, _valid_wrapper: VectorNetworkWrapper + ): # Given - test_wrapper = VectorNetworkWrapper(_config_fixture) - files = test_wrapper.network_dict["files"] - file_crs = test_wrapper.network_dict["file_crs"] + assert not _valid_wrapper.directed # When - vector = test_wrapper._read_vector_to_project_region_and_crs(files, file_crs) + vector = _valid_wrapper._read_vector_to_project_region_and_crs() # Then assert isinstance(vector, gpd.GeoDataFrame) - def test_read_vector_to_project_region_and_crs_with_region(self, _config_fixture): - # Given - _config_fixture["project"]["region"] = _test_dir / "_test_polygon.geojson" - test_wrapper = VectorNetworkWrapper(_config_fixture) - files = test_wrapper.network_dict["files"] - file_crs = test_wrapper.network_dict["file_crs"] - - # When - vector = test_wrapper._read_vector_to_project_region_and_crs(files, file_crs) - - # Then - assert vector.crs == test_wrapper.region.crs - assert test_wrapper.region.covers(vector.unary_union).all() - - def test_setup_network_from_vector(self, _config_fixture): + def test_read_vector_to_project_region_and_crs_with_region( + self, _valid_wrapper: VectorNetworkWrapper + ): # Given - test_wrapper = VectorNetworkWrapper(_config_fixture) + _valid_wrapper.region_path = _test_dir / "_test_polygon.geojson" + _expected_region = gpd.read_file(_valid_wrapper.region_path) + assert isinstance(_expected_region, gpd.GeoDataFrame) # When - graph, edges = test_wrapper.get_network_from_vector() + vector = _valid_wrapper._read_vector_to_project_region_and_crs() # Then - assert isinstance(graph, nx.MultiGraph) - assert isinstance(edges, gpd.GeoDataFrame) + assert vector.crs == _expected_region.crs + assert _expected_region.covers(vector.unary_union).all() - def test_setup_network_from_vector_with_region(self, _config_fixture): + @pytest.mark.parametrize( + "region_path", + [ + pytest.param(None, id="No region"), + pytest.param(_test_dir / "_test_polygon.geojson", id="With region"), + ], + ) + def test_get_network_from_vector( + self, _valid_wrapper: VectorNetworkWrapper, region_path: Path + ): # Given - _config_fixture["project"]["region"] = _test_dir / "_test_polygon.geojson" - test_wrapper = VectorNetworkWrapper(_config_fixture) + _valid_wrapper.region_path = region_path # When - graph, edges = test_wrapper.get_network_from_vector() + graph, edges = _valid_wrapper.get_network_from_vector() # Then assert isinstance(graph, nx.MultiGraph) assert isinstance(edges, gpd.GeoDataFrame) - def test_clean_vector(self, lines_gdf): + def test_clean_vector(self, lines_gdf: gpd.GeoDataFrame): # Given gdf1 = VectorNetworkWrapper.explode_and_deduplicate_geometries(lines_gdf) @@ -199,9 +122,9 @@ def test_clean_vector(self, lines_gdf): # Then assert gdf1.equals(gdf2) - def test_setup_graph_from_vector(self, lines_gdf): + def test_get_indirect_graph_from_vector(self, lines_gdf: gpd.GeoDataFrame): # When - graph = VectorNetworkWrapper.setup_graph_from_vector(lines_gdf) + graph = VectorNetworkWrapper.get_indirect_graph_from_vector(lines_gdf) # Then assert graph.nodes(data="geometry") is not None @@ -209,18 +132,18 @@ def test_setup_graph_from_vector(self, lines_gdf): assert graph.graph["crs"] == lines_gdf.crs assert isinstance(graph, nx.Graph) and not isinstance(graph, nx.DiGraph) - def test_setup_digraph_from_vector(self, lines_gdf): + def test_get_direct_graph_from_vector(self, lines_gdf: gpd.GeoDataFrame): # When - graph = VectorNetworkWrapper.setup_digraph_from_vector(lines_gdf) + graph = VectorNetworkWrapper.get_direct_graph_from_vector(lines_gdf) # Then assert isinstance(graph, nx.DiGraph) - def test_setup_network_edges_and_nodes_from_graph( + def test_get_network_edges_and_nodes_from_graph( self, mock_graph, points_gdf, lines_gdf ): # When - edges, nodes = VectorNetworkWrapper.setup_network_edges_and_nodes_from_graph( + edges, nodes = VectorNetworkWrapper.get_network_edges_and_nodes_from_graph( mock_graph ) From 3b3c15d29cc9be5a7c6d437776c3a8a7903d3326 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 14:02:20 +0200 Subject: [PATCH 05/26] chore: Added new wrapper for shp files --- ra2ce/graph/networks.py | 198 +++-------------- .../shp_network_wrapper.py | 199 ++++++++++++++++++ .../vector_network_wrapper.py | 1 - 3 files changed, 227 insertions(+), 171 deletions(-) create mode 100644 ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 2c9358476..9c839a4b4 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -29,7 +29,6 @@ import networkx as nx import pandas as pd import pyproj -from shapely.geometry import MultiLineString from ra2ce.common.io.readers import GraphPickleReader from ra2ce.graph import networks_utils as nut @@ -38,6 +37,7 @@ from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper from ra2ce.graph.segmentation import Segmentation +from ra2ce.graph.shp_network_wrapper.shp_network_wrapper import ShpNetworkWrapper from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper @@ -51,11 +51,6 @@ class Network: config: A dictionary with the configuration details on how to create and adjust the network. """ - output_graph_dir: Path - files: dict - base_graph_crs: Any - base_network_crs: Any - def __init__(self, network_config: NetworkConfigData, files: dict): # General self.project_name = network_config.project.name @@ -96,6 +91,17 @@ def __init__(self, network_config: NetworkConfigData, files: dict): # files self.files = files + def _get_shp_paths(self, shp_str: str) -> Path: + return self._network_dir.joinpath(shp_str) + + def _create_network_from_shp( + self, + ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + logging.info("Start creating a network from the submitted shapefile.") + if self._any_cleanup_enabled(): + return self.network_shp() + return self.network_cleanshp() + def network_shp( self, crs: int = 4326 ) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: @@ -112,103 +118,25 @@ def network_shp( edges_complex (GeoDataFrame): The resulting network. """ # Make a pyproj CRS from the EPSG code - crs = pyproj.CRS.from_user_input(crs) - - lines = self.read_merge_shp(crs) - - logging.info( - "Function [read_merge_shp]: executed with {} {}".format( - self._network_config.primary_file, - self._network_config.diversion_file, - ) + _shp_network_wrapper = ShpNetworkWrapper( + primary_files=list( + map(self._get_shp_paths, self._network_config.primary_file.split(",")) + ), + diversion_files=list( + map(self._get_shp_paths, self._network_config.diversion_file.split(",")) + ), + region_path=None, + crs_value=crs, + is_directed=self._network_config.directed, ) - - # Check which of the lines are merged, also for the fid. The fid of the first line with a traffic count is taken. - # The list of fid's is reduced by the fid's that are not anymore in the merged lines - if self._cleanup.merge_lines: - aadt_names = [] - edges, lines_merged = nut.merge_lines_automatic( - lines, self._network_config.file_id, aadt_names, crs - ) - logging.info( - "Function [merge_lines_automatic]: executed with properties {}".format( - list(edges.columns) - ) - ) - else: - edges, lines_merged = lines, gpd.GeoDataFrame() - - edges, id_name = nut.gdf_check_create_unique_ids( - edges, self._network_config.file_id + graph_complex, edges_complex = _shp_network_wrapper.get_network( + self._cleanup.merge_lines, + self._cleanup.snapping_threshold, + self.output_graph_dir, + self.project_name, + self._cleanup.segmentation_length, ) - if self._cleanup.snapping_threshold: - edges = nut.snap_endpoints_lines( - edges, self._cleanup.snapping_threshold, id_name, crs - ) - logging.info( - "Function [snap_endpoints_lines]: executed with threshold = {}".format( - self._cleanup.snapping_threshold - ) - ) - - # merge merged lines if there are any merged lines - if not lines_merged.empty: - # save the merged lines to a shapefile - CHECK if there are lines merged that should not be merged (e.g. main + secondary road) - lines_merged.set_geometry( - col="geometry", inplace=True - ) # To ensure the object is a GeoDataFrame and not a Series - lines_merged.to_file( - os.path.join( - self.output_graph_dir, - "{}_lines_that_merged.shp".format(self.project_name), - ) - ) - logging.info( - "Function [edges_to_shp]: saved at {}".format( - os.path.join( - self.output_graph_dir, - "{}_lines_that_merged".format(self.project_name), - ) - ) - ) - - # Get the unique points at the end of lines and at intersections to create nodes - nodes = nut.create_nodes(edges, crs, self._cleanup.cut_at_intersections) - logging.info("Function [create_nodes]: executed") - - edges = nut.cut_lines( - edges, nodes, id_name, tolerance=0.00001, crs_=crs - ) ## PAY ATTENTION TO THE TOLERANCE, THE UNIT IS DEGREES - logging.info("Function [cut_lines]: executed") - - if not edges.crs: - edges.crs = crs - - # create tuples from the adjecent nodes and add as column in geodataframe - edges_complex = nut.join_nodes_edges(nodes, edges, id_name) - edges_complex.crs = crs # set the right CRS - edges_complex.dropna(subset=["node_A", "node_B"], inplace=True) - - assert ( - edges_complex["node_A"].isnull().sum() == 0 - ), "Some edges cannot be assigned nodes, please check your input shapefile." - assert ( - edges_complex["node_B"].isnull().sum() == 0 - ), "Some edges cannot be assigned nodes, please check your input shapefile." - - # Create networkx graph from geodataframe - graph_complex = nut.graph_from_gdf(edges_complex, nodes, node_id="node_fid") - logging.info("Function [graph_from_gdf]: executed") - - if not math.isnan(self._cleanup.segmentation_length): - edges_complex = Segmentation( - edges_complex, self._cleanup.segmentation_length - ) - edges_complex = edges_complex.apply_segmentation() - if edges_complex.crs is None: # The CRS might have dissapeared. - edges_complex.crs = crs # set the right CRS - self.base_graph_crs = pyproj.CRS.from_user_input(crs) self.base_network_crs = pyproj.CRS.from_user_input(crs) @@ -247,14 +175,6 @@ def network_cleanshp(self) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: return graph_complex, edges_complex - def _create_network_from_shp( - self, - ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - logging.info("Start creating a network from the submitted shapefile.") - if self._any_cleanup_enabled(): - return self.network_shp() - return self.network_cleanshp() - def _export_linking_tables(self, linking_tables: List[Any]) -> None: _exporter = JsonExporter() _exporter.export( @@ -440,68 +360,6 @@ def generate_origins_from_raster(self): return out_fn - def _get_shp_paths(self, shp_str: str) -> Path: - return self._network_dir.joinpath(shp_str) - - def read_merge_shp(self, crs_: pyproj.CRS) -> gpd.GeoDataFrame: - """Imports shapefile(s) and saves attributes in a pandas dataframe. - - Args: - crs_ (int): the EPSG number of the coordinate reference system that is used - Returns: - lines (list of shapely LineStrings): full list of linestrings - properties (pandas dataframe): attributes of shapefile(s), in order of the linestrings in lines - """ - - # read shapefiles and add to list with path - - shapefiles_analysis = list( - map(self._get_shp_paths, self._network_config.primary_file.split(",")) - ) - shapefiles_diversion = list( - map(self._get_shp_paths, self._network_config.diversion_file.split(",")) - ) - - # concatenate all shapefile into one geodataframe and set analysis to 1 or 0 for diversions - lines = [gpd.read_file(shp) for shp in shapefiles_analysis] - - if self._network_config.diversion_file: - lines.extend( - [ - nut.check_crs_gdf(gpd.read_file(shp), crs_) - for shp in shapefiles_diversion - ] - ) - lines = pd.concat(lines) - - lines.crs = crs_ - - # Check if there are any multilinestrings and convert them to linestrings. - if lines["geometry"].apply(lambda row: isinstance(row, MultiLineString)).any(): - mls_idx = lines.loc[ - lines["geometry"].apply(lambda row: isinstance(row, MultiLineString)) - ].index - for idx in mls_idx: - # Multilinestrings to linestrings - new_rows_geoms = list(lines.iloc[idx]["geometry"].geoms) - for nrg in new_rows_geoms: - dict_attributes = dict(lines.iloc[idx]) - dict_attributes["geometry"] = nrg - lines.loc[max(lines.index) + 1] = dict_attributes - - lines = lines.drop(labels=mls_idx, axis=0) - - # append the length of the road stretches - lines["length"] = lines["geometry"].apply(lambda x: nut.line_length(x, crs_)) - - logging.info( - "Shapefile(s) loaded with attributes: {}.".format( - list(lines.columns.values) - ) - ) # fill in parameter names - - return lines - def get_avg_speed( self, original_graph: nx.classes.graph.Graph ) -> nx.classes.graph.Graph: diff --git a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py new file mode 100644 index 000000000..8dda060b1 --- /dev/null +++ b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py @@ -0,0 +1,199 @@ +""" + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Risk Assessment and Adaptation for Critical Infrastructure (RA2CE). + Copyright (C) 2023 Stichting Deltares + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +import math +from pathlib import Path +from pyproj import CRS +import geopandas as gpd +import pandas as pd +import ra2ce.graph.networks_utils as nut +from shapely.geometry import MultiLineString +import logging +import networkx as nx + +from ra2ce.graph.segmentation import Segmentation + + +class ShpNetworkWrapper: + primary_files: list[Path] + region_path: Path + crs: CRS + directed: bool + file_id: str + + def __init__( + self, + primary_files: list[Path], + diversion_files: list[Path], + region_path: Path, + crs_value: str, + is_directed: bool, + file_id: str, + ) -> None: + """Initializes the VectorNetworkWrapper object. + + Args: + config (dict): Configuration dictionary. + + Raises: + ValueError: If the config is None or doesn't contain a network dictionary, + or if config['network'] is not a dictionary. + """ + self.primary_files = primary_files + self.diversion_files = diversion_files + self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") + self.region_path = region_path + self.directed = is_directed + self.file_id = file_id + + def read_merge_shp(self) -> gpd.GeoDataFrame: + """Imports shapefile(s) and saves attributes in a pandas dataframe. + + Returns: + lines (list of shapely LineStrings): full list of linestrings + properties (pandas dataframe): attributes of shapefile(s), in order of the linestrings in lines + """ + # concatenate all shapefile into one geodataframe and set analysis to 1 or 0 for diversions + lines = [gpd.read_file(shp) for shp in self.primary_files] + + if any(self.diversion_files): + lines.extend( + [ + nut.check_crs_gdf(gpd.read_file(shp), self.crs) + for shp in self.diversion_files + ] + ) + lines = pd.concat(lines) + + lines.crs = self.crs + + # Check if there are any multilinestrings and convert them to linestrings. + if lines["geometry"].apply(lambda row: isinstance(row, MultiLineString)).any(): + mls_idx = lines.loc[ + lines["geometry"].apply(lambda row: isinstance(row, MultiLineString)) + ].index + for idx in mls_idx: + # Multilinestrings to linestrings + new_rows_geoms = list(lines.iloc[idx]["geometry"].geoms) + for nrg in new_rows_geoms: + dict_attributes = dict(lines.iloc[idx]) + dict_attributes["geometry"] = nrg + lines.loc[max(lines.index) + 1] = dict_attributes + + lines = lines.drop(labels=mls_idx, axis=0) + + # append the length of the road stretches + lines["length"] = lines["geometry"].apply( + lambda x: nut.line_length(x, self.crs) + ) + + logging.info( + "Shapefile(s) loaded with attributes: {}.".format( + list(lines.columns.values) + ) + ) # fill in parameter names + + return lines + + def _get_complex_graph_and_edges( + self, edges: gpd.GeoDataFrame, id_name: str + ) -> tuple[nx.MultiGraph, gpd.GeoDataFrame]: + # Get the unique points at the end of lines and at intersections to create nodes + nodes = nut.create_nodes(edges, self.crs, self._cleanup.cut_at_intersections) + logging.info("Function [create_nodes]: executed") + + edges = nut.cut_lines( + edges, nodes, id_name, tolerance=0.00001, crs_=self.crs + ) ## PAY ATTENTION TO THE TOLERANCE, THE UNIT IS DEGREES + logging.info("Function [cut_lines]: executed") + + if not edges.crs: + edges.crs = self.crs + + # create tuples from the adjecent nodes and add as column in geodataframe + edges_complex = nut.join_nodes_edges(nodes, edges, id_name) + edges_complex.crs = self.crs # set the right CRS + edges_complex.dropna(subset=["node_A", "node_B"], inplace=True) + + assert ( + edges_complex["node_A"].isnull().sum() == 0 + ), "Some edges cannot be assigned nodes, please check your input shapefile." + assert ( + edges_complex["node_B"].isnull().sum() == 0 + ), "Some edges cannot be assigned nodes, please check your input shapefile." + + # Create networkx graph from geodataframe + graph_complex = nut.graph_from_gdf(edges_complex, nodes, node_id="node_fid") + logging.info("Function [graph_from_gdf]: executed") + return graph_complex, edges_complex + + def get_network(self, merge_lines: bool, snapping_threshold: bool): + lines = self.read_merge_shp() + + # Check which of the lines are merged, also for the fid. The fid of the first line with a traffic count is taken. + # The list of fid's is reduced by the fid's that are not anymore in the merged lines + edges, lines_merged = lines, gpd.GeoDataFrame() + if merge_lines: + aadt_names = [] + edges, lines_merged = nut.merge_lines_automatic( + lines, self.file_id, aadt_names, self.crs + ) + logging.info( + "Function [merge_lines_automatic]: executed with properties {}".format( + list(edges.columns) + ) + ) + + edges, id_name = nut.gdf_check_create_unique_ids(edges, self.file_id) + + if snapping_threshold: + # TODO: snapping threshold it's a bool yet here we expect a float. + edges = nut.snap_endpoints_lines( + edges, snapping_threshold, id_name, self.crs + ) + logging.info( + "Function [snap_endpoints_lines]: executed with threshold = {}".format( + snapping_threshold + ) + ) + + # merge merged lines if there are any merged lines + if not lines_merged.empty: + # save the merged lines to a shapefile - CHECK if there are lines merged that should not be merged (e.g. main + secondary road) + lines_merged.set_geometry( + col="geometry", inplace=True + ) # To ensure the object is a GeoDataFrame and not a Series + _emerged_lines_file = output_graph_dir.joinpath( + f"{project_name}_lines_that_merged.shp" + ) + lines_merged.to_file(_emerged_lines_file) + logging.info( + "Function [edges_to_shp]: saved at {}".format(_emerged_lines_file) + ) + + graph_complex, edges_complex = self._get_complex_graph_and_edges(edges, id_name) + + if not math.isnan(segmentation_length): + edges_complex = Segmentation(edges_complex, segmentation_length) + edges_complex = edges_complex.apply_segmentation() + if edges_complex.crs is None: # The CRS might have dissapeared. + edges_complex.crs = self.crs # set the right CRS + return graph_complex, edges_complex diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py index 7a751658f..977028243 100644 --- a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py @@ -1,6 +1,5 @@ import logging from pathlib import Path -from typing import Union import networkx as nx import pandas as pd From 7f31f43af4ad772883b4a01e2252c2559cee2baa Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 14:06:38 +0200 Subject: [PATCH 06/26] chore: Small corrections in the shp network wrapper --- ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py index 8dda060b1..9781d1e7e 100644 --- a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py @@ -145,7 +145,14 @@ def _get_complex_graph_and_edges( logging.info("Function [graph_from_gdf]: executed") return graph_complex, edges_complex - def get_network(self, merge_lines: bool, snapping_threshold: bool): + def get_network( + self, + merge_lines: bool, + snapping_threshold: bool, + output_graph_dir: Path, + project_name: str, + segmentation_length: float, + ): lines = self.read_merge_shp() # Check which of the lines are merged, also for the fid. The fid of the first line with a traffic count is taken. From eaacc3eb8a2fc65b0853cd73e1e769fce80d5d71 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 15:59:58 +0200 Subject: [PATCH 07/26] chore: Added network wrapper protocol; Adapted tests --- .../network_config_data.py | 10 +- .../network_config_data_reader.py | 47 ++++++++-- ra2ce/graph/network_wrapper_protocol.py | 15 +++ ra2ce/graph/networks.py | 91 ++++--------------- .../osm_network_wrapper.py | 47 +++++++++- .../shp_network_wrapper.py | 64 ++++++------- .../vector_network_wrapper.py | 5 +- .../test_osm_network_wrapper.py | 11 ++- tests/graph/shp_network_Wrapper/__init__.py | 0 .../test_shp_network_wrapper.py | 20 ++++ .../test_vector_network_wrapper.py | 4 +- 11 files changed, 194 insertions(+), 120 deletions(-) create mode 100644 ra2ce/graph/network_wrapper_protocol.py create mode 100644 tests/graph/shp_network_Wrapper/__init__.py create mode 100644 tests/graph/shp_network_Wrapper/test_shp_network_wrapper.py rename tests/graph/{ => shp_network_Wrapper}/test_vector_network_wrapper.py (96%) diff --git a/ra2ce/graph/network_config_data/network_config_data.py b/ra2ce/graph/network_config_data/network_config_data.py index e0f826753..0bdf36641 100644 --- a/ra2ce/graph/network_config_data/network_config_data.py +++ b/ra2ce/graph/network_config_data/network_config_data.py @@ -37,8 +37,8 @@ class ProjectSection: class NetworkSection: directed: bool = False source: str = "" # should be enum - primary_file: str = "" # TODO. Unclear whether this is `Path` or `list[Path]` - diversion_file: str = "" # TODO. Unclear whether this is `Path` or `list[Path]` + primary_file: list[Path] = field(default_factory=list) + diversion_file: list[Path] = field(default_factory=list) file_id: str = "" polygon: str = "" # TODO. Unclear whether this is `str`` or `Path` network_type: str = "" # Should be enum @@ -106,6 +106,12 @@ def output_graph_dir(self) -> Optional[Path]: return None return self.static_path.joinpath("output_graph") + @property + def network_dir(self) -> Optional[Path]: + if not self.static_path: + return None + return self.static_path.joinpath("network") + def to_dict(self) -> dict: _dict = self.__dict__ _dict["project"] = self.project.__dict__ diff --git a/ra2ce/graph/network_config_data/network_config_data_reader.py b/ra2ce/graph/network_config_data/network_config_data_reader.py index 14ce57685..2a4285e23 100644 --- a/ra2ce/graph/network_config_data/network_config_data_reader.py +++ b/ra2ce/graph/network_config_data/network_config_data_reader.py @@ -1,6 +1,6 @@ from configparser import ConfigParser from pathlib import Path -from typing import Union +from typing import Any, Union from ra2ce.common.configuration.ini_configuration_reader_protocol import ( ConfigDataReaderProtocol, @@ -58,6 +58,15 @@ def _select_to_correct(path_value: Union[list[Path], Path]) -> bool: return _select_to_correct(path_value[0]) return not path_value.exists() + def _correct_list(path_root: Path, path_value_list: list[Path]) -> list[Path]: + _corrected_list = [] + for _path_value in path_value_list: + if not _path_value.exists(): + _corrected_list.append(path_root.joinpath(_path_value)) + else: + _corrected_list.append(_path_value) + return _corrected_list + # Relative to network directory. _network_directory = config_data.static_path.joinpath("network") if _select_to_correct(config_data.origins_destinations.origins): @@ -70,15 +79,23 @@ def _select_to_correct(path_value: Union[list[Path], Path]) -> bool: config_data.origins_destinations.destinations ) + if _select_to_correct(config_data.origins_destinations.region): + config_data.origins_destinations.region = _network_directory.joinpath( + config_data.origins_destinations.region + ) + + config_data.network.primary_file = _correct_list( + _network_directory, config_data.network.primary_file + ) + config_data.network.diversion_file = _correct_list( + _network_directory, config_data.network.diversion_file + ) + # Relative to hazard directory. _hazard_directory = config_data.static_path.joinpath("hazard") - if _select_to_correct(config_data.hazard.hazard_map): - config_data.hazard.hazard_map = list( - map( - lambda x: _hazard_directory.joinpath(x), - config_data.hazard.hazard_map, - ) - ) + config_data.hazard.hazard_map = _correct_list( + _hazard_directory, config_data.hazard.hazard_map + ) def _get_str_as_path(self, str_value: Union[str, Path]) -> Path: if str_value and not isinstance(str_value, Path): @@ -107,9 +124,23 @@ def _get_sections(self) -> dict: def get_project_section(self) -> ProjectSection: return ProjectSection(**self._parser["project"]) + def _get_path_list( + self, section_name: str, property: str, fallback_opt: Any + ) -> list[Path]: + _value_list = self._parser.getlist( + section_name, property, fallback=fallback_opt + ) + return list(map(self._get_str_as_path, _value_list)) + def get_network_section(self) -> NetworkSection: _section = "network" _network_section = NetworkSection(**self._parser[_section]) + _network_section.primary_file = self._get_path_list( + _section, "primary_file", _network_section.primary_file + ) + _network_section.diversion_file = self._get_path_list( + _section, "diversion_file", _network_section.diversion_file + ) _network_section.directed = self._parser.getboolean( _section, "directed", fallback=_network_section.directed ) diff --git a/ra2ce/graph/network_wrapper_protocol.py b/ra2ce/graph/network_wrapper_protocol.py new file mode 100644 index 000000000..6590fc558 --- /dev/null +++ b/ra2ce/graph/network_wrapper_protocol.py @@ -0,0 +1,15 @@ +from typing import Protocol, runtime_checkable +from geopandas import GeoDataFrame +from networkx import MultiGraph + + +@runtime_checkable +class NetworkWrapperProtocol(Protocol): + def get_network(self, **kwargs) -> tuple[MultiGraph, GeoDataFrame]: + """ + Gets a network built within this wrapper instance. + + Returns: + tuple[MultiGraph, GeoDataFrame]: Tuple of MultiGraph representing the graph and GeoDataFrame representing the network. + """ + pass diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 9c839a4b4..5abb254e6 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -20,10 +20,7 @@ """ import logging -import math -import os -from pathlib import Path -from typing import Any, List, Tuple +from typing import Any import geopandas as gpd import networkx as nx @@ -32,7 +29,6 @@ from ra2ce.common.io.readers import GraphPickleReader from ra2ce.graph import networks_utils as nut -from ra2ce.graph.exporters.json_exporter import JsonExporter from ra2ce.graph.exporters.network_exporter_factory import NetworkExporterFactory from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper @@ -54,12 +50,12 @@ class Network: def __init__(self, network_config: NetworkConfigData, files: dict): # General self.project_name = network_config.project.name - self.output_graph_dir = network_config.static_path.joinpath("output_graph") + self.output_graph_dir = network_config.output_graph_dir if not self.output_graph_dir.is_dir(): self.output_graph_dir.mkdir(parents=True) # Network - self._network_dir = network_config.static_path.joinpath("network") + self._network_dir = network_config.network_dir self.base_graph_crs = None # Initiate variable self.base_network_crs = None # Initiate variable @@ -76,13 +72,7 @@ def __init__(self, network_config: NetworkConfigData, files: dict): ) self.origin_count = _origins_destinations.origin_count self.od_category = _origins_destinations.category - self.region = ( - None - if not _origins_destinations.region - else network_config.static_path.joinpath( - "network", _origins_destinations.region - ) - ) + self.region = _origins_destinations.region self.region_var = _origins_destinations.region_var # Cleanup @@ -91,9 +81,6 @@ def __init__(self, network_config: NetworkConfigData, files: dict): # files self.files = files - def _get_shp_paths(self, shp_str: str) -> Path: - return self._network_dir.joinpath(shp_str) - def _create_network_from_shp( self, ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: @@ -104,7 +91,7 @@ def _create_network_from_shp( def network_shp( self, crs: int = 4326 - ) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """Creates a (graph) network from a shapefile. Returns the same geometries for the network (GeoDataFrame) as for the graph (NetworkX graph), because @@ -119,22 +106,14 @@ def network_shp( """ # Make a pyproj CRS from the EPSG code _shp_network_wrapper = ShpNetworkWrapper( - primary_files=list( - map(self._get_shp_paths, self._network_config.primary_file.split(",")) - ), - diversion_files=list( - map(self._get_shp_paths, self._network_config.diversion_file.split(",")) - ), - region_path=None, + network_options=self._network_config, + cleanup_options=self._cleanup, + region_path=self.region, crs_value=crs, - is_directed=self._network_config.directed, ) graph_complex, edges_complex = _shp_network_wrapper.get_network( - self._cleanup.merge_lines, - self._cleanup.snapping_threshold, self.output_graph_dir, self.project_name, - self._cleanup.segmentation_length, ) self.base_graph_crs = pyproj.CRS.from_user_input(crs) @@ -143,7 +122,7 @@ def network_shp( # Exporting complex graph because the shapefile should be kept the same as much as possible. return graph_complex, edges_complex - def network_cleanshp(self) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + def network_cleanshp(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """Creates a (graph) network from a clean shapefile (primary_file - no further advance cleanup is needed) Returns the same geometries for the network (GeoDataFrame) as for the graph (NetworkX graph), because @@ -155,9 +134,7 @@ def network_cleanshp(self) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """ # initialise vector network wrapper vector_network_wrapper = VectorNetworkWrapper( - list( - map(self._get_shp_paths, self._network_config.primary_file.split(",")) - ), + self._network_config.primary_file, self.region, "", self._network_config.directed, @@ -167,7 +144,7 @@ def network_cleanshp(self) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: ( graph_complex, edges_complex, - ) = vector_network_wrapper.get_network_from_vector() + ) = vector_network_wrapper.get_network() # Set the CRS of the graph and network to wrapper crs self.base_graph_crs = vector_network_wrapper.crs @@ -175,18 +152,9 @@ def network_cleanshp(self) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: return graph_complex, edges_complex - def _export_linking_tables(self, linking_tables: List[Any]) -> None: - _exporter = JsonExporter() - _exporter.export( - self.output_graph_dir.joinpath("simple_to_complex.json"), linking_tables[0] - ) - _exporter.export( - self.output_graph_dir.joinpath("complex_to_simple.json"), linking_tables[1] - ) - def network_trails_import( self, crs: int = 4326 - ) -> Tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """Creates a network which has been prepared in the TRAILS package #Todo: we might later simply import the whole trails code as a package, and directly use these functions @@ -210,13 +178,11 @@ def network_trails_import( # Make a pyproj CRS from the EPSG code crs = pyproj.CRS.from_user_input(crs) - edge_file = self._network_dir.joinpath(self._network_config.primary_file) + edge_file = self._network_config.primary_file[0] edges = gpd.read_feather(edge_file) edges = edges.set_crs(crs) - corresponding_node_file = self._network_dir.joinpath( - self._network_config.primary_file.replace("edges", "nodes") - ) + corresponding_node_file = edge_file.replace("edges", "nodes") assert ( corresponding_node_file.exists() ), "The node file could not be found while importing from TRAILS" @@ -268,29 +234,11 @@ def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame road_types=self._network_config.road_types, graph_crs="", polygon_path=self._network_dir.joinpath(self._network_config.polygon), + directed=self._network_config.directed, ) - graph_complex = osm_network.get_clean_graph_from_osm() - - # Create 'graph_simple' - graph_simple, graph_complex, link_tables = nut.create_simplified_graph( - graph_complex - ) - - # Create 'edges_complex', convert complex graph to geodataframe - logging.info("Start converting the graph to a geodataframe") - edges_complex, node_complex = nut.graph_to_gdf(graph_complex) - logging.info("Finished converting the graph to a geodataframe") - - # Save the link tables linking complex and simple IDs - self._export_linking_tables(link_tables) - - # If the user wants to use undirected graphs, turn into an undirected graph (default). - if not self._network_config.directed: - if type(graph_simple) == nx.classes.multidigraph.MultiDiGraph: - graph_simple = graph_simple.to_undirected() + graph_simple, edges_complex = osm_network.get_network() # No segmentation required, the non-simplified road segments from OSM are already small enough - self.base_graph_crs = pyproj.CRS.from_user_input( "EPSG:4326" ) # Graphs from OSM download are always in this CRS. @@ -392,7 +340,7 @@ def get_avg_speed( return original_graph def _export_network_files( - self, network: Any, graph_name: str, types_to_export: List[str] + self, network: Any, graph_name: str, types_to_export: list[str] ): _exporter = NetworkExporterFactory() _exporter.export( @@ -448,6 +396,8 @@ def create(self) -> dict: base_graph = nut.add_missing_geoms_graph( base_graph, geom_name="geometry" ) + base_graph = self.get_avg_speed(base_graph) + elif self._network_config.source == "pickle": logging.info("Start importing a network from pickle") base_graph = GraphPickleReader().read( @@ -475,9 +425,6 @@ def create(self) -> dict: lambda x: nut.line_length(x, self.base_network_crs) ) - if self._network_config.source == "OSM download": - base_graph = self.get_avg_speed(base_graph) - # Save the graph and geodataframe self._export_network_files(base_graph, "base_graph", to_save) self._export_network_files(network_gdf, "base_network", to_save) diff --git a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py index 45950730a..0100c28e1 100644 --- a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py +++ b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py @@ -16,19 +16,22 @@ """ import logging from pathlib import Path +from typing import Any import networkx as nx import osmnx -from networkx import MultiDiGraph +from geopandas import GeoDataFrame +from networkx import MultiDiGraph, MultiGraph from osmnx import consolidate_intersections from shapely.geometry.base import BaseGeometry -from ra2ce.graph.network_config_data.network_config_data import NetworkSection +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.exporters.json_exporter import JsonExporter import ra2ce.graph.networks_utils as nut from ra2ce.graph.osm_network_wrapper.extremities_data import ExtremitiesData -class OsmNetworkWrapper: +class OsmNetworkWrapper(NetworkWrapperProtocol): network_type: str road_types: list[str] graph_crs: str @@ -40,15 +43,53 @@ def __init__( road_types: list[str], graph_crs: str, polygon_path: Path, + directed: bool, ) -> None: self.network_type = network_type self.road_types = road_types self.polygon_path = polygon_path + self.is_directed = directed self.graph_crs = graph_crs if not graph_crs: # Set default value self.graph_crs = "epsg:4326" + def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: + """ + Gets an indirected graph + + Returns: + tuple[MultiGraph, GeoDataFrame]: _description_ + """ + graph_complex = self.get_clean_graph_from_osm() + + # Create 'graph_simple' + graph_simple, graph_complex, link_tables = nut.create_simplified_graph( + graph_complex + ) + + # Create 'edges_complex', convert complex graph to geodataframe + logging.info("Start converting the graph to a geodataframe") + edges_complex, node_complex = nut.graph_to_gdf(graph_complex) + logging.info("Finished converting the graph to a geodataframe") + + # Save the link tables linking complex and simple IDs + self._export_linking_tables(link_tables) + + if not self.is_directed and isinstance(graph_simple, MultiDiGraph): + graph_simple = graph_simple.to_undirected() + + return graph_simple, edges_complex + + def _export_linking_tables(self, linking_tables: tuple[Any]) -> None: + _exporter = JsonExporter() + _exporter.export( + self.output_graph_dir.joinpath("simple_to_complex.json"), linking_tables[0] + ) + _exporter.export( + self.output_graph_dir.joinpath("complex_to_simple.json"), linking_tables[1] + ) + def get_clean_graph_from_osm(self) -> MultiDiGraph: """ Creates a network from a polygon by by downloading via the OSM API in its extent. diff --git a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py index 9781d1e7e..c937f4adf 100644 --- a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py @@ -24,6 +24,11 @@ from pyproj import CRS import geopandas as gpd import pandas as pd +from ra2ce.graph.network_config_data.network_config_data import ( + NetworkSection, + CleanupSection, +) +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol import ra2ce.graph.networks_utils as nut from shapely.geometry import MultiLineString import logging @@ -32,21 +37,13 @@ from ra2ce.graph.segmentation import Segmentation -class ShpNetworkWrapper: - primary_files: list[Path] - region_path: Path - crs: CRS - directed: bool - file_id: str - +class ShpNetworkWrapper(NetworkWrapperProtocol): def __init__( self, - primary_files: list[Path], - diversion_files: list[Path], + network_options: NetworkSection, + cleanup_options: CleanupSection, region_path: Path, crs_value: str, - is_directed: bool, - file_id: str, ) -> None: """Initializes the VectorNetworkWrapper object. @@ -57,14 +54,23 @@ def __init__( ValueError: If the config is None or doesn't contain a network dictionary, or if config['network'] is not a dictionary. """ - self.primary_files = primary_files - self.diversion_files = diversion_files + # Network options + self.primary_files = network_options.primary_files + self.diversion_files = network_options.diversion_files + self.directed = network_options.directed + self.file_id = network_options.file_id + + # Cleanup options + self.merge_lines = cleanup_options.merge_lines + self.snapping_threshold = cleanup_options.snapping_threshold + self.segmentation_length = cleanup_options.segmentation_length + self.cut_at_intersections = cleanup_options.cut_at_intersections + + # Other self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") self.region_path = region_path - self.directed = is_directed - self.file_id = file_id - def read_merge_shp(self) -> gpd.GeoDataFrame: + def _read_merge_shp(self) -> gpd.GeoDataFrame: """Imports shapefile(s) and saves attributes in a pandas dataframe. Returns: @@ -117,7 +123,7 @@ def _get_complex_graph_and_edges( self, edges: gpd.GeoDataFrame, id_name: str ) -> tuple[nx.MultiGraph, gpd.GeoDataFrame]: # Get the unique points at the end of lines and at intersections to create nodes - nodes = nut.create_nodes(edges, self.crs, self._cleanup.cut_at_intersections) + nodes = nut.create_nodes(edges, self.crs, self.cut_at_intersections) logging.info("Function [create_nodes]: executed") edges = nut.cut_lines( @@ -147,21 +153,17 @@ def _get_complex_graph_and_edges( def get_network( self, - merge_lines: bool, - snapping_threshold: bool, output_graph_dir: Path, project_name: str, - segmentation_length: float, - ): - lines = self.read_merge_shp() - + ) -> tuple[nx.MultiGraph, gpd.GeoDataFrame]: + edges = self._read_merge_shp() + lines_merged = gpd.GeoDataFrame() # Check which of the lines are merged, also for the fid. The fid of the first line with a traffic count is taken. # The list of fid's is reduced by the fid's that are not anymore in the merged lines - edges, lines_merged = lines, gpd.GeoDataFrame() - if merge_lines: + if self.merge_lines: aadt_names = [] edges, lines_merged = nut.merge_lines_automatic( - lines, self.file_id, aadt_names, self.crs + edges, self.file_id, aadt_names, self.crs ) logging.info( "Function [merge_lines_automatic]: executed with properties {}".format( @@ -171,14 +173,14 @@ def get_network( edges, id_name = nut.gdf_check_create_unique_ids(edges, self.file_id) - if snapping_threshold: + if self.snapping_threshold: # TODO: snapping threshold it's a bool yet here we expect a float. edges = nut.snap_endpoints_lines( - edges, snapping_threshold, id_name, self.crs + edges, self.snapping_threshold, id_name, self.crs ) logging.info( "Function [snap_endpoints_lines]: executed with threshold = {}".format( - snapping_threshold + self.snapping_threshold ) ) @@ -198,8 +200,8 @@ def get_network( graph_complex, edges_complex = self._get_complex_graph_and_edges(edges, id_name) - if not math.isnan(segmentation_length): - edges_complex = Segmentation(edges_complex, segmentation_length) + if not math.isnan(self.segmentation_length): + edges_complex = Segmentation(edges_complex, self.segmentation_length) edges_complex = edges_complex.apply_segmentation() if edges_complex.crs is None: # The CRS might have dissapeared. edges_complex.crs = self.crs # set the right CRS diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py index 977028243..961e19a90 100644 --- a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py @@ -8,10 +8,11 @@ from shapely.geometry import Point from pyproj import CRS +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol import ra2ce.graph.networks_utils as nut -class VectorNetworkWrapper: +class VectorNetworkWrapper(NetworkWrapperProtocol): """A class for handling and manipulating vector files. Provides methods for reading vector data, cleaning it, and setting up graph and @@ -44,7 +45,7 @@ def __init__( self.region_path = region_path self.directed = is_directed - def get_network_from_vector( + def get_network( self, ) -> tuple[nx.MultiGraph, gpd.GeoDataFrame]: """Gets a network built from vector files. diff --git a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py index 4661f933f..fd0d58415 100644 --- a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py +++ b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py @@ -5,6 +5,7 @@ from networkx.utils import graphs_equal from shapely.geometry import LineString, Polygon from shapely.geometry.base import BaseGeometry +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data, slow_test import ra2ce.graph.networks_utils as nut @@ -13,8 +14,15 @@ class TestOsmNetworkWrapper: def test_initialize_without_graph_crs(self): - _wrapper = OsmNetworkWrapper("a_network", ["r"], "", Path()) + _wrapper = OsmNetworkWrapper( + network_type="a_network", + road_types=["r"], + graph_crs="", + polygon_path=Path(), + directed=False, + ) assert isinstance(_wrapper, OsmNetworkWrapper) + assert isinstance(_wrapper, NetworkWrapperProtocol) assert _wrapper.graph_crs == "epsg:4326" @pytest.fixture @@ -26,6 +34,7 @@ def _network_wrapper_without_polygon(self) -> OsmNetworkWrapper: road_types=_road_types, graph_crs="", polygon_path=None, + directed=True, ) def test_download_clean_graph_from_osm_with_invalid_polygon_arg( diff --git a/tests/graph/shp_network_Wrapper/__init__.py b/tests/graph/shp_network_Wrapper/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/graph/shp_network_Wrapper/test_shp_network_wrapper.py b/tests/graph/shp_network_Wrapper/test_shp_network_wrapper.py new file mode 100644 index 000000000..5ea75be8e --- /dev/null +++ b/tests/graph/shp_network_Wrapper/test_shp_network_wrapper.py @@ -0,0 +1,20 @@ +from ra2ce.graph.network_config_data.network_config_data import ( + NetworkSection, + CleanupSection, +) +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.shp_network_wrapper.shp_network_wrapper import ShpNetworkWrapper + + +class TestShpNetworkWrapper: + def test_init(self): + # 1.Define test data. + _network_options = NetworkSection() + _cleanup_options = CleanupSection() + + _wrapper = ShpNetworkWrapper(_network_options, _cleanup_options, None, "") + + # Verify expectations. + assert isinstance(_wrapper, ShpNetworkWrapper) + assert isinstance(_wrapper, NetworkWrapperProtocol) + assert _wrapper.crs.to_epsg() == 4326 diff --git a/tests/graph/test_vector_network_wrapper.py b/tests/graph/shp_network_Wrapper/test_vector_network_wrapper.py similarity index 96% rename from tests/graph/test_vector_network_wrapper.py rename to tests/graph/shp_network_Wrapper/test_vector_network_wrapper.py index 51775280c..f7ce44841 100644 --- a/tests/graph/test_vector_network_wrapper.py +++ b/tests/graph/shp_network_Wrapper/test_vector_network_wrapper.py @@ -4,6 +4,7 @@ import geopandas as gpd import networkx as nx from shapely.geometry import LineString, Point, MultiLineString +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper @@ -49,6 +50,7 @@ def test_init_without_crs_sts_default(self): # 3. Verify expectations. assert isinstance(_wrapper, VectorNetworkWrapper) + assert isinstance(_wrapper, NetworkWrapperProtocol) assert _wrapper.primary_files == _primary_files assert _wrapper.region_path == _region assert str(_wrapper.crs) == "epsg:4326" @@ -104,7 +106,7 @@ def test_get_network_from_vector( _valid_wrapper.region_path = region_path # When - graph, edges = _valid_wrapper.get_network_from_vector() + graph, edges = _valid_wrapper.get_network() # Then assert isinstance(graph, nx.MultiGraph) From 7ba424335acc66e757a61429ca9f15abfc4906e8 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 16:00:49 +0200 Subject: [PATCH 08/26] test: Renamed test module --- .../{shp_network_Wrapper => shp_network_wrapper}/__init__.py | 0 .../test_shp_network_wrapper.py | 0 .../test_vector_network_wrapper.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/graph/{shp_network_Wrapper => shp_network_wrapper}/__init__.py (100%) rename tests/graph/{shp_network_Wrapper => shp_network_wrapper}/test_shp_network_wrapper.py (100%) rename tests/graph/{shp_network_Wrapper => shp_network_wrapper}/test_vector_network_wrapper.py (100%) diff --git a/tests/graph/shp_network_Wrapper/__init__.py b/tests/graph/shp_network_wrapper/__init__.py similarity index 100% rename from tests/graph/shp_network_Wrapper/__init__.py rename to tests/graph/shp_network_wrapper/__init__.py diff --git a/tests/graph/shp_network_Wrapper/test_shp_network_wrapper.py b/tests/graph/shp_network_wrapper/test_shp_network_wrapper.py similarity index 100% rename from tests/graph/shp_network_Wrapper/test_shp_network_wrapper.py rename to tests/graph/shp_network_wrapper/test_shp_network_wrapper.py diff --git a/tests/graph/shp_network_Wrapper/test_vector_network_wrapper.py b/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py similarity index 100% rename from tests/graph/shp_network_Wrapper/test_vector_network_wrapper.py rename to tests/graph/shp_network_wrapper/test_vector_network_wrapper.py From ec4129c1dd7b0a25e5ce295fbb06a0d8f90a19dd Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 16:15:08 +0200 Subject: [PATCH 09/26] chore: Fixed shp_network_wrapper --- ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py index c937f4adf..5bb833a5e 100644 --- a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py @@ -55,8 +55,8 @@ def __init__( or if config['network'] is not a dictionary. """ # Network options - self.primary_files = network_options.primary_files - self.diversion_files = network_options.diversion_files + self.primary_files = network_options.primary_file + self.diversion_files = network_options.diversion_file self.directed = network_options.directed self.file_id = network_options.file_id From f0e54b6f47c56d6030621d259fbe4a0d24539a2f Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 16:37:29 +0200 Subject: [PATCH 10/26] chore: Added missing property --- ra2ce/graph/networks.py | 1 + ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py | 2 ++ .../graph/osm_network_wrapper/test_osm_network_wrapper.py | 7 ++++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 5abb254e6..537457944 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -235,6 +235,7 @@ def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame graph_crs="", polygon_path=self._network_dir.joinpath(self._network_config.polygon), directed=self._network_config.directed, + output_graph_dir=self.output_graph_dir, ) graph_simple, edges_complex = osm_network.get_network() diff --git a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py index 0100c28e1..1921aa9e4 100644 --- a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py +++ b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py @@ -44,12 +44,14 @@ def __init__( graph_crs: str, polygon_path: Path, directed: bool, + output_graph_dir: Path, ) -> None: self.network_type = network_type self.road_types = road_types self.polygon_path = polygon_path self.is_directed = directed self.graph_crs = graph_crs + self.output_graph_dir = output_graph_dir if not graph_crs: # Set default value self.graph_crs = "epsg:4326" diff --git a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py index fd0d58415..7dfa6c6b9 100644 --- a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py +++ b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py @@ -7,7 +7,7 @@ from shapely.geometry.base import BaseGeometry from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol -from tests import test_data, slow_test +from tests import test_data, slow_test, test_results import ra2ce.graph.networks_utils as nut from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper @@ -20,6 +20,7 @@ def test_initialize_without_graph_crs(self): graph_crs="", polygon_path=Path(), directed=False, + output_graph_dir=test_results.joinpath("test_osm_network_wrapper") ) assert isinstance(_wrapper, OsmNetworkWrapper) assert isinstance(_wrapper, NetworkWrapperProtocol) @@ -29,12 +30,16 @@ def test_initialize_without_graph_crs(self): def _network_wrapper_without_polygon(self) -> OsmNetworkWrapper: _network_type = "drive" _road_types = ["road_link"] + _output_dir = test_results.joinpath("test_osm_network_wrapper") + if not _output_dir.exists(): + _output_dir.mkdir(parents=True) yield OsmNetworkWrapper( network_type=_network_type, road_types=_road_types, graph_crs="", polygon_path=None, directed=True, + output_graph_dir=_output_dir ) def test_download_clean_graph_from_osm_with_invalid_polygon_arg( From 8820c16967c630a00d8572d55429dfcbdc670f44 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 17:15:46 +0200 Subject: [PATCH 11/26] chore: Further reworks to bring more logic towards osm_network_wrapper --- .../network_config_data.py | 2 +- .../network_config_data_reader.py | 5 +++ ra2ce/graph/networks.py | 41 +------------------ .../osm_network_wrapper.py | 40 +++++++++++++++--- 4 files changed, 41 insertions(+), 47 deletions(-) diff --git a/ra2ce/graph/network_config_data/network_config_data.py b/ra2ce/graph/network_config_data/network_config_data.py index 0bdf36641..deebbce90 100644 --- a/ra2ce/graph/network_config_data/network_config_data.py +++ b/ra2ce/graph/network_config_data/network_config_data.py @@ -40,7 +40,7 @@ class NetworkSection: primary_file: list[Path] = field(default_factory=list) diversion_file: list[Path] = field(default_factory=list) file_id: str = "" - polygon: str = "" # TODO. Unclear whether this is `str`` or `Path` + polygon: Optional[Path] = None network_type: str = "" # Should be enum road_types: list[str] = field(default_factory=list) save_shp: bool = False diff --git a/ra2ce/graph/network_config_data/network_config_data_reader.py b/ra2ce/graph/network_config_data/network_config_data_reader.py index 2a4285e23..cbd5197d1 100644 --- a/ra2ce/graph/network_config_data/network_config_data_reader.py +++ b/ra2ce/graph/network_config_data/network_config_data_reader.py @@ -84,6 +84,11 @@ def _correct_list(path_root: Path, path_value_list: list[Path]) -> list[Path]: config_data.origins_destinations.region ) + if _select_to_correct(config_data.network.polygon): + config_data.network.polygon = _network_directory.joinpath( + config_data.network.polygon + ) + config_data.network.primary_file = _correct_list( _network_directory, config_data.network.primary_file ) diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 537457944..0ff1ce9e1 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -24,7 +24,6 @@ import geopandas as gpd import networkx as nx -import pandas as pd import pyproj from ra2ce.common.io.readers import GraphPickleReader @@ -233,7 +232,7 @@ def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame network_type=self._network_config.network_type, road_types=self._network_config.road_types, graph_crs="", - polygon_path=self._network_dir.joinpath(self._network_config.polygon), + polygon_path=self._network_config.polygon, directed=self._network_config.directed, output_graph_dir=self.output_graph_dir, ) @@ -309,37 +308,6 @@ def generate_origins_from_raster(self): return out_fn - def get_avg_speed( - self, original_graph: nx.classes.graph.Graph - ) -> nx.classes.graph.Graph: - if all(["length" in e for u, v, e in original_graph.edges.data()]) and any( - ["maxspeed" in e for u, v, e in original_graph.edges.data()] - ): - # Add time weighing - Define and assign average speeds; or take the average speed from an existing CSV - path_avg_speed = self.output_graph_dir.joinpath("avg_speed.csv") - if path_avg_speed.is_file(): - avg_speeds = pd.read_csv(path_avg_speed) - else: - avg_speeds = nut.calc_avg_speed( - original_graph, - "highway", - save_csv=True, - save_path=path_avg_speed, - ) - original_graph = nut.assign_avg_speed(original_graph, avg_speeds, "highway") - - # make a time value of seconds, length of road streches is in meters - for u, v, k, edata in original_graph.edges.data(keys=True): - hours = (edata["length"] / 1000) / edata["avgspeed"] - original_graph[u][v][k]["time"] = round(hours * 3600, 0) - - return original_graph - else: - logging.info( - "No attributes found in the graph to estimate average speed per network segment." - ) - return original_graph - def _export_network_files( self, network: Any, graph_name: str, types_to_export: list[str] ): @@ -392,13 +360,6 @@ def create(self) -> dict: elif self._network_config.source == "OSM download": logging.info("Start downloading a network from OSM.") base_graph, network_gdf = self.network_osm_download() - # Graph & Network from OSM download - # Check if all geometries between nodes are there, if not, add them as a straight line. - base_graph = nut.add_missing_geoms_graph( - base_graph, geom_name="geometry" - ) - base_graph = self.get_avg_speed(base_graph) - elif self._network_config.source == "pickle": logging.info("Start importing a network from pickle") base_graph = GraphPickleReader().read( diff --git a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py index 1921aa9e4..a59cfd211 100644 --- a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py +++ b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py @@ -26,17 +26,12 @@ from shapely.geometry.base import BaseGeometry from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from ra2ce.graph.exporters.json_exporter import JsonExporter - +import pandas as pd import ra2ce.graph.networks_utils as nut from ra2ce.graph.osm_network_wrapper.extremities_data import ExtremitiesData class OsmNetworkWrapper(NetworkWrapperProtocol): - network_type: str - road_types: list[str] - graph_crs: str - polygon_path: Path - def __init__( self, network_type: str, @@ -81,8 +76,41 @@ def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: if not self.is_directed and isinstance(graph_simple, MultiDiGraph): graph_simple = graph_simple.to_undirected() + # Check if all geometries between nodes are there, if not, add them as a straight line. + graph_simple = nut.add_missing_geoms_graph(graph_simple, geom_name="geometry") + graph_simple = self._get_avg_speed(graph_simple) return graph_simple, edges_complex + def _get_avg_speed( + self, original_graph: nx.classes.graph.Graph + ) -> nx.classes.graph.Graph: + if all(["length" in e for u, v, e in original_graph.edges.data()]) and any( + ["maxspeed" in e for u, v, e in original_graph.edges.data()] + ): + # Add time weighing - Define and assign average speeds; or take the average speed from an existing CSV + path_avg_speed = self.output_graph_dir.joinpath("avg_speed.csv") + if path_avg_speed.is_file(): + avg_speeds = pd.read_csv(path_avg_speed) + else: + avg_speeds = nut.calc_avg_speed( + original_graph, + "highway", + save_csv=True, + save_path=path_avg_speed, + ) + original_graph = nut.assign_avg_speed(original_graph, avg_speeds, "highway") + + # make a time value of seconds, length of road streches is in meters + for u, v, k, edata in original_graph.edges.data(keys=True): + hours = (edata["length"] / 1000) / edata["avgspeed"] + original_graph[u][v][k]["time"] = round(hours * 3600, 0) + + return original_graph + logging.info( + "No attributes found in the graph to estimate average speed per network segment." + ) + return original_graph + def _export_linking_tables(self, linking_tables: tuple[Any]) -> None: _exporter = JsonExporter() _exporter.export( From 3eb0bc8312af234da884749d78099c5a210b1fc2 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Thu, 27 Jul 2023 20:59:45 +0200 Subject: [PATCH 12/26] chore: Updated reading method of network.polygon attribute --- ra2ce/graph/network_config_data/network_config_data_reader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ra2ce/graph/network_config_data/network_config_data_reader.py b/ra2ce/graph/network_config_data/network_config_data_reader.py index cbd5197d1..8cc0347e6 100644 --- a/ra2ce/graph/network_config_data/network_config_data_reader.py +++ b/ra2ce/graph/network_config_data/network_config_data_reader.py @@ -155,6 +155,7 @@ def get_network_section(self) -> NetworkSection: _network_section.road_types = self._parser.getlist( _section, "road_types", fallback=_network_section.road_types ) + _network_section.polygon = self._get_str_as_path(_network_section.polygon) return _network_section def get_origins_destinations_section(self) -> OriginsDestinationsSection: From f49bd666dfd191c0431b96f8740cd24cc82fb41d Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 09:21:57 +0200 Subject: [PATCH 13/26] chore: Modified wrappers to use the network section for simplicity --- ra2ce/graph/networks.py | 12 +++------ .../osm_network_wrapper.py | 23 +++++++--------- .../vector_network_wrapper.py | 13 +++------- .../test_osm_network_wrapper.py | 26 ++++++++++--------- .../test_vector_network_wrapper.py | 14 +++++++--- 5 files changed, 43 insertions(+), 45 deletions(-) diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 0ff1ce9e1..9dd9aea82 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -133,10 +133,9 @@ def network_cleanshp(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """ # initialise vector network wrapper vector_network_wrapper = VectorNetworkWrapper( - self._network_config.primary_file, - self.region, - "", - self._network_config.directed, + network_data=self._network_config, + region_path=self.region, + crs_value="", ) # setup network using the wrapper @@ -229,11 +228,8 @@ def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: Tuple of Simplified graph (for use in the indirect analyses) and Complex graph (for use in the direct analyses). """ osm_network = OsmNetworkWrapper( - network_type=self._network_config.network_type, - road_types=self._network_config.road_types, + network_data=self._network_config, graph_crs="", - polygon_path=self._network_config.polygon, - directed=self._network_config.directed, output_graph_dir=self.output_graph_dir, ) graph_simple, edges_complex = osm_network.get_network() diff --git a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py index a59cfd211..171bb8eb9 100644 --- a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py +++ b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py @@ -24,6 +24,9 @@ from networkx import MultiDiGraph, MultiGraph from osmnx import consolidate_intersections from shapely.geometry.base import BaseGeometry +from ra2ce.graph.network_config_data.network_config_data import ( + NetworkSection, +) from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from ra2ce.graph.exporters.json_exporter import JsonExporter import pandas as pd @@ -34,22 +37,16 @@ class OsmNetworkWrapper(NetworkWrapperProtocol): def __init__( self, - network_type: str, - road_types: list[str], - graph_crs: str, - polygon_path: Path, - directed: bool, + network_data: NetworkSection, output_graph_dir: Path, + graph_crs: str, ) -> None: - self.network_type = network_type - self.road_types = road_types - self.polygon_path = polygon_path - self.is_directed = directed - self.graph_crs = graph_crs + self.network_type = network_data.network_type + self.road_types = network_data.road_types + self.polygon_path = network_data.polygon + self.is_directed = network_data.directed self.output_graph_dir = output_graph_dir - if not graph_crs: - # Set default value - self.graph_crs = "epsg:4326" + self.graph_crs = graph_crs if graph_crs else "epsg:4326" def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: """ diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py index 961e19a90..7711b7ee0 100644 --- a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py @@ -8,6 +8,7 @@ from shapely.geometry import Point from pyproj import CRS +from ra2ce.graph.network_config_data.network_config_data import NetworkSection from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol import ra2ce.graph.networks_utils as nut @@ -19,17 +20,11 @@ class VectorNetworkWrapper(NetworkWrapperProtocol): network. """ - primary_files: list[Path] - region_path: Path - crs: CRS - directed: bool - def __init__( self, - primary_files: list[Path], + network_data: NetworkSection, region_path: Path, crs_value: str, - is_directed: bool, ) -> None: """Initializes the VectorNetworkWrapper object. @@ -40,10 +35,10 @@ def __init__( ValueError: If the config is None or doesn't contain a network dictionary, or if config['network'] is not a dictionary. """ - self.primary_files = primary_files + self.primary_files = network_data.primary_file + self.directed = network_data.directed self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") self.region_path = region_path - self.directed = is_directed def get_network( self, diff --git a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py index 7dfa6c6b9..62dd12bce 100644 --- a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py +++ b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py @@ -5,6 +5,7 @@ from networkx.utils import graphs_equal from shapely.geometry import LineString, Polygon from shapely.geometry.base import BaseGeometry +from ra2ce.graph.network_config_data.network_config_data import NetworkSection from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data, slow_test, test_results @@ -14,32 +15,33 @@ class TestOsmNetworkWrapper: def test_initialize_without_graph_crs(self): + # 1. Define test data. + _network_section = NetworkSection(network_type="a_network", road_types=["r"]) + + # 2. Run test. _wrapper = OsmNetworkWrapper( - network_type="a_network", - road_types=["r"], + network_data=_network_section, + output_graph_dir=test_results.joinpath("test_osm_network_wrapper"), graph_crs="", - polygon_path=Path(), - directed=False, - output_graph_dir=test_results.joinpath("test_osm_network_wrapper") ) + + # 3. Verify final expectations. assert isinstance(_wrapper, OsmNetworkWrapper) assert isinstance(_wrapper, NetworkWrapperProtocol) assert _wrapper.graph_crs == "epsg:4326" @pytest.fixture def _network_wrapper_without_polygon(self) -> OsmNetworkWrapper: - _network_type = "drive" - _road_types = ["road_link"] + _network_section = NetworkSection( + network_type="drive", road_types=["road_link"], directed=True + ) _output_dir = test_results.joinpath("test_osm_network_wrapper") if not _output_dir.exists(): _output_dir.mkdir(parents=True) yield OsmNetworkWrapper( - network_type=_network_type, - road_types=_road_types, + network_data=_network_section, + output_graph_dir=_output_dir, graph_crs="", - polygon_path=None, - directed=True, - output_graph_dir=_output_dir ) def test_download_clean_graph_from_osm_with_invalid_polygon_arg( diff --git a/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py b/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py index f7ce44841..8c795d5cf 100644 --- a/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py +++ b/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py @@ -4,6 +4,7 @@ import geopandas as gpd import networkx as nx from shapely.geometry import LineString, Point, MultiLineString +from ra2ce.graph.network_config_data.network_config_data import NetworkSection from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data @@ -41,12 +42,17 @@ def mock_graph(self): def test_init_without_crs_sts_default(self): # 1. Define test data. + _network_data = NetworkSection( + primary_file=[Path("dummy_primary")], directed=False + ) _primary_files = [Path("dummy_primary")] _region = Path("dummy_region") _crs_value = "" # 2. Run test. - _wrapper = VectorNetworkWrapper(_primary_files, _region, _crs_value, False) + _wrapper = VectorNetworkWrapper( + network_data=_network_data, region_path=_region, crs_value=_crs_value + ) # 3. Verify expectations. assert isinstance(_wrapper, VectorNetworkWrapper) @@ -58,11 +64,13 @@ def test_init_without_crs_sts_default(self): @pytest.fixture def _valid_wrapper(self) -> VectorNetworkWrapper: _network_dir = _test_dir.joinpath("static", "network") + _network_data = NetworkSection( + primary_file=[_network_dir.joinpath("_test_lines.geojson")], directed=False + ) yield VectorNetworkWrapper( - primary_files=[_network_dir.joinpath("_test_lines.geojson")], + network_data=_network_data, region_path=None, crs_value=4326, - is_directed=False, ) def test_read_vector_to_project_region_and_crs( From de9b578c038861e26c8827933149e2d53606e81e Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 09:49:55 +0200 Subject: [PATCH 14/26] feat: Extracted logic to generate networks from different sources. --- ra2ce/graph/networks.py | 94 +++------------ .../vector_network_wrapper.py | 21 ++++ .../graph/trails_network_wrapper/__init__.py | 0 .../trails_network_wrapper.py | 107 ++++++++++++++++++ 4 files changed, 142 insertions(+), 80 deletions(-) create mode 100644 ra2ce/graph/trails_network_wrapper/__init__.py create mode 100644 ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 9dd9aea82..d4a9a63a1 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -31,9 +31,11 @@ from ra2ce.graph.exporters.network_exporter_factory import NetworkExporterFactory from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper -from ra2ce.graph.segmentation import Segmentation from ra2ce.graph.shp_network_wrapper.shp_network_wrapper import ShpNetworkWrapper from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper +from ra2ce.graph.trails_network_wrapper.trails_network_wrapper import ( + TrailsNetworkWrapper, +) class Network: @@ -80,6 +82,14 @@ def __init__(self, network_config: NetworkConfigData, files: dict): # files self.files = files + def _any_cleanup_enabled(self) -> bool: + return ( + self._cleanup.snapping_threshold + or self._cleanup.pruning_threshold + or self._cleanup.merge_lines + or self._cleanup.cut_at_intersections + ) + def _create_network_from_shp( self, ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: @@ -150,76 +160,6 @@ def network_cleanshp(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: return graph_complex, edges_complex - def network_trails_import( - self, crs: int = 4326 - ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - """Creates a network which has been prepared in the TRAILS package - - #Todo: we might later simply import the whole trails code as a package, and directly use these functions - #Todo: because TRAILS is still in beta version we better wait with that untill the first stable version is - # released - - Returns: - graph_simple (NetworkX graph): Simplified graph (for use in the indirect analyses). - complex_edges (GeoDataFrame): Complex graph (for use in the direct analyses). - """ - - logging.info( - "TRAILS importer: Reads the provided primary edge file: {}, assumes there also is a_nodes file".format( - self._network_config.primary_file - ) - ) - - logging.warning( - "Any coordinate projection information in the feather file will be overwritten (with default WGS84)" - ) - # Make a pyproj CRS from the EPSG code - crs = pyproj.CRS.from_user_input(crs) - - edge_file = self._network_config.primary_file[0] - edges = gpd.read_feather(edge_file) - edges = edges.set_crs(crs) - - corresponding_node_file = edge_file.replace("edges", "nodes") - assert ( - corresponding_node_file.exists() - ), "The node file could not be found while importing from TRAILS" - nodes = gpd.read_feather(corresponding_node_file) - nodes = nodes.set_crs(crs) - # nodes = pd.read_pickle( - # corresponding_node_file - # ) # Todo: Throw exception if nodes file is not present - - logging.info("TRAILS importer: start generating graph") - # tempfix to rename columns - edges = edges.rename({"from_id": "node_A", "to_id": "node_B"}, axis="columns") - node_id = "id" - graph_simple = nut.graph_from_gdf(edges, nodes, name="network", node_id=node_id) - - logging.info("TRAILS importer: graph generating was succesfull.") - logging.warning( - "RA2CE will not clean-up your graph, assuming that it is already done in TRAILS" - ) - - if self._cleanup.segmentation_length: - logging.info("TRAILS importer: start segmentating graph") - to_segment = Segmentation(edges, self._cleanup.segmentation_length) - edges_simple_segmented = to_segment.apply_segmentation() - if edges_simple_segmented.crs is None: # The CRS might have dissapeared. - edges_simple_segmented.crs = edges.crs # set the right CRS - edges_complex = edges_simple_segmented - - else: - edges_complex = edges - - graph_complex = graph_simple # NOTE THAT DIFFERENCE - # BETWEEN SIMPLE AND COMPLEX DOES NOT EXIST WHEN IMPORTING WITH TRAILS - - # Todo: better control over metadata in trails - # Todo: better control over where things are saved in the pipeline - - return graph_complex, edges_complex - def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """ Creates a network from a polygon by downloading via the OSM API in the extent of the polygon. @@ -316,14 +256,6 @@ def _export_network_files( ) self.files[graph_name] = _exporter.get_pickle_path() - def _any_cleanup_enabled(self) -> bool: - return ( - self._cleanup.snapping_threshold - or self._cleanup.pruning_threshold - or self._cleanup.merge_lines - or self._cleanup.cut_at_intersections - ) - def create(self) -> dict: """Handler function with the logic to call the right functions to create a network. @@ -349,7 +281,9 @@ def create(self) -> dict: ) # base_graph, network_gdf = self.network_osm_pbf() #The old approach is depreciated - base_graph, network_gdf = self.network_trails_import() + base_graph, network_gdf = TrailsNetworkWrapper( + network_data=self._network_config, crs_value=4326 + ).get_network() self.base_network_crs = network_gdf.crs diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py index 7711b7ee0..bec127fc4 100644 --- a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py @@ -1,3 +1,24 @@ +""" + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Risk Assessment and Adaptation for Critical Infrastructure (RA2CE). + Copyright (C) 2023 Stichting Deltares + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + import logging from pathlib import Path diff --git a/ra2ce/graph/trails_network_wrapper/__init__.py b/ra2ce/graph/trails_network_wrapper/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py b/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py new file mode 100644 index 000000000..9304c4219 --- /dev/null +++ b/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py @@ -0,0 +1,107 @@ +""" + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Risk Assessment and Adaptation for Critical Infrastructure (RA2CE). + Copyright (C) 2023 Stichting Deltares + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +from geopandas import GeoDataFrame +from networkx import MultiGraph +from ra2ce.graph.network_config_data.network_config_data import ( + CleanupSection, + NetworkSection, +) +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.networks_utils import graph_from_gdf +from ra2ce.graph.segmentation import Segmentation +from pyproj import CRS +import logging +import geopandas as gpd + + +class TrailsNetworkWrapper(NetworkWrapperProtocol): + def __init__( + self, + network_data: NetworkSection, + cleanup_section: CleanupSection, + crs_value: str, + ) -> None: + self.primary_files = network_data.primary_file + self.segmentation_length = cleanup_section.segmentation_length + self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") + + def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: + """Creates a network which has been prepared in the TRAILS package + + #Todo: we might later simply import the whole trails code as a package, and directly use these functions + #Todo: because TRAILS is still in beta version we better wait with that untill the first stable version is + # released + + Returns: + graph_simple (NetworkX graph): Simplified graph (for use in the indirect analyses). + complex_edges (GeoDataFrame): Complex graph (for use in the direct analyses). + """ + logging.info( + "TRAILS importer: Reads the provided primary edge file: {}, assumes there also is a_nodes file".format( + self.primary_files + ) + ) + + logging.warning( + "Any coordinate projection information in the feather file will be overwritten (with default WGS84)" + ) + # Make a pyproj CRS from the EPSG code + + _edge_file = self.primary_files[0] + edges = gpd.read_feather(_edge_file) + edges = edges.set_crs(self.crs) + + corresponding_node_file = _edge_file.replace("edges", "nodes") + if not corresponding_node_file.exists(): + raise FileNotFoundError( + "The node file could not be found while importing from TRAILS" + ) + + nodes = gpd.read_feather(corresponding_node_file) + nodes = nodes.set_crs(self.crs) + + logging.info("TRAILS importer: start generating graph") + # tempfix to rename columns + edges = edges.rename({"from_id": "node_A", "to_id": "node_B"}, axis="columns") + node_id = "id" + graph_simple = graph_from_gdf(edges, nodes, name="network", node_id=node_id) + + logging.info("TRAILS importer: graph generating was succesfull.") + logging.warning( + "RA2CE will not clean-up your graph, assuming that it is already done in TRAILS" + ) + + edges_complex = edges + if self.segmentation_length: + logging.info("TRAILS importer: start segmentating graph") + to_segment = Segmentation(edges, self._cleanup.segmentation_length) + edges_simple_segmented = to_segment.apply_segmentation() + if edges_simple_segmented.crs is None: # The CRS might have dissapeared. + edges_simple_segmented.crs = edges.crs # set the right CRS + edges_complex = edges_simple_segmented + + graph_complex = graph_simple # NOTE THAT DIFFERENCE + # BETWEEN SIMPLE AND COMPLEX DOES NOT EXIST WHEN IMPORTING WITH TRAILS + + # Todo: better control over metadata in trails + # Todo: better control over where things are saved in the pipeline + return graph_complex, edges_complex From db643b019cda56004092f5ca6a25c958a3bcaeff Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 10:38:36 +0200 Subject: [PATCH 15/26] chore: Refactored networks to better organize how networks are created --- ra2ce/graph/networks.py | 170 +++++++++--------- .../osm_network_wrapper.py | 1 + .../trails_network_wrapper.py | 5 + 3 files changed, 89 insertions(+), 87 deletions(-) diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index d4a9a63a1..9231e8c18 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -20,6 +20,7 @@ """ import logging +from pathlib import Path from typing import Any import geopandas as gpd @@ -91,16 +92,14 @@ def _any_cleanup_enabled(self) -> bool: ) def _create_network_from_shp( - self, + self, crs_value: int ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: logging.info("Start creating a network from the submitted shapefile.") if self._any_cleanup_enabled(): - return self.network_shp() + return self.network_shp(crs_value) return self.network_cleanshp() - def network_shp( - self, crs: int = 4326 - ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + def network_shp(self, crs: int) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """Creates a (graph) network from a shapefile. Returns the same geometries for the network (GeoDataFrame) as for the graph (NetworkX graph), because @@ -125,9 +124,6 @@ def network_shp( self.project_name, ) - self.base_graph_crs = pyproj.CRS.from_user_input(crs) - self.base_network_crs = pyproj.CRS.from_user_input(crs) - # Exporting complex graph because the shapefile should be kept the same as much as possible. return graph_complex, edges_complex @@ -154,13 +150,11 @@ def network_cleanshp(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: edges_complex, ) = vector_network_wrapper.get_network() - # Set the CRS of the graph and network to wrapper crs - self.base_graph_crs = vector_network_wrapper.crs - self.base_network_crs = vector_network_wrapper.crs - return graph_complex, edges_complex - def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + def network_osm_download( + self, crs: int + ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """ Creates a network from a polygon by downloading via the OSM API in the extent of the polygon. @@ -169,19 +163,12 @@ def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame """ osm_network = OsmNetworkWrapper( network_data=self._network_config, - graph_crs="", + graph_crs=crs, output_graph_dir=self.output_graph_dir, ) graph_simple, edges_complex = osm_network.get_network() # No segmentation required, the non-simplified road segments from OSM are already small enough - self.base_graph_crs = pyproj.CRS.from_user_input( - "EPSG:4326" - ) # Graphs from OSM download are always in this CRS. - self.base_network_crs = pyproj.CRS.from_user_input( - "EPSG:4326" - ) # Graphs from OSM download are always in this CRS. - return graph_simple, edges_complex def add_od_nodes( @@ -256,6 +243,77 @@ def _export_network_files( ) self.files[graph_name] = _exporter.get_pickle_path() + def _create_new_network_and_graph( + self, source: str, crs_value: int + ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + # Create the network from the network source + if source == "shapefile": + return self._create_network_from_shp(crs_value) + elif source == "OSM PBF": + return TrailsNetworkWrapper( + network_data=self._network_config, crs_value=crs_value + ).get_network() + elif source == "OSM download": + return self.network_osm_download(crs_value) + elif source == "pickle": + logging.info("Start importing a network from pickle") + base_graph = GraphPickleReader().read( + self.output_graph_dir.joinpath("base_graph.p") + ) + network_gdf = gpd.read_feather( + self.output_graph_dir.joinpath("base_network.feather") + ) + return base_graph, network_gdf + + def _get_new_network_and_graph( + self, source: str, export_types: list[str] + ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + _base_graph, _network_gdf = self._create_new_network_and_graph(source, 4326) + + # Set the road lengths to meters for both the base_graph and network_gdf + # TODO: rename "length" column to "length [m]" to be explicit + edges_lengths_meters = { + (e[0], e[1], e[2]): { + "length": nut.line_length(e[-1]["geometry"], _network_gdf.crs) + } + for e in _base_graph.edges.data(keys=True) + } + nx.set_edge_attributes(_base_graph, edges_lengths_meters) + + _network_gdf["length"] = _network_gdf["geometry"].apply( + lambda x: nut.line_length(x, _network_gdf.crs) + ) + + # Save the graph and geodataframe + self._export_network_files(_base_graph, "base_graph", export_types) + self._export_network_files(_network_gdf, "base_network", export_types) + return _base_graph, _network_gdf + + def _get_stored_network_and_graph( + self, base_graph_filepath: Path, base_network_filepath: Path + ): + logging.info( + "Apparently, you already did create a network with ra2ce earlier. " + + "Ra2ce will use this: {}".format(base_graph_filepath) + ) + + def check_base_file(file_type: str, file_path: Path): + if not isinstance(base_graph_filepath) or not base_graph_filepath.is_file(): + raise FileNotFoundError( + "No base {} file found at {}.".format(file_type, file_path) + ) + + check_base_file("graph", base_graph_filepath) + check_base_file("network", base_network_filepath) + + _base_graph = GraphPickleReader().read(base_graph_filepath) + _network_gdf = gpd.read_feather(base_network_filepath) + + # Assuming the same CRS for both the network and graph + self.base_graph_crs = _network_gdf.crs + self.base_network_crs = _network_gdf.crs + return _base_graph, _network_gdf + def create(self) -> dict: """Handler function with the logic to call the right functions to create a network. @@ -270,76 +328,14 @@ def create(self) -> dict: # For all graph and networks - check if it exists, otherwise, make the graph and/or network. if not (self.files["base_graph"] or self.files["base_network"]): - # Create the network from the network source - if self._network_config.source == "shapefile": - base_graph, network_gdf = self._create_network_from_shp() - elif self._network_config.source == "OSM PBF": - logging.info( - """The original OSM PBF import is no longer supported. - Instead, the beta version of package TRAILS is used. - First stable release of TRAILS is expected in 2023.""" - ) - - # base_graph, network_gdf = self.network_osm_pbf() #The old approach is depreciated - base_graph, network_gdf = TrailsNetworkWrapper( - network_data=self._network_config, crs_value=4326 - ).get_network() - - self.base_network_crs = network_gdf.crs - - elif self._network_config.source == "OSM download": - logging.info("Start downloading a network from OSM.") - base_graph, network_gdf = self.network_osm_download() - elif self._network_config.source == "pickle": - logging.info("Start importing a network from pickle") - base_graph = GraphPickleReader().read( - self.output_graph_dir.joinpath("base_graph.p") - ) - network_gdf = gpd.read_feather( - self.output_graph_dir.joinpath("base_network.feather") - ) - - # Assuming the same CRS for both the network and graph - self.base_graph_crs = pyproj.CRS.from_user_input(network_gdf.crs) - self.base_network_crs = pyproj.CRS.from_user_input(network_gdf.crs) - - # Set the road lengths to meters for both the base_graph and network_gdf - # TODO: rename "length" column to "length [m]" to be explicit - edges_lengths_meters = { - (e[0], e[1], e[2]): { - "length": nut.line_length(e[-1]["geometry"], self.base_graph_crs) - } - for e in base_graph.edges.data(keys=True) - } - nx.set_edge_attributes(base_graph, edges_lengths_meters) - - network_gdf["length"] = network_gdf["geometry"].apply( - lambda x: nut.line_length(x, self.base_network_crs) + base_graph, network_gdf = self._get_new_network_and_graph( + self._network_config.source, to_save ) - - # Save the graph and geodataframe - self._export_network_files(base_graph, "base_graph", to_save) - self._export_network_files(network_gdf, "base_network", to_save) else: - logging.info( - "Apparently, you already did create a network with ra2ce earlier. " - + "Ra2ce will use this: {}".format(self.files["base_graph"]) + base_graph, network_gdf = self._get_stored_network_and_graph( + self.files["base_graph"], self.files["base_network"] ) - if self.files["base_graph"] is not None: - base_graph = GraphPickleReader().read(self.files["base_graph"]) - else: - base_graph = None - - if self.files["base_network"] is not None: - network_gdf = gpd.read_feather(self.files["base_network"]) - else: - network_gdf = None - - # Assuming the same CRS for both the network and graph - self.base_graph_crs = pyproj.CRS.from_user_input(network_gdf.crs) - self.base_network_crs = pyproj.CRS.from_user_input(network_gdf.crs) - # create origins destinations graph if ( (self.origins) diff --git a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py index 171bb8eb9..5be54b26f 100644 --- a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py +++ b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py @@ -55,6 +55,7 @@ def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: Returns: tuple[MultiGraph, GeoDataFrame]: _description_ """ + logging.info("Start downloading a network from OSM.") graph_complex = self.get_clean_graph_from_osm() # Create 'graph_simple' diff --git a/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py b/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py index 9304c4219..51d03f2e1 100644 --- a/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py +++ b/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py @@ -40,6 +40,11 @@ def __init__( cleanup_section: CleanupSection, crs_value: str, ) -> None: + logging.info( + """The original OSM PBF import is no longer supported. + Instead, the beta version of package TRAILS is used. + First stable release of TRAILS is expected in 2023.""" + ) self.primary_files = network_data.primary_file self.segmentation_length = cleanup_section.segmentation_length self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") From 4e20d8b1dd249c25b169af0e5df087031d38010b Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:16:05 +0200 Subject: [PATCH 16/26] chore: Updated tests to use network config data as an input --- .../network_config_data.py | 5 +- ra2ce/graph/network_wrapper_protocol.py | 4 +- ra2ce/graph/networks.py | 85 +++---------------- .../osm_network_wrapper.py | 19 +++-- .../shp_network_wrapper.py | 50 ++++------- .../vector_network_wrapper.py | 24 ++---- .../trails_network_wrapper.py | 14 ++- .../test_osm_network_wrapper.py | 18 ++-- .../test_vector_network_wrapper.py | 42 ++++----- 9 files changed, 88 insertions(+), 173 deletions(-) diff --git a/ra2ce/graph/network_config_data/network_config_data.py b/ra2ce/graph/network_config_data/network_config_data.py index deebbce90..624fc4db6 100644 --- a/ra2ce/graph/network_config_data/network_config_data.py +++ b/ra2ce/graph/network_config_data/network_config_data.py @@ -26,6 +26,7 @@ from typing import Optional from ra2ce.common.configuration.config_data_protocol import ConfigDataProtocol +from pyproj import CRS @dataclass @@ -90,7 +91,8 @@ class NetworkConfigData(ConfigDataProtocol): input_path: Optional[Path] = None output_path: Optional[Path] = None static_path: Optional[Path] = None - + # CRS is not yet supported in the ini file, it might be relocated to a subsection. + crs: CRS = field(default_factory=lambda: CRS.from_user_input(4326)) project: ProjectSection = field(default_factory=lambda: ProjectSection()) network: NetworkSection = field(default_factory=lambda: NetworkSection()) origins_destinations: OriginsDestinationsSection = field( @@ -114,6 +116,7 @@ def network_dir(self) -> Optional[Path]: def to_dict(self) -> dict: _dict = self.__dict__ + _dict["crs"] = self.crs.to_epsg() _dict["project"] = self.project.__dict__ _dict["network"] = self.network.__dict__ _dict["origins_destinations"] = self.origins_destinations.__dict__ diff --git a/ra2ce/graph/network_wrapper_protocol.py b/ra2ce/graph/network_wrapper_protocol.py index 6590fc558..4523038e7 100644 --- a/ra2ce/graph/network_wrapper_protocol.py +++ b/ra2ce/graph/network_wrapper_protocol.py @@ -5,9 +5,9 @@ @runtime_checkable class NetworkWrapperProtocol(Protocol): - def get_network(self, **kwargs) -> tuple[MultiGraph, GeoDataFrame]: + def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: """ - Gets a network built within this wrapper instance. + Gets a network built within this wrapper instance. No arguments are accepted, the `__init__` method is meant to assign all required attributes for a wrapper. Returns: tuple[MultiGraph, GeoDataFrame]: Tuple of MultiGraph representing the graph and GeoDataFrame representing the network. diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 9231e8c18..1ff48c3f5 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -51,6 +51,7 @@ class Network: def __init__(self, network_config: NetworkConfigData, files: dict): # General + self._config_data = network_config self.project_name = network_config.project.name self.output_graph_dir = network_config.output_graph_dir if not self.output_graph_dir.is_dir(): @@ -92,84 +93,22 @@ def _any_cleanup_enabled(self) -> bool: ) def _create_network_from_shp( - self, crs_value: int + self, ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: logging.info("Start creating a network from the submitted shapefile.") if self._any_cleanup_enabled(): - return self.network_shp(crs_value) - return self.network_cleanshp() + return ShpNetworkWrapper(self._config_data).get_network() + return VectorNetworkWrapper(self._config_data).get_network() - def network_shp(self, crs: int) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - """Creates a (graph) network from a shapefile. - - Returns the same geometries for the network (GeoDataFrame) as for the graph (NetworkX graph), because - it is assumed that the user wants to keep the same geometries as their shapefile input. - - Args: - crs (int): the EPSG number of the coordinate reference system that is used - - Returns: - graph_complex (NetworkX graph): The resulting graph. - edges_complex (GeoDataFrame): The resulting network. - """ - # Make a pyproj CRS from the EPSG code - _shp_network_wrapper = ShpNetworkWrapper( - network_options=self._network_config, - cleanup_options=self._cleanup, - region_path=self.region, - crs_value=crs, - ) - graph_complex, edges_complex = _shp_network_wrapper.get_network( - self.output_graph_dir, - self.project_name, - ) - - # Exporting complex graph because the shapefile should be kept the same as much as possible. - return graph_complex, edges_complex - - def network_cleanshp(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - """Creates a (graph) network from a clean shapefile (primary_file - no further advance cleanup is needed) - - Returns the same geometries for the network (GeoDataFrame) as for the graph (NetworkX graph), because - it is assumed that the user wants to keep the same geometries as their shapefile input. - - Returns: - graph_complex (NetworkX graph): The resulting graph. - edges_complex (GeoDataFrame): The resulting network. - """ - # initialise vector network wrapper - vector_network_wrapper = VectorNetworkWrapper( - network_data=self._network_config, - region_path=self.region, - crs_value="", - ) - - # setup network using the wrapper - ( - graph_complex, - edges_complex, - ) = vector_network_wrapper.get_network() - - return graph_complex, edges_complex - - def network_osm_download( - self, crs: int - ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: + def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: """ Creates a network from a polygon by downloading via the OSM API in the extent of the polygon. Returns: tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: Tuple of Simplified graph (for use in the indirect analyses) and Complex graph (for use in the direct analyses). """ - osm_network = OsmNetworkWrapper( - network_data=self._network_config, - graph_crs=crs, - output_graph_dir=self.output_graph_dir, - ) - graph_simple, edges_complex = osm_network.get_network() - # No segmentation required, the non-simplified road segments from OSM are already small enough - return graph_simple, edges_complex + return OsmNetworkWrapper(self._config_data).get_network() def add_od_nodes( self, graph: nx.classes.graph.Graph, crs: pyproj.CRS @@ -244,17 +183,15 @@ def _export_network_files( self.files[graph_name] = _exporter.get_pickle_path() def _create_new_network_and_graph( - self, source: str, crs_value: int + self, source: str ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: # Create the network from the network source if source == "shapefile": - return self._create_network_from_shp(crs_value) + return self._create_network_from_shp() elif source == "OSM PBF": - return TrailsNetworkWrapper( - network_data=self._network_config, crs_value=crs_value - ).get_network() + return TrailsNetworkWrapper(self._config_data).get_network() elif source == "OSM download": - return self.network_osm_download(crs_value) + return self.network_osm_download() elif source == "pickle": logging.info("Start importing a network from pickle") base_graph = GraphPickleReader().read( @@ -268,7 +205,7 @@ def _create_new_network_and_graph( def _get_new_network_and_graph( self, source: str, export_types: list[str] ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - _base_graph, _network_gdf = self._create_new_network_and_graph(source, 4326) + _base_graph, _network_gdf = self._create_new_network_and_graph(source) # Set the road lengths to meters for both the base_graph and network_gdf # TODO: rename "length" column to "length [m]" to be explicit diff --git a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py index 5be54b26f..fb79ce9f7 100644 --- a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py +++ b/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py @@ -25,6 +25,7 @@ from osmnx import consolidate_intersections from shapely.geometry.base import BaseGeometry from ra2ce.graph.network_config_data.network_config_data import ( + NetworkConfigData, NetworkSection, ) from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol @@ -37,16 +38,16 @@ class OsmNetworkWrapper(NetworkWrapperProtocol): def __init__( self, - network_data: NetworkSection, - output_graph_dir: Path, - graph_crs: str, + config_data: NetworkConfigData ) -> None: - self.network_type = network_data.network_type - self.road_types = network_data.road_types - self.polygon_path = network_data.polygon - self.is_directed = network_data.directed - self.output_graph_dir = output_graph_dir - self.graph_crs = graph_crs if graph_crs else "epsg:4326" + self.output_graph_dir = config_data.output_graph_dir + self.graph_crs = config_data.crs + + # Network options + self.network_type = config_data.network.network_type + self.road_types = config_data.network.road_types + self.polygon_path = config_data.network.polygon + self.is_directed = config_data.network.directed def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: """ diff --git a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py index 5bb833a5e..f05fdd02d 100644 --- a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py @@ -20,14 +20,9 @@ """ import math -from pathlib import Path -from pyproj import CRS import geopandas as gpd import pandas as pd -from ra2ce.graph.network_config_data.network_config_data import ( - NetworkSection, - CleanupSection, -) +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol import ra2ce.graph.networks_utils as nut from shapely.geometry import MultiLineString @@ -40,35 +35,28 @@ class ShpNetworkWrapper(NetworkWrapperProtocol): def __init__( self, - network_options: NetworkSection, - cleanup_options: CleanupSection, - region_path: Path, - crs_value: str, + config_data: NetworkConfigData, ) -> None: - """Initializes the VectorNetworkWrapper object. + _network_options = config_data.network + _cleanup_options = config_data.cleanup - Args: - config (dict): Configuration dictionary. + self.project_name = config_data.project.name + self.crs = config_data.crs - Raises: - ValueError: If the config is None or doesn't contain a network dictionary, - or if config['network'] is not a dictionary. - """ # Network options - self.primary_files = network_options.primary_file - self.diversion_files = network_options.diversion_file - self.directed = network_options.directed - self.file_id = network_options.file_id + self.primary_files = _network_options.primary_file + self.diversion_files = _network_options.diversion_file + self.directed = _network_options.directed + self.file_id = _network_options.file_id # Cleanup options - self.merge_lines = cleanup_options.merge_lines - self.snapping_threshold = cleanup_options.snapping_threshold - self.segmentation_length = cleanup_options.segmentation_length - self.cut_at_intersections = cleanup_options.cut_at_intersections + self.merge_lines = _cleanup_options.merge_lines + self.snapping_threshold = _cleanup_options.snapping_threshold + self.segmentation_length = _cleanup_options.segmentation_length + self.cut_at_intersections = _cleanup_options.cut_at_intersections - # Other - self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") - self.region_path = region_path + # Origins Destinations + self.region_path = config_data.origins_destinations.region def _read_merge_shp(self) -> gpd.GeoDataFrame: """Imports shapefile(s) and saves attributes in a pandas dataframe. @@ -153,8 +141,6 @@ def _get_complex_graph_and_edges( def get_network( self, - output_graph_dir: Path, - project_name: str, ) -> tuple[nx.MultiGraph, gpd.GeoDataFrame]: edges = self._read_merge_shp() lines_merged = gpd.GeoDataFrame() @@ -190,8 +176,8 @@ def get_network( lines_merged.set_geometry( col="geometry", inplace=True ) # To ensure the object is a GeoDataFrame and not a Series - _emerged_lines_file = output_graph_dir.joinpath( - f"{project_name}_lines_that_merged.shp" + _emerged_lines_file = self.output_graph_dir.joinpath( + f"{self.project_name}_lines_that_merged.shp" ) lines_merged.to_file(_emerged_lines_file) logging.info( diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py index bec127fc4..bac358584 100644 --- a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py +++ b/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py @@ -28,8 +28,7 @@ import momepy from shapely.geometry import Point -from pyproj import CRS -from ra2ce.graph.network_config_data.network_config_data import NetworkSection +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol import ra2ce.graph.networks_utils as nut @@ -43,23 +42,16 @@ class VectorNetworkWrapper(NetworkWrapperProtocol): def __init__( self, - network_data: NetworkSection, - region_path: Path, - crs_value: str, + config_data: NetworkConfigData, ) -> None: - """Initializes the VectorNetworkWrapper object. + self.crs = config_data.crs - Args: - config (dict): Configuration dictionary. + # Network options + self.primary_files = config_data.network.primary_file + self.directed = config_data.network.directed - Raises: - ValueError: If the config is None or doesn't contain a network dictionary, - or if config['network'] is not a dictionary. - """ - self.primary_files = network_data.primary_file - self.directed = network_data.directed - self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") - self.region_path = region_path + # Origins Destinations + self.region_path = config_data.origins_destinations.region def get_network( self, diff --git a/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py b/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py index 51d03f2e1..eb3743068 100644 --- a/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py +++ b/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py @@ -22,13 +22,11 @@ from geopandas import GeoDataFrame from networkx import MultiGraph from ra2ce.graph.network_config_data.network_config_data import ( - CleanupSection, - NetworkSection, + NetworkConfigData, ) from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from ra2ce.graph.networks_utils import graph_from_gdf from ra2ce.graph.segmentation import Segmentation -from pyproj import CRS import logging import geopandas as gpd @@ -36,18 +34,16 @@ class TrailsNetworkWrapper(NetworkWrapperProtocol): def __init__( self, - network_data: NetworkSection, - cleanup_section: CleanupSection, - crs_value: str, + config_data: NetworkConfigData ) -> None: logging.info( """The original OSM PBF import is no longer supported. Instead, the beta version of package TRAILS is used. First stable release of TRAILS is expected in 2023.""" ) - self.primary_files = network_data.primary_file - self.segmentation_length = cleanup_section.segmentation_length - self.crs = CRS.from_user_input(crs_value if crs_value else "epsg:4326") + self.primary_files = config_data.network.primary_file + self.segmentation_length = config_data.cleanup.segmentation_length + self.crs = config_data.crs def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: """Creates a network which has been prepared in the TRAILS package diff --git a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py index 62dd12bce..3ca430607 100644 --- a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py +++ b/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py @@ -5,7 +5,10 @@ from networkx.utils import graphs_equal from shapely.geometry import LineString, Polygon from shapely.geometry.base import BaseGeometry -from ra2ce.graph.network_config_data.network_config_data import NetworkSection +from ra2ce.graph.network_config_data.network_config_data import ( + NetworkConfigData, + NetworkSection, +) from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data, slow_test, test_results @@ -17,18 +20,15 @@ class TestOsmNetworkWrapper: def test_initialize_without_graph_crs(self): # 1. Define test data. _network_section = NetworkSection(network_type="a_network", road_types=["r"]) + _network_config_data = NetworkConfigData(network=_network_section) # 2. Run test. - _wrapper = OsmNetworkWrapper( - network_data=_network_section, - output_graph_dir=test_results.joinpath("test_osm_network_wrapper"), - graph_crs="", - ) + _wrapper = OsmNetworkWrapper(_network_config_data) # 3. Verify final expectations. assert isinstance(_wrapper, OsmNetworkWrapper) assert isinstance(_wrapper, NetworkWrapperProtocol) - assert _wrapper.graph_crs == "epsg:4326" + assert _wrapper.graph_crs.to_epsg() == 4326 @pytest.fixture def _network_wrapper_without_polygon(self) -> OsmNetworkWrapper: @@ -39,9 +39,7 @@ def _network_wrapper_without_polygon(self) -> OsmNetworkWrapper: if not _output_dir.exists(): _output_dir.mkdir(parents=True) yield OsmNetworkWrapper( - network_data=_network_section, - output_graph_dir=_output_dir, - graph_crs="", + NetworkConfigData(network=_network_section, output_path=_output_dir) ) def test_download_clean_graph_from_osm_with_invalid_polygon_arg( diff --git a/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py b/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py index 8c795d5cf..2bbee37ac 100644 --- a/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py +++ b/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py @@ -1,10 +1,14 @@ from pathlib import Path +from pyproj import CRS import pytest import geopandas as gpd import networkx as nx from shapely.geometry import LineString, Point, MultiLineString -from ra2ce.graph.network_config_data.network_config_data import NetworkSection +from ra2ce.graph.network_config_data.network_config_data import ( + NetworkConfigData, + NetworkSection, +) from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data @@ -40,37 +44,35 @@ def mock_graph(self): return graph - def test_init_without_crs_sts_default(self): + def test_initialize(self): # 1. Define test data. - _network_data = NetworkSection( - primary_file=[Path("dummy_primary")], directed=False - ) - _primary_files = [Path("dummy_primary")] - _region = Path("dummy_region") - _crs_value = "" + _config_data = NetworkConfigData() + _config_data.network.primary_file = [Path("dummy_primary")] + _config_data.network.directed = False + _config_data.origins_destinations.region = Path("dummy_region") # 2. Run test. - _wrapper = VectorNetworkWrapper( - network_data=_network_data, region_path=_region, crs_value=_crs_value - ) + _wrapper = VectorNetworkWrapper(_config_data) # 3. Verify expectations. assert isinstance(_wrapper, VectorNetworkWrapper) assert isinstance(_wrapper, NetworkWrapperProtocol) - assert _wrapper.primary_files == _primary_files - assert _wrapper.region_path == _region - assert str(_wrapper.crs) == "epsg:4326" + assert _wrapper.primary_files == _config_data.network.primary_file + assert _wrapper.region_path == _config_data.origins_destinations.region + assert _wrapper.crs.to_epsg() == 4326 @pytest.fixture def _valid_wrapper(self) -> VectorNetworkWrapper: _network_dir = _test_dir.joinpath("static", "network") - _network_data = NetworkSection( - primary_file=[_network_dir.joinpath("_test_lines.geojson")], directed=False - ) + _config_data = NetworkConfigData() + _config_data.network.primary_file = [ + _network_dir.joinpath("_test_lines.geojson") + ] + _config_data.network.directed = False + _config_data.origins_destinations.region = None + _config_data.crs = CRS.from_user_input(4326) yield VectorNetworkWrapper( - network_data=_network_data, - region_path=None, - crs_value=4326, + config_data=_config_data, ) def test_read_vector_to_project_region_and_crs( From 085c3fd52ca12392c2cc3f71c01825f4931cdbb4 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:16:46 +0200 Subject: [PATCH 17/26] test: Added missing test --- tests/graph/trails_network_wrapper/__init__.py | 0 .../test_trails_network_wrapper.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/graph/trails_network_wrapper/__init__.py create mode 100644 tests/graph/trails_network_wrapper/test_trails_network_wrapper.py diff --git a/tests/graph/trails_network_wrapper/__init__.py b/tests/graph/trails_network_wrapper/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/graph/trails_network_wrapper/test_trails_network_wrapper.py b/tests/graph/trails_network_wrapper/test_trails_network_wrapper.py new file mode 100644 index 000000000..e20852c78 --- /dev/null +++ b/tests/graph/trails_network_wrapper/test_trails_network_wrapper.py @@ -0,0 +1,17 @@ +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.trails_network_wrapper.trails_network_wrapper import TrailsNetworkWrapper + + +class TestTrailsNetworkWrapper: + + def test_initialize(self): + # 1. Define test data. + _config_data = NetworkConfigData() + + # 2. Create wrapper. + _wrapper = TrailsNetworkWrapper(_config_data) + + # 3. Verify expectations. + assert isinstance(_wrapper, TrailsNetworkWrapper) + assert isinstance(_wrapper, NetworkWrapperProtocol) \ No newline at end of file From d3e1617b626075b311aa1823f649ffe9786032f7 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:32:49 +0200 Subject: [PATCH 18/26] feat: Created network wrappers as a separate module --- ra2ce/graph/network_wrapper_factory.py | 48 +++++++++++++++++ ra2ce/graph/network_wrappers/__init__.py | 0 ra2ce/graph/networks.py | 66 +++--------------------- 3 files changed, 54 insertions(+), 60 deletions(-) create mode 100644 ra2ce/graph/network_wrapper_factory.py create mode 100644 ra2ce/graph/network_wrappers/__init__.py diff --git a/ra2ce/graph/network_wrapper_factory.py b/ra2ce/graph/network_wrapper_factory.py new file mode 100644 index 000000000..2ef0ed0f4 --- /dev/null +++ b/ra2ce/graph/network_wrapper_factory.py @@ -0,0 +1,48 @@ +from geopandas import GeoDataFrame +from networkx import MultiGraph +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData +from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.shp_network_wrapper.shp_network_wrapper import ShpNetworkWrapper +from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper +import logging +from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper +from ra2ce.graph.trails_network_wrapper.trails_network_wrapper import ( + TrailsNetworkWrapper, +) +from ra2ce.common.io.readers import GraphPickleReader +import geopandas as gpd + + +class NetworkWrapperFactory(NetworkWrapperProtocol): + def __init__(self, config_data: NetworkConfigData) -> None: + self._config_data = config_data + + def _any_cleanup_enabled(self) -> bool: + _cleanup = self._config_data.cleanup + return ( + _cleanup.snapping_threshold + or _cleanup.pruning_threshold + or _cleanup.merge_lines + or _cleanup.cut_at_intersections + ) + + def get_network(self) -> tuple[MultiGraph, GeoDataFrame]: + logging.info("Start creating a network from the submitted shapefile.") + source = self._config_data.network.source + if source == "shapefile": + if self._any_cleanup_enabled(): + return ShpNetworkWrapper(self._config_data).get_network() + return VectorNetworkWrapper(self._config_data).get_network() + elif source == "OSM PBF": + return TrailsNetworkWrapper(self._config_data).get_network() + elif source == "OSM download": + return OsmNetworkWrapper(self._config_data).get_network() + elif source == "pickle": + logging.info("Start importing a network from pickle") + base_graph = GraphPickleReader().read( + self.output_graph_dir.joinpath("base_graph.p") + ) + network_gdf = gpd.read_feather( + self.output_graph_dir.joinpath("base_network.feather") + ) + return base_graph, network_gdf diff --git a/ra2ce/graph/network_wrappers/__init__.py b/ra2ce/graph/network_wrappers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 1ff48c3f5..a5288e863 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -31,12 +31,7 @@ from ra2ce.graph import networks_utils as nut from ra2ce.graph.exporters.network_exporter_factory import NetworkExporterFactory from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData -from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper -from ra2ce.graph.shp_network_wrapper.shp_network_wrapper import ShpNetworkWrapper -from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper -from ra2ce.graph.trails_network_wrapper.trails_network_wrapper import ( - TrailsNetworkWrapper, -) +from ra2ce.graph.network_wrapper_factory import NetworkWrapperFactory class Network: @@ -78,38 +73,9 @@ def __init__(self, network_config: NetworkConfigData, files: dict): self.region = _origins_destinations.region self.region_var = _origins_destinations.region_var - # Cleanup - self._cleanup = network_config.cleanup - # files self.files = files - def _any_cleanup_enabled(self) -> bool: - return ( - self._cleanup.snapping_threshold - or self._cleanup.pruning_threshold - or self._cleanup.merge_lines - or self._cleanup.cut_at_intersections - ) - - def _create_network_from_shp( - self, - ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - logging.info("Start creating a network from the submitted shapefile.") - if self._any_cleanup_enabled(): - return ShpNetworkWrapper(self._config_data).get_network() - return VectorNetworkWrapper(self._config_data).get_network() - - def network_osm_download(self) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - """ - Creates a network from a polygon by downloading via the OSM API in the extent of the polygon. - - Returns: - tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: Tuple of Simplified graph (for use in the indirect analyses) and Complex graph (for use in the direct analyses). - """ - - return OsmNetworkWrapper(self._config_data).get_network() - def add_od_nodes( self, graph: nx.classes.graph.Graph, crs: pyproj.CRS ) -> nx.classes.graph.Graph: @@ -182,30 +148,12 @@ def _export_network_files( ) self.files[graph_name] = _exporter.get_pickle_path() - def _create_new_network_and_graph( - self, source: str - ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - # Create the network from the network source - if source == "shapefile": - return self._create_network_from_shp() - elif source == "OSM PBF": - return TrailsNetworkWrapper(self._config_data).get_network() - elif source == "OSM download": - return self.network_osm_download() - elif source == "pickle": - logging.info("Start importing a network from pickle") - base_graph = GraphPickleReader().read( - self.output_graph_dir.joinpath("base_graph.p") - ) - network_gdf = gpd.read_feather( - self.output_graph_dir.joinpath("base_network.feather") - ) - return base_graph, network_gdf - def _get_new_network_and_graph( - self, source: str, export_types: list[str] + self, export_types: list[str] ) -> tuple[nx.classes.graph.Graph, gpd.GeoDataFrame]: - _base_graph, _network_gdf = self._create_new_network_and_graph(source) + _base_graph, _network_gdf = NetworkWrapperFactory( + self._config_data + ).get_network() # Set the road lengths to meters for both the base_graph and network_gdf # TODO: rename "length" column to "length [m]" to be explicit @@ -265,9 +213,7 @@ def create(self) -> dict: # For all graph and networks - check if it exists, otherwise, make the graph and/or network. if not (self.files["base_graph"] or self.files["base_network"]): - base_graph, network_gdf = self._get_new_network_and_graph( - self._network_config.source, to_save - ) + base_graph, network_gdf = self._get_new_network_and_graph(to_save) else: base_graph, network_gdf = self._get_stored_network_and_graph( self.files["base_graph"], self.files["base_network"] From 74e1da9c737ad28ad694adbda191c0eff97b35ff Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:36:29 +0200 Subject: [PATCH 19/26] chore: Refactored graph.network_wrappers --- ra2ce/graph/{ => network_wrappers}/network_wrapper_factory.py | 0 ra2ce/graph/{ => network_wrappers}/network_wrapper_protocol.py | 0 .../graph/{ => network_wrappers}/osm_network_wrapper/__init__.py | 0 .../osm_network_wrapper/extremities_data.py | 0 .../osm_network_wrapper/osm_network_wrapper.py | 0 .../graph/{ => network_wrappers}/osm_network_wrapper/osm_utils.py | 0 .../shp_network_wrapper.py | 0 .../trails_network_wrapper.py | 0 .../vector_network_wrapper.py | 0 ra2ce/graph/shp_network_wrapper/__init__.py | 0 ra2ce/graph/trails_network_wrapper/__init__.py | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename ra2ce/graph/{ => network_wrappers}/network_wrapper_factory.py (100%) rename ra2ce/graph/{ => network_wrappers}/network_wrapper_protocol.py (100%) rename ra2ce/graph/{ => network_wrappers}/osm_network_wrapper/__init__.py (100%) rename ra2ce/graph/{ => network_wrappers}/osm_network_wrapper/extremities_data.py (100%) rename ra2ce/graph/{ => network_wrappers}/osm_network_wrapper/osm_network_wrapper.py (100%) rename ra2ce/graph/{ => network_wrappers}/osm_network_wrapper/osm_utils.py (100%) rename ra2ce/graph/{shp_network_wrapper => network_wrappers}/shp_network_wrapper.py (100%) rename ra2ce/graph/{trails_network_wrapper => network_wrappers}/trails_network_wrapper.py (100%) rename ra2ce/graph/{shp_network_wrapper => network_wrappers}/vector_network_wrapper.py (100%) delete mode 100644 ra2ce/graph/shp_network_wrapper/__init__.py delete mode 100644 ra2ce/graph/trails_network_wrapper/__init__.py diff --git a/ra2ce/graph/network_wrapper_factory.py b/ra2ce/graph/network_wrappers/network_wrapper_factory.py similarity index 100% rename from ra2ce/graph/network_wrapper_factory.py rename to ra2ce/graph/network_wrappers/network_wrapper_factory.py diff --git a/ra2ce/graph/network_wrapper_protocol.py b/ra2ce/graph/network_wrappers/network_wrapper_protocol.py similarity index 100% rename from ra2ce/graph/network_wrapper_protocol.py rename to ra2ce/graph/network_wrappers/network_wrapper_protocol.py diff --git a/ra2ce/graph/osm_network_wrapper/__init__.py b/ra2ce/graph/network_wrappers/osm_network_wrapper/__init__.py similarity index 100% rename from ra2ce/graph/osm_network_wrapper/__init__.py rename to ra2ce/graph/network_wrappers/osm_network_wrapper/__init__.py diff --git a/ra2ce/graph/osm_network_wrapper/extremities_data.py b/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py similarity index 100% rename from ra2ce/graph/osm_network_wrapper/extremities_data.py rename to ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py diff --git a/ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/network_wrappers/osm_network_wrapper/osm_network_wrapper.py similarity index 100% rename from ra2ce/graph/osm_network_wrapper/osm_network_wrapper.py rename to ra2ce/graph/network_wrappers/osm_network_wrapper/osm_network_wrapper.py diff --git a/ra2ce/graph/osm_network_wrapper/osm_utils.py b/ra2ce/graph/network_wrappers/osm_network_wrapper/osm_utils.py similarity index 100% rename from ra2ce/graph/osm_network_wrapper/osm_utils.py rename to ra2ce/graph/network_wrappers/osm_network_wrapper/osm_utils.py diff --git a/ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py b/ra2ce/graph/network_wrappers/shp_network_wrapper.py similarity index 100% rename from ra2ce/graph/shp_network_wrapper/shp_network_wrapper.py rename to ra2ce/graph/network_wrappers/shp_network_wrapper.py diff --git a/ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py b/ra2ce/graph/network_wrappers/trails_network_wrapper.py similarity index 100% rename from ra2ce/graph/trails_network_wrapper/trails_network_wrapper.py rename to ra2ce/graph/network_wrappers/trails_network_wrapper.py diff --git a/ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py b/ra2ce/graph/network_wrappers/vector_network_wrapper.py similarity index 100% rename from ra2ce/graph/shp_network_wrapper/vector_network_wrapper.py rename to ra2ce/graph/network_wrappers/vector_network_wrapper.py diff --git a/ra2ce/graph/shp_network_wrapper/__init__.py b/ra2ce/graph/shp_network_wrapper/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ra2ce/graph/trails_network_wrapper/__init__.py b/ra2ce/graph/trails_network_wrapper/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 20ee285fc5679baebc682b58622f589295ddba75 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:41:22 +0200 Subject: [PATCH 20/26] chore: Moved tests and updated references --- .../network_wrappers/network_wrapper_factory.py | 12 +++++++----- ra2ce/graph/networks.py | 2 +- .../__init__.py | 0 .../test_osm_network_wrapper.py | 6 ++++-- .../test_osm_utils.py | 4 +++- .../test_shp_network_wrapper.py | 6 ++++-- .../test_trails_network_wrapper.py | 7 +++---- .../test_vector_network_wrapper.py | 5 ++--- tests/graph/shp_network_wrapper/__init__.py | 0 tests/graph/trails_network_wrapper/__init__.py | 0 10 files changed, 24 insertions(+), 18 deletions(-) rename tests/graph/{osm_network_wrapper => network_wrappers}/__init__.py (100%) rename tests/graph/{osm_network_wrapper => network_wrappers}/test_osm_network_wrapper.py (98%) rename tests/graph/{osm_network_wrapper => network_wrappers}/test_osm_utils.py (95%) rename tests/graph/{shp_network_wrapper => network_wrappers}/test_shp_network_wrapper.py (75%) rename tests/graph/{trails_network_wrapper => network_wrappers}/test_trails_network_wrapper.py (62%) rename tests/graph/{shp_network_wrapper => network_wrappers}/test_vector_network_wrapper.py (97%) delete mode 100644 tests/graph/shp_network_wrapper/__init__.py delete mode 100644 tests/graph/trails_network_wrapper/__init__.py diff --git a/ra2ce/graph/network_wrappers/network_wrapper_factory.py b/ra2ce/graph/network_wrappers/network_wrapper_factory.py index 2ef0ed0f4..74763cb53 100644 --- a/ra2ce/graph/network_wrappers/network_wrapper_factory.py +++ b/ra2ce/graph/network_wrappers/network_wrapper_factory.py @@ -1,12 +1,14 @@ from geopandas import GeoDataFrame from networkx import MultiGraph from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol -from ra2ce.graph.shp_network_wrapper.shp_network_wrapper import ShpNetworkWrapper -from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.network_wrappers.shp_network_wrapper import ShpNetworkWrapper +from ra2ce.graph.network_wrappers.vector_network_wrapper import VectorNetworkWrapper import logging -from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper -from ra2ce.graph.trails_network_wrapper.trails_network_wrapper import ( +from ra2ce.graph.network_wrappers.osm_network_wrapper.osm_network_wrapper import ( + OsmNetworkWrapper, +) +from ra2ce.graph.network_wrappers.trails_network_wrapper import ( TrailsNetworkWrapper, ) from ra2ce.common.io.readers import GraphPickleReader diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index a5288e863..23fa80eeb 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -31,7 +31,7 @@ from ra2ce.graph import networks_utils as nut from ra2ce.graph.exporters.network_exporter_factory import NetworkExporterFactory from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData -from ra2ce.graph.network_wrapper_factory import NetworkWrapperFactory +from ra2ce.graph.network_wrappers.network_wrapper_factory import NetworkWrapperFactory class Network: diff --git a/tests/graph/osm_network_wrapper/__init__.py b/tests/graph/network_wrappers/__init__.py similarity index 100% rename from tests/graph/osm_network_wrapper/__init__.py rename to tests/graph/network_wrappers/__init__.py diff --git a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py b/tests/graph/network_wrappers/test_osm_network_wrapper.py similarity index 98% rename from tests/graph/osm_network_wrapper/test_osm_network_wrapper.py rename to tests/graph/network_wrappers/test_osm_network_wrapper.py index 3ca430607..367f3a9fc 100644 --- a/tests/graph/osm_network_wrapper/test_osm_network_wrapper.py +++ b/tests/graph/network_wrappers/test_osm_network_wrapper.py @@ -9,11 +9,13 @@ NetworkConfigData, NetworkSection, ) -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data, slow_test, test_results import ra2ce.graph.networks_utils as nut -from ra2ce.graph.osm_network_wrapper.osm_network_wrapper import OsmNetworkWrapper +from ra2ce.graph.network_wrappers.osm_network_wrapper.osm_network_wrapper import ( + OsmNetworkWrapper, +) class TestOsmNetworkWrapper: diff --git a/tests/graph/osm_network_wrapper/test_osm_utils.py b/tests/graph/network_wrappers/test_osm_utils.py similarity index 95% rename from tests/graph/osm_network_wrapper/test_osm_utils.py rename to tests/graph/network_wrappers/test_osm_utils.py index e4963b6d6..dc2f31c4c 100644 --- a/tests/graph/osm_network_wrapper/test_osm_utils.py +++ b/tests/graph/network_wrappers/test_osm_utils.py @@ -2,7 +2,9 @@ import pytest -from ra2ce.graph.osm_network_wrapper.osm_utils import from_shapefile_to_poly +from ra2ce.graph.network_wrappers.osm_network_wrapper.osm_utils import ( + from_shapefile_to_poly, +) from tests import test_data, test_results diff --git a/tests/graph/shp_network_wrapper/test_shp_network_wrapper.py b/tests/graph/network_wrappers/test_shp_network_wrapper.py similarity index 75% rename from tests/graph/shp_network_wrapper/test_shp_network_wrapper.py rename to tests/graph/network_wrappers/test_shp_network_wrapper.py index 5ea75be8e..03291add5 100644 --- a/tests/graph/shp_network_wrapper/test_shp_network_wrapper.py +++ b/tests/graph/network_wrappers/test_shp_network_wrapper.py @@ -2,8 +2,10 @@ NetworkSection, CleanupSection, ) -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol -from ra2ce.graph.shp_network_wrapper.shp_network_wrapper import ShpNetworkWrapper +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.network_wrappers.shp_network_wrapper import ( + ShpNetworkWrapper, +) class TestShpNetworkWrapper: diff --git a/tests/graph/trails_network_wrapper/test_trails_network_wrapper.py b/tests/graph/network_wrappers/test_trails_network_wrapper.py similarity index 62% rename from tests/graph/trails_network_wrapper/test_trails_network_wrapper.py rename to tests/graph/network_wrappers/test_trails_network_wrapper.py index e20852c78..58c0fbbee 100644 --- a/tests/graph/trails_network_wrapper/test_trails_network_wrapper.py +++ b/tests/graph/network_wrappers/test_trails_network_wrapper.py @@ -1,10 +1,9 @@ from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol -from ra2ce.graph.trails_network_wrapper.trails_network_wrapper import TrailsNetworkWrapper +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.network_wrappers.trails_network_wrapper import TrailsNetworkWrapper class TestTrailsNetworkWrapper: - def test_initialize(self): # 1. Define test data. _config_data = NetworkConfigData() @@ -14,4 +13,4 @@ def test_initialize(self): # 3. Verify expectations. assert isinstance(_wrapper, TrailsNetworkWrapper) - assert isinstance(_wrapper, NetworkWrapperProtocol) \ No newline at end of file + assert isinstance(_wrapper, NetworkWrapperProtocol) diff --git a/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py b/tests/graph/network_wrappers/test_vector_network_wrapper.py similarity index 97% rename from tests/graph/shp_network_wrapper/test_vector_network_wrapper.py rename to tests/graph/network_wrappers/test_vector_network_wrapper.py index 2bbee37ac..546dab77e 100644 --- a/tests/graph/shp_network_wrapper/test_vector_network_wrapper.py +++ b/tests/graph/network_wrappers/test_vector_network_wrapper.py @@ -7,12 +7,11 @@ from shapely.geometry import LineString, Point, MultiLineString from ra2ce.graph.network_config_data.network_config_data import ( NetworkConfigData, - NetworkSection, ) -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol from tests import test_data -from ra2ce.graph.shp_network_wrapper.vector_network_wrapper import VectorNetworkWrapper +from ra2ce.graph.network_wrappers.vector_network_wrapper import VectorNetworkWrapper _test_dir = test_data / "vector_network_wrapper" diff --git a/tests/graph/shp_network_wrapper/__init__.py b/tests/graph/shp_network_wrapper/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/graph/trails_network_wrapper/__init__.py b/tests/graph/trails_network_wrapper/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 661731baf20f309536f6161db06783ad7d982d51 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:44:31 +0200 Subject: [PATCH 21/26] chore: Updated references after refactor --- .../analysis_config_data.py | 1 + ...s_config_data_validator_without_network.py | 3 +- .../readers/analysis_config_reader_factory.py | 6 +- .../analysis_config_reader_with_network.py | 6 +- .../analysis_config_reader_without_network.py | 6 +- .../analysis_config_wrapper_base.py | 2 +- .../analysis_config_wrapper_factory.py | 6 +- .../analysis_config_wrapper_with_network.py | 10 ++- ...analysis_config_wrapper_without_network.py | 10 ++- .../configuration/config_data_protocol.py | 2 +- .../ini_configuration_reader_protocol.py | 1 + ra2ce/configuration/config_factory.py | 8 +-- .../network_config_data.py | 3 +- .../network_config_data_validator.py | 1 + ra2ce/graph/network_config_wrapper.py | 1 + .../network_wrapper_factory.py | 16 ++--- .../network_wrapper_protocol.py | 1 + .../osm_network_wrapper/extremities_data.py | 65 +++++++++++++------ .../osm_network_wrapper.py | 24 +++---- .../network_wrappers/shp_network_wrapper.py | 11 ++-- .../trails_network_wrapper.py | 17 ++--- .../vector_network_wrapper.py | 10 +-- ...est_analysis_config_reader_with_network.py | 1 + ..._analysis_config_reader_without_network.py | 5 +- ...ysis_config_data_validator_with_network.py | 8 ++- ...s_config_data_validator_without_network.py | 1 - .../test_analysis_config_wrapper_factory.py | 6 +- ...st_analysis_config_wrapper_with_network.py | 2 +- ...analysis_config_wrapper_without_network.py | 7 +- .../test_osm_network_wrapper.py | 7 +- .../test_shp_network_wrapper.py | 6 +- .../test_vector_network_wrapper.py | 14 ++-- 32 files changed, 149 insertions(+), 118 deletions(-) diff --git a/ra2ce/analyses/analysis_config_data/analysis_config_data.py b/ra2ce/analyses/analysis_config_data/analysis_config_data.py index 26ad15bca..b662d296c 100644 --- a/ra2ce/analyses/analysis_config_data/analysis_config_data.py +++ b/ra2ce/analyses/analysis_config_data/analysis_config_data.py @@ -21,6 +21,7 @@ from __future__ import annotations + from ra2ce.common.configuration.config_data_protocol import ConfigDataProtocol diff --git a/ra2ce/analyses/analysis_config_data/analysis_config_data_validator_without_network.py b/ra2ce/analyses/analysis_config_data/analysis_config_data_validator_without_network.py index c3fd95cd7..f9c03d842 100644 --- a/ra2ce/analyses/analysis_config_data/analysis_config_data_validator_without_network.py +++ b/ra2ce/analyses/analysis_config_data/analysis_config_data_validator_without_network.py @@ -21,17 +21,16 @@ from pathlib import Path + from ra2ce.analyses.analysis_config_data.analysis_config_data import ( AnalysisConfigDataWithoutNetwork, ) - from ra2ce.common.validation.ra2ce_validator_protocol import Ra2ceIoValidator from ra2ce.common.validation.validation_report import ValidationReport from ra2ce.graph.network_config_data.network_config_data_validator import ( NetworkDictValues, ) - IndirectAnalysisNameList: list[str] = [ "single_link_redundancy", "multi_link_redundancy", diff --git a/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_factory.py b/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_factory.py index 0b997fda2..304104a06 100644 --- a/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_factory.py +++ b/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_factory.py @@ -23,9 +23,6 @@ from pathlib import Path from typing import Optional -from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( - AnalysisConfigWrapperBase, -) from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_base import ( AnalysisConfigReaderBase, ) @@ -35,6 +32,9 @@ from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_without_network import ( AnalysisConfigReaderWithoutNetwork, ) +from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( + AnalysisConfigWrapperBase, +) from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData diff --git a/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_with_network.py b/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_with_network.py index 8603ecbce..48df087a7 100644 --- a/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_with_network.py +++ b/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_with_network.py @@ -22,15 +22,15 @@ from pathlib import Path -from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( - AnalysisConfigWrapperBase, -) from ra2ce.analyses.analysis_config_data.analysis_config_data import ( AnalysisConfigDataWithNetwork, ) from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_base import ( AnalysisConfigReaderBase, ) +from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( + AnalysisConfigWrapperBase, +) from ra2ce.graph.network_config_wrapper import NetworkConfigWrapper diff --git a/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_without_network.py b/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_without_network.py index b4114e322..aac67030e 100644 --- a/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_without_network.py +++ b/ra2ce/analyses/analysis_config_data/readers/analysis_config_reader_without_network.py @@ -23,15 +23,15 @@ import logging from pathlib import Path -from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( - AnalysisConfigWrapperBase, -) from ra2ce.analyses.analysis_config_data.analysis_config_data import ( AnalysisConfigDataWithoutNetwork, ) from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_base import ( AnalysisConfigReaderBase, ) +from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( + AnalysisConfigWrapperBase, +) from ra2ce.graph.network_config_data.network_config_data_reader import ( NetworkConfigDataReader, ) diff --git a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_base.py b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_base.py index fafd2f6f8..0c6e5838d 100644 --- a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_base.py +++ b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_base.py @@ -20,7 +20,7 @@ """ -from abc import abstractmethod, abstractclassmethod +from abc import abstractclassmethod, abstractmethod from pathlib import Path from typing import Optional diff --git a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_factory.py b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_factory.py index 6d72124f9..b73cf46a3 100644 --- a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_factory.py +++ b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_factory.py @@ -23,14 +23,14 @@ from pathlib import Path from typing import Optional -from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( - AnalysisConfigWrapperBase, -) from ra2ce.analyses.analysis_config_data.analysis_config_data import ( AnalysisConfigData, AnalysisConfigDataWithNetwork, AnalysisConfigDataWithoutNetwork, ) +from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( + AnalysisConfigWrapperBase, +) from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_with_network import ( AnalysisConfigWrapperWithNetwork, ) diff --git a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_with_network.py b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_with_network.py index 093cd0968..11de109af 100644 --- a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_with_network.py +++ b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_with_network.py @@ -23,12 +23,14 @@ from __future__ import annotations from pathlib import Path -from ra2ce.analyses.analysis_config_data.analysis_config_data_validator_with_network import AnalysisConfigDataValidatorWithNetwork +from ra2ce.analyses.analysis_config_data.analysis_config_data import AnalysisConfigData +from ra2ce.analyses.analysis_config_data.analysis_config_data_validator_with_network import ( + AnalysisConfigDataValidatorWithNetwork, +) from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( AnalysisConfigWrapperBase, ) -from ra2ce.analyses.analysis_config_data.analysis_config_data import AnalysisConfigData from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.network_config_wrapper import NetworkConfigWrapper @@ -98,5 +100,7 @@ def configure(self) -> None: def is_valid(self) -> bool: _file_is_valid = self.ini_file.is_file() and self.ini_file.suffix == ".ini" - _validation_report = AnalysisConfigDataValidatorWithNetwork(self.config_data).validate() + _validation_report = AnalysisConfigDataValidatorWithNetwork( + self.config_data + ).validate() return _file_is_valid and _validation_report.is_valid() diff --git a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_without_network.py b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_without_network.py index f948d769e..b16c9d4fb 100644 --- a/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_without_network.py +++ b/ra2ce/analyses/analysis_config_wrapper/analysis_config_wrapper_without_network.py @@ -24,12 +24,14 @@ import logging from pathlib import Path -from ra2ce.analyses.analysis_config_data.analysis_config_data_validator_without_network import AnalysisConfigDataValidatorWithoutNetwork +from ra2ce.analyses.analysis_config_data.analysis_config_data import AnalysisConfigData +from ra2ce.analyses.analysis_config_data.analysis_config_data_validator_without_network import ( + AnalysisConfigDataValidatorWithoutNetwork, +) from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( AnalysisConfigWrapperBase, ) -from ra2ce.analyses.analysis_config_data.analysis_config_data import AnalysisConfigData from ra2ce.graph.network_config_wrapper import NetworkConfigWrapper @@ -77,5 +79,7 @@ def configure(self) -> None: def is_valid(self) -> bool: _file_is_valid = self.ini_file.is_file() and self.ini_file.suffix == ".ini" - _validation_report = AnalysisConfigDataValidatorWithoutNetwork(self.config_data).validate() + _validation_report = AnalysisConfigDataValidatorWithoutNetwork( + self.config_data + ).validate() return _file_is_valid and _validation_report.is_valid() diff --git a/ra2ce/common/configuration/config_data_protocol.py b/ra2ce/common/configuration/config_data_protocol.py index 5960901dc..4f505f126 100644 --- a/ra2ce/common/configuration/config_data_protocol.py +++ b/ra2ce/common/configuration/config_data_protocol.py @@ -33,4 +33,4 @@ def to_dict(self) -> dict: Returns: dict: Dictionary representing the `ConfigDataProtocol` instance. """ - pass \ No newline at end of file + pass diff --git a/ra2ce/common/configuration/ini_configuration_reader_protocol.py b/ra2ce/common/configuration/ini_configuration_reader_protocol.py index 1c081257a..edf92c54a 100644 --- a/ra2ce/common/configuration/ini_configuration_reader_protocol.py +++ b/ra2ce/common/configuration/ini_configuration_reader_protocol.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import Protocol, runtime_checkable + from ra2ce.common.configuration.config_data_protocol import ConfigDataProtocol from ra2ce.common.io.readers.file_reader_protocol import FileReaderProtocol diff --git a/ra2ce/configuration/config_factory.py b/ra2ce/configuration/config_factory.py index 956fb3dd8..d3da83cb7 100644 --- a/ra2ce/configuration/config_factory.py +++ b/ra2ce/configuration/config_factory.py @@ -20,19 +20,19 @@ """ -from pathlib import Path import shutil +from pathlib import Path from typing import Optional +from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_factory import ( + AnalysisConfigReaderFactory, +) from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_base import ( AnalysisConfigWrapperBase, ) from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_factory import ( AnalysisConfigWrapperFactory, ) -from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_factory import ( - AnalysisConfigReaderFactory, -) from ra2ce.configuration.config_wrapper import ConfigWrapper from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.network_config_data.network_config_data_reader import ( diff --git a/ra2ce/graph/network_config_data/network_config_data.py b/ra2ce/graph/network_config_data/network_config_data.py index 624fc4db6..74ab33976 100644 --- a/ra2ce/graph/network_config_data/network_config_data.py +++ b/ra2ce/graph/network_config_data/network_config_data.py @@ -25,9 +25,10 @@ from pathlib import Path from typing import Optional -from ra2ce.common.configuration.config_data_protocol import ConfigDataProtocol from pyproj import CRS +from ra2ce.common.configuration.config_data_protocol import ConfigDataProtocol + @dataclass class ProjectSection: diff --git a/ra2ce/graph/network_config_data/network_config_data_validator.py b/ra2ce/graph/network_config_data/network_config_data_validator.py index f411c7ca9..1fe334168 100644 --- a/ra2ce/graph/network_config_data/network_config_data_validator.py +++ b/ra2ce/graph/network_config_data/network_config_data_validator.py @@ -20,6 +20,7 @@ """ from typing import Any + from ra2ce.common.validation.ra2ce_validator_protocol import Ra2ceIoValidator from ra2ce.common.validation.validation_report import ValidationReport from ra2ce.graph.network_config_data.network_config_data import ( diff --git a/ra2ce/graph/network_config_wrapper.py b/ra2ce/graph/network_config_wrapper.py index a25ef1d04..03e01168b 100644 --- a/ra2ce/graph/network_config_wrapper.py +++ b/ra2ce/graph/network_config_wrapper.py @@ -37,6 +37,7 @@ ) from ra2ce.graph.networks import Network + class NetworkConfigWrapper(ConfigWrapperProtocol): files: Dict[str, Path] = {} config_data: NetworkConfigData diff --git a/ra2ce/graph/network_wrappers/network_wrapper_factory.py b/ra2ce/graph/network_wrappers/network_wrapper_factory.py index 74763cb53..792cb7256 100644 --- a/ra2ce/graph/network_wrappers/network_wrapper_factory.py +++ b/ra2ce/graph/network_wrappers/network_wrapper_factory.py @@ -1,18 +1,18 @@ +import logging + +import geopandas as gpd from geopandas import GeoDataFrame from networkx import MultiGraph + +from ra2ce.common.io.readers import GraphPickleReader from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol -from ra2ce.graph.network_wrappers.shp_network_wrapper import ShpNetworkWrapper -from ra2ce.graph.network_wrappers.vector_network_wrapper import VectorNetworkWrapper -import logging from ra2ce.graph.network_wrappers.osm_network_wrapper.osm_network_wrapper import ( OsmNetworkWrapper, ) -from ra2ce.graph.network_wrappers.trails_network_wrapper import ( - TrailsNetworkWrapper, -) -from ra2ce.common.io.readers import GraphPickleReader -import geopandas as gpd +from ra2ce.graph.network_wrappers.shp_network_wrapper import ShpNetworkWrapper +from ra2ce.graph.network_wrappers.trails_network_wrapper import TrailsNetworkWrapper +from ra2ce.graph.network_wrappers.vector_network_wrapper import VectorNetworkWrapper class NetworkWrapperFactory(NetworkWrapperProtocol): diff --git a/ra2ce/graph/network_wrappers/network_wrapper_protocol.py b/ra2ce/graph/network_wrappers/network_wrapper_protocol.py index 4523038e7..49494fa6d 100644 --- a/ra2ce/graph/network_wrappers/network_wrapper_protocol.py +++ b/ra2ce/graph/network_wrappers/network_wrapper_protocol.py @@ -1,4 +1,5 @@ from typing import Protocol, runtime_checkable + from geopandas import GeoDataFrame from networkx import MultiGraph diff --git a/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py b/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py index 6e4308de1..33c1f7a65 100644 --- a/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py +++ b/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py @@ -13,8 +13,13 @@ class ExtremitiesData: to_from_coor: tuple = None @staticmethod - def get_extremities_data_for_sub_graph(from_node_id: int, to_node_id: int, sub_graph: MultiDiGraph, - graph: MultiDiGraph, shared_elements: set): + def get_extremities_data_for_sub_graph( + from_node_id: int, + to_node_id: int, + sub_graph: MultiDiGraph, + graph: MultiDiGraph, + shared_elements: set, + ): """Both extremities should be in the unique_graph still makes an edge between similar node to u (the node with u coordinates and different id, included in the unique_graph) and v Here, sub_graph is the unique_graph and graph is complex_graph Shared elements are shared btw sub_graph and graph, which are elements to include @@ -25,55 +30,75 @@ def get_extremities_data_for_sub_graph(from_node_id: int, to_node_id: int, sub_g return ExtremitiesData.arrange_extremities_data( from_node_id=from_node_id, to_node_id=to_node_id, graph=sub_graph ) - elif (graph.nodes[from_node_id]['x'], graph.nodes[from_node_id]['y']) in shared_elements and \ - to_node_id in sub_graph.nodes(): + elif ( + graph.nodes[from_node_id]["x"], + graph.nodes[from_node_id]["y"], + ) in shared_elements and to_node_id in sub_graph.nodes(): from_node_id_prime = ExtremitiesData.find_node_id_by_coor( - sub_graph, graph.nodes[from_node_id]['x'], graph.nodes[from_node_id]['y'] + sub_graph, + graph.nodes[from_node_id]["x"], + graph.nodes[from_node_id]["y"], ) if from_node_id_prime == to_node_id: return ExtremitiesData() else: - return ExtremitiesData.arrange_extremities_data(from_node_id=from_node_id_prime, to_node_id=to_node_id, - graph=sub_graph) + return ExtremitiesData.arrange_extremities_data( + from_node_id=from_node_id_prime, + to_node_id=to_node_id, + graph=sub_graph, + ) - elif from_node_id in sub_graph.nodes() and \ - (graph.nodes[to_node_id]['x'], graph.nodes[to_node_id]['y']) in shared_elements: + elif ( + from_node_id in sub_graph.nodes() + and (graph.nodes[to_node_id]["x"], graph.nodes[to_node_id]["y"]) + in shared_elements + ): to_node_id_prime = ExtremitiesData.find_node_id_by_coor( - sub_graph, graph.nodes[to_node_id]['x'], graph.nodes[to_node_id]['y'] + sub_graph, graph.nodes[to_node_id]["x"], graph.nodes[to_node_id]["y"] ) if from_node_id == to_node_id_prime: return ExtremitiesData() else: - return ExtremitiesData.arrange_extremities_data(from_node_id=from_node_id, to_node_id=to_node_id_prime, - graph=sub_graph) + return ExtremitiesData.arrange_extremities_data( + from_node_id=from_node_id, + to_node_id=to_node_id_prime, + graph=sub_graph, + ) else: return ExtremitiesData() @staticmethod - def arrange_extremities_data(from_node_id: int, to_node_id: int, graph: MultiDiGraph): + def arrange_extremities_data( + from_node_id: int, to_node_id: int, graph: MultiDiGraph + ): return ExtremitiesData( from_id=from_node_id, to_id=to_node_id, from_to_id=(from_node_id, to_node_id), to_from_id=(to_node_id, from_node_id), from_to_coor=( - (graph.nodes[from_node_id]['x'], graph.nodes[to_node_id]['x']), - (graph.nodes[from_node_id]['y'], graph.nodes[to_node_id]['y']) + (graph.nodes[from_node_id]["x"], graph.nodes[to_node_id]["x"]), + (graph.nodes[from_node_id]["y"], graph.nodes[to_node_id]["y"]), ), to_from_coor=( - (graph.nodes[to_node_id]['x'], graph.nodes[from_node_id]['x']), - (graph.nodes[to_node_id]['y'], graph.nodes[from_node_id]['y']) - ) + (graph.nodes[to_node_id]["x"], graph.nodes[from_node_id]["x"]), + (graph.nodes[to_node_id]["y"], graph.nodes[from_node_id]["y"]), + ), ) @staticmethod def find_node_id_by_coor(graph: MultiDiGraph, target_x: float, target_y: float): """ - finds the node in unique graph with the same coor + finds the node in unique graph with the same coor """ for node, data in graph.nodes(data=True): - if 'x' in data and 'y' in data and data['x'] == target_x and data['y'] == target_y: + if ( + "x" in data + and "y" in data + and data["x"] == target_x + and data["y"] == target_y + ): return node return None diff --git a/ra2ce/graph/network_wrappers/osm_network_wrapper/osm_network_wrapper.py b/ra2ce/graph/network_wrappers/osm_network_wrapper/osm_network_wrapper.py index fb79ce9f7..82d9c9b31 100644 --- a/ra2ce/graph/network_wrappers/osm_network_wrapper/osm_network_wrapper.py +++ b/ra2ce/graph/network_wrappers/osm_network_wrapper/osm_network_wrapper.py @@ -20,26 +20,22 @@ import networkx as nx import osmnx +import pandas as pd from geopandas import GeoDataFrame from networkx import MultiDiGraph, MultiGraph -from osmnx import consolidate_intersections from shapely.geometry.base import BaseGeometry -from ra2ce.graph.network_config_data.network_config_data import ( - NetworkConfigData, - NetworkSection, -) -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol -from ra2ce.graph.exporters.json_exporter import JsonExporter -import pandas as pd + import ra2ce.graph.networks_utils as nut -from ra2ce.graph.osm_network_wrapper.extremities_data import ExtremitiesData +from ra2ce.graph.exporters.json_exporter import JsonExporter +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol +from ra2ce.graph.network_wrappers.osm_network_wrapper.extremities_data import ( + ExtremitiesData, +) class OsmNetworkWrapper(NetworkWrapperProtocol): - def __init__( - self, - config_data: NetworkConfigData - ) -> None: + def __init__(self, config_data: NetworkConfigData) -> None: self.output_graph_dir = config_data.output_graph_dir self.graph_crs = config_data.crs @@ -304,7 +300,7 @@ def valid_extremity_data(u, v, data) -> tuple[ExtremitiesData, dict]: @staticmethod def snap_nodes_to_nodes(graph: MultiDiGraph, threshold: float) -> MultiDiGraph: - return consolidate_intersections( + return osmnx.consolidate_intersections( G=graph, rebuild_graph=True, tolerance=threshold, dead_ends=False ) diff --git a/ra2ce/graph/network_wrappers/shp_network_wrapper.py b/ra2ce/graph/network_wrappers/shp_network_wrapper.py index f05fdd02d..aedc6780b 100644 --- a/ra2ce/graph/network_wrappers/shp_network_wrapper.py +++ b/ra2ce/graph/network_wrappers/shp_network_wrapper.py @@ -19,16 +19,17 @@ along with this program. If not, see . """ +import logging import math + import geopandas as gpd +import networkx as nx import pandas as pd -from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol -import ra2ce.graph.networks_utils as nut from shapely.geometry import MultiLineString -import logging -import networkx as nx +import ra2ce.graph.networks_utils as nut +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol from ra2ce.graph.segmentation import Segmentation diff --git a/ra2ce/graph/network_wrappers/trails_network_wrapper.py b/ra2ce/graph/network_wrappers/trails_network_wrapper.py index eb3743068..5e95669b2 100644 --- a/ra2ce/graph/network_wrappers/trails_network_wrapper.py +++ b/ra2ce/graph/network_wrappers/trails_network_wrapper.py @@ -19,23 +19,20 @@ along with this program. If not, see . """ +import logging + +import geopandas as gpd from geopandas import GeoDataFrame from networkx import MultiGraph -from ra2ce.graph.network_config_data.network_config_data import ( - NetworkConfigData, -) -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol + +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol from ra2ce.graph.networks_utils import graph_from_gdf from ra2ce.graph.segmentation import Segmentation -import logging -import geopandas as gpd class TrailsNetworkWrapper(NetworkWrapperProtocol): - def __init__( - self, - config_data: NetworkConfigData - ) -> None: + def __init__(self, config_data: NetworkConfigData) -> None: logging.info( """The original OSM PBF import is no longer supported. Instead, the beta version of package TRAILS is used. diff --git a/ra2ce/graph/network_wrappers/vector_network_wrapper.py b/ra2ce/graph/network_wrappers/vector_network_wrapper.py index bac358584..3ba60c472 100644 --- a/ra2ce/graph/network_wrappers/vector_network_wrapper.py +++ b/ra2ce/graph/network_wrappers/vector_network_wrapper.py @@ -22,15 +22,15 @@ import logging from pathlib import Path -import networkx as nx -import pandas as pd import geopandas as gpd import momepy - +import networkx as nx +import pandas as pd from shapely.geometry import Point -from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData -from ra2ce.graph.network_wrapper_protocol import NetworkWrapperProtocol + import ra2ce.graph.networks_utils as nut +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol class VectorNetworkWrapper(NetworkWrapperProtocol): diff --git a/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_with_network.py b/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_with_network.py index 2c9aed388..8f332b53e 100644 --- a/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_with_network.py +++ b/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_with_network.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import Any + import pytest from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_with_network import ( diff --git a/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_without_network.py b/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_without_network.py index 409895a89..24055f1af 100644 --- a/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_without_network.py +++ b/tests/analyses/analysis_config_data/readers/test_analysis_config_reader_without_network.py @@ -1,5 +1,8 @@ from pathlib import Path from typing import Any + +import pytest + from ra2ce.analyses.analysis_config_data.readers.analysis_config_reader_base import ( AnalysisConfigReaderBase, ) @@ -7,8 +10,6 @@ AnalysisConfigReaderWithoutNetwork, ) -import pytest - class TestAnalysisWithoutNetworkConfigReader: def test_initialize(self): diff --git a/tests/analyses/analysis_config_data/test_analysis_config_data_validator_with_network.py b/tests/analyses/analysis_config_data/test_analysis_config_data_validator_with_network.py index d9ba07eeb..a9ab3302f 100644 --- a/tests/analyses/analysis_config_data/test_analysis_config_data_validator_with_network.py +++ b/tests/analyses/analysis_config_data/test_analysis_config_data_validator_with_network.py @@ -1,3 +1,8 @@ +from pathlib import Path +from typing import Optional + +import pytest + from ra2ce.analyses.analysis_config_data.analysis_config_data import ( AnalysisConfigDataWithNetwork, ) @@ -6,10 +11,7 @@ ) from ra2ce.common.validation.ra2ce_validator_protocol import Ra2ceIoValidator from ra2ce.common.validation.validation_report import ValidationReport -import pytest from tests import test_data -from typing import Optional -from pathlib import Path class TestAnalysisConfigDataValidatorWithNetwork: diff --git a/tests/analyses/analysis_config_data/test_analysis_config_data_validator_without_network.py b/tests/analyses/analysis_config_data/test_analysis_config_data_validator_without_network.py index 56c0fcb85..2bb0a0120 100644 --- a/tests/analyses/analysis_config_data/test_analysis_config_data_validator_without_network.py +++ b/tests/analyses/analysis_config_data/test_analysis_config_data_validator_without_network.py @@ -9,7 +9,6 @@ from ra2ce.analyses.analysis_config_data.analysis_config_data_validator_without_network import ( AnalysisConfigDataValidatorWithoutNetwork, ) - from ra2ce.common.validation.validation_report import ValidationReport from tests import test_data, test_results diff --git a/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_factory.py b/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_factory.py index d3ba7ac27..d5ad4b13a 100644 --- a/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_factory.py +++ b/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_factory.py @@ -1,13 +1,13 @@ import pytest -from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_factory import ( - AnalysisConfigWrapperFactory, -) from ra2ce.analyses.analysis_config_data.analysis_config_data import ( AnalysisConfigData, AnalysisConfigDataWithNetwork, AnalysisConfigDataWithoutNetwork, ) +from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_factory import ( + AnalysisConfigWrapperFactory, +) from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_with_network import ( AnalysisConfigWrapperWithNetwork, ) diff --git a/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_with_network.py b/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_with_network.py index 88e75a021..4be363067 100644 --- a/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_with_network.py +++ b/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_with_network.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path import pytest @@ -8,7 +9,6 @@ ) from ra2ce.graph.network_config_wrapper import NetworkConfigWrapper from tests import test_data, test_results -import shutil class TestAnalysisWithNetworkConfig: diff --git a/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_without_network.py b/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_without_network.py index 702273fed..5bcc22d41 100644 --- a/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_without_network.py +++ b/tests/analyses/analysis_config_wrapper/test_analysis_config_wrapper_without_network.py @@ -1,13 +1,12 @@ +import shutil + import pytest -from ra2ce.analyses.analysis_config_data.analysis_config_data import ( - AnalysisConfigData, -) +from ra2ce.analyses.analysis_config_data.analysis_config_data import AnalysisConfigData from ra2ce.analyses.analysis_config_wrapper.analysis_config_wrapper_without_network import ( AnalysisConfigWrapperWithoutNetwork, ) from tests import acceptance_test_data, test_results -import shutil class TestAnalysisWithoutNetworkConfiguration: diff --git a/tests/graph/network_wrappers/test_osm_network_wrapper.py b/tests/graph/network_wrappers/test_osm_network_wrapper.py index 367f3a9fc..1dde51df2 100644 --- a/tests/graph/network_wrappers/test_osm_network_wrapper.py +++ b/tests/graph/network_wrappers/test_osm_network_wrapper.py @@ -1,21 +1,22 @@ from pathlib import Path + import networkx as nx import pytest from networkx import Graph, MultiDiGraph from networkx.utils import graphs_equal from shapely.geometry import LineString, Polygon from shapely.geometry.base import BaseGeometry + +import ra2ce.graph.networks_utils as nut from ra2ce.graph.network_config_data.network_config_data import ( NetworkConfigData, NetworkSection, ) from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol - -from tests import test_data, slow_test, test_results -import ra2ce.graph.networks_utils as nut from ra2ce.graph.network_wrappers.osm_network_wrapper.osm_network_wrapper import ( OsmNetworkWrapper, ) +from tests import slow_test, test_data, test_results class TestOsmNetworkWrapper: diff --git a/tests/graph/network_wrappers/test_shp_network_wrapper.py b/tests/graph/network_wrappers/test_shp_network_wrapper.py index 03291add5..96f7b5d5d 100644 --- a/tests/graph/network_wrappers/test_shp_network_wrapper.py +++ b/tests/graph/network_wrappers/test_shp_network_wrapper.py @@ -1,11 +1,9 @@ from ra2ce.graph.network_config_data.network_config_data import ( - NetworkSection, CleanupSection, + NetworkSection, ) from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol -from ra2ce.graph.network_wrappers.shp_network_wrapper import ( - ShpNetworkWrapper, -) +from ra2ce.graph.network_wrappers.shp_network_wrapper import ShpNetworkWrapper class TestShpNetworkWrapper: diff --git a/tests/graph/network_wrappers/test_vector_network_wrapper.py b/tests/graph/network_wrappers/test_vector_network_wrapper.py index 546dab77e..1bbfb9a83 100644 --- a/tests/graph/network_wrappers/test_vector_network_wrapper.py +++ b/tests/graph/network_wrappers/test_vector_network_wrapper.py @@ -1,17 +1,15 @@ from pathlib import Path -from pyproj import CRS -import pytest import geopandas as gpd import networkx as nx -from shapely.geometry import LineString, Point, MultiLineString -from ra2ce.graph.network_config_data.network_config_data import ( - NetworkConfigData, -) -from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol +import pytest +from pyproj import CRS +from shapely.geometry import LineString, MultiLineString, Point -from tests import test_data +from ra2ce.graph.network_config_data.network_config_data import NetworkConfigData +from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol from ra2ce.graph.network_wrappers.vector_network_wrapper import VectorNetworkWrapper +from tests import test_data _test_dir = test_data / "vector_network_wrapper" From d020fb2a9ccf8add3f4e9e4c375d875574499ceb Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:45:11 +0200 Subject: [PATCH 22/26] docs: added missing headers --- .../network_wrapper_factory.py | 21 +++++++++++++++++++ .../network_wrapper_protocol.py | 20 ++++++++++++++++++ .../osm_network_wrapper/extremities_data.py | 21 +++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/ra2ce/graph/network_wrappers/network_wrapper_factory.py b/ra2ce/graph/network_wrappers/network_wrapper_factory.py index 792cb7256..be558019a 100644 --- a/ra2ce/graph/network_wrappers/network_wrapper_factory.py +++ b/ra2ce/graph/network_wrappers/network_wrapper_factory.py @@ -1,3 +1,24 @@ +""" + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Risk Assessment and Adaptation for Critical Infrastructure (RA2CE). + Copyright (C) 2023 Stichting Deltares + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + import logging import geopandas as gpd diff --git a/ra2ce/graph/network_wrappers/network_wrapper_protocol.py b/ra2ce/graph/network_wrappers/network_wrapper_protocol.py index 49494fa6d..b6159869d 100644 --- a/ra2ce/graph/network_wrappers/network_wrapper_protocol.py +++ b/ra2ce/graph/network_wrappers/network_wrapper_protocol.py @@ -1,3 +1,23 @@ +""" + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Risk Assessment and Adaptation for Critical Infrastructure (RA2CE). + Copyright (C) 2023 Stichting Deltares + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" from typing import Protocol, runtime_checkable from geopandas import GeoDataFrame diff --git a/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py b/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py index 33c1f7a65..e96c7697b 100644 --- a/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py +++ b/ra2ce/graph/network_wrappers/osm_network_wrapper/extremities_data.py @@ -1,3 +1,24 @@ +""" + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Risk Assessment and Adaptation for Critical Infrastructure (RA2CE). + Copyright (C) 2023 Stichting Deltares + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + from dataclasses import dataclass from networkx import MultiDiGraph From 7a4e5322e1d0aa158a3a017dff1918ab82cb6219 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:52:14 +0200 Subject: [PATCH 23/26] docs: Small documentation update on the purpose of the new module --- ra2ce/graph/network_wrappers/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 ra2ce/graph/network_wrappers/README.md diff --git a/ra2ce/graph/network_wrappers/README.md b/ra2ce/graph/network_wrappers/README.md new file mode 100644 index 000000000..bc04349e7 --- /dev/null +++ b/ra2ce/graph/network_wrappers/README.md @@ -0,0 +1,8 @@ +# Network wrappers + +In this module we define wrappers for reading a ra2ce network (`tuple[MultiGraph, GeoDataFrame]`). Each different class defines how a network will be created and cleaned. +Network wrappers need to instantiate the `NetworkWrapperProtocol`. + +An instance of the `NetworkWrapperProtocol` defines the `get_network` method, which will return the aforementioend ra2ce network. + +Any network wrapper can be created with the use of a `NetworkConfigData` instance. At the same time, if the user does not know which wrapper to use, the `NetworkWrapperFactory` will resolve this for us. \ No newline at end of file From 7ba86d1abca9f1026f7db58e6b3339a41fefbcf5 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 11:58:09 +0200 Subject: [PATCH 24/26] test: Fixed failing test --- .../network_wrappers/test_shp_network_wrapper.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/graph/network_wrappers/test_shp_network_wrapper.py b/tests/graph/network_wrappers/test_shp_network_wrapper.py index 96f7b5d5d..56a7c79fc 100644 --- a/tests/graph/network_wrappers/test_shp_network_wrapper.py +++ b/tests/graph/network_wrappers/test_shp_network_wrapper.py @@ -1,6 +1,5 @@ from ra2ce.graph.network_config_data.network_config_data import ( - CleanupSection, - NetworkSection, + NetworkConfigData, ) from ra2ce.graph.network_wrappers.network_wrapper_protocol import NetworkWrapperProtocol from ra2ce.graph.network_wrappers.shp_network_wrapper import ShpNetworkWrapper @@ -9,12 +8,12 @@ class TestShpNetworkWrapper: def test_init(self): # 1.Define test data. - _network_options = NetworkSection() - _cleanup_options = CleanupSection() + _config_data = NetworkConfigData() - _wrapper = ShpNetworkWrapper(_network_options, _cleanup_options, None, "") + # 2. Run test. + _wrapper = ShpNetworkWrapper(_config_data) - # Verify expectations. + # 3. Verify expectations. assert isinstance(_wrapper, ShpNetworkWrapper) assert isinstance(_wrapper, NetworkWrapperProtocol) assert _wrapper.crs.to_epsg() == 4326 From b4c81dc735a4a1c1c19aca07488978403507957f Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 12:01:36 +0200 Subject: [PATCH 25/26] test: Fixed failing tests --- ra2ce/graph/network_wrappers/shp_network_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ra2ce/graph/network_wrappers/shp_network_wrapper.py b/ra2ce/graph/network_wrappers/shp_network_wrapper.py index aedc6780b..a3d558174 100644 --- a/ra2ce/graph/network_wrappers/shp_network_wrapper.py +++ b/ra2ce/graph/network_wrappers/shp_network_wrapper.py @@ -42,6 +42,7 @@ def __init__( _cleanup_options = config_data.cleanup self.project_name = config_data.project.name + self.output_graph_dir = config_data.output_graph_dir self.crs = config_data.crs # Network options From 07162c19ac29cb22e09b46c0db7247455855d9e3 Mon Sep 17 00:00:00 2001 From: "Carles S. Soriano Perez" Date: Fri, 28 Jul 2023 12:20:42 +0200 Subject: [PATCH 26/26] chore: Added missing reassignment of projection --- ra2ce/graph/networks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ra2ce/graph/networks.py b/ra2ce/graph/networks.py index 23fa80eeb..ac05c21a2 100644 --- a/ra2ce/graph/networks.py +++ b/ra2ce/graph/networks.py @@ -155,6 +155,9 @@ def _get_new_network_and_graph( self._config_data ).get_network() + self.base_graph_crs = _network_gdf.crs + self.base_network_crs = _network_gdf.crs + # Set the road lengths to meters for both the base_graph and network_gdf # TODO: rename "length" column to "length [m]" to be explicit edges_lengths_meters = {