From 2ff0aa8eec6962ffbf342e20cb17409076f0faf5 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Tue, 10 Sep 2024 10:06:26 +0200 Subject: [PATCH 01/13] Fixing issue of small grunnriss remaining after generalization. --- file_manager/n100/file_manager_buildings.py | 27 ++++ .../n100/building/finalizing_buildings.py | 16 ++- ..._and_erasing_polygons_in_water_features.py | 130 +++++++++++------- 3 files changed, 125 insertions(+), 48 deletions(-) diff --git a/file_manager/n100/file_manager_buildings.py b/file_manager/n100/file_manager_buildings.py index c09bd8c3..63d5a42c 100644 --- a/file_manager/n100/file_manager_buildings.py +++ b/file_manager/n100/file_manager_buildings.py @@ -844,6 +844,26 @@ class Building_N100(Enum): description="erased_polygons", ) + removing_points_and_erasing_polygons_in_water_features___correct_sized_polygons___n100_building = file_manager.generate_file_name_gdb( + script_source_name=removing_points_and_erasing_polygons_in_water_features, + description="correct_sized_polygons", + ) + + removing_points_and_erasing_polygons_in_water_features___too_small_polygons___n100_building = file_manager.generate_file_name_gdb( + script_source_name=removing_points_and_erasing_polygons_in_water_features, + description="too_small_polygons", + ) + + removing_points_and_erasing_polygons_in_water_features___polygons_to_points___n100_building = file_manager.generate_file_name_gdb( + script_source_name=removing_points_and_erasing_polygons_in_water_features, + description="polygons_to_points", + ) + + removing_points_and_erasing_polygons_in_water_features___points_polygons_to_points_merged___n100_building = file_manager.generate_file_name_gdb( + script_source_name=removing_points_and_erasing_polygons_in_water_features, + description="points_polygons_to_points_merged", + ) + removing_points_and_erasing_polygons_in_water_features___simplified_polygons___n100_building = file_manager.generate_file_name_gdb( script_source_name=removing_points_and_erasing_polygons_in_water_features, description="simplified_polygons", @@ -1401,6 +1421,13 @@ class Building_N100(Enum): ) ) + finalizing_buildings___polygon_to_line_joined_fields___n100_building = ( + file_manager.generate_file_name_gdb( + script_source_name=finalizing_buildings, + description="polygon_to_line_joined_fields", + ) + ) + finalizing_buildings___hospitals_and_churches_pictogram___n100_building = ( file_manager.generate_file_name_gdb( script_source_name=finalizing_buildings, diff --git a/generalization/n100/building/finalizing_buildings.py b/generalization/n100/building/finalizing_buildings.py index 97b7ed9e..2635e3b9 100644 --- a/generalization/n100/building/finalizing_buildings.py +++ b/generalization/n100/building/finalizing_buildings.py @@ -106,6 +106,20 @@ def building_polygons_to_line(): neighbor_option="IDENTIFY_NEIGHBORS", ) + arcpy.analysis.SpatialJoin( + target_features=Building_N100.finalizing_buildings___polygon_to_line___n100_building.value, + join_features=Building_N100.removing_overlapping_polygons_and_points___building_polygons_not_intersecting_church_hospitals___n100_building.value, + out_feature_class=Building_N100.finalizing_buildings___polygon_to_line_joined_fields___n100_building.value, + match_option="SHARE_A_LINE_SEGMENT_WITH", + ) + + arcpy.CalculateField_management( + in_table=Building_N100.finalizing_buildings___polygon_to_line_joined_fields___n100_building.value, + field="objtype", + expression='"Takkant"', + expression_type="PYTHON3", + ) + def selecting_hospital_and_churches_for_pictogram_featureclass(): """ @@ -141,7 +155,7 @@ def assigning_final_file_names(): ) arcpy.management.CopyFeatures( - Building_N100.finalizing_buildings___polygon_to_line___n100_building.value, + Building_N100.finalizing_buildings___polygon_to_line_joined_fields___n100_building.value, Building_N100.OmrissLinje.value, ) diff --git a/generalization/n100/building/removing_points_and_erasing_polygons_in_water_features.py b/generalization/n100/building/removing_points_and_erasing_polygons_in_water_features.py index f5e17d54..88ba7e60 100644 --- a/generalization/n100/building/removing_points_and_erasing_polygons_in_water_features.py +++ b/generalization/n100/building/removing_points_and_erasing_polygons_in_water_features.py @@ -4,6 +4,7 @@ # Importing custom files from file_manager.n100.file_manager_buildings import Building_N100 from input_data.input_symbology import SymbologyN100 +from constants.n100_constants import N100_Values # Import custom modules from custom_tools.general_tools import custom_arcpy @@ -16,11 +17,11 @@ def main(): environment_setup.main() selecting_water_polygon_features() selecting_water_features_close_to_building_polygons() - removing_points_in_water_features() buffering_water_polygon_features() selecting_building_polygons() erasing_parts_of_building_polygons_in_water_features() merge_polygons() + removing_points_in_water_features() @timing_decorator @@ -38,51 +39,6 @@ def selecting_water_polygon_features(): ) -@timing_decorator -def removing_points_in_water_features(): - """ - Summary: - Selects points that do not intersect with any water features and applies symbology to the filtered points. - """ - sql_tourist_cabins = "byggtyp_nbr = 956" - - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=Building_N100.point_resolve_building_conflicts___POINT_OUTPUT___n100_building.value, - expression=sql_tourist_cabins, - output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___tourist_cabins___n100_building.value, - ) - - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=Building_N100.point_resolve_building_conflicts___POINT_OUTPUT___n100_building.value, - expression=sql_tourist_cabins, - output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___not_tourist_cabins___n100_building.value, - inverted=True, - ) - - # Select points that DO NOT intersect any waterfeatures - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___not_tourist_cabins___n100_building.value, - overlap_type=custom_arcpy.OverlapType.INTERSECT, - select_features=Building_N100.removing_points_and_erasing_polygons_in_water_features___water_features___n100_building.value, - output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___points_that_do_not_intersect_water_features___n100_building.value, - inverted=True, - ) - - arcpy.management.Merge( - inputs=[ - Building_N100.removing_points_and_erasing_polygons_in_water_features___tourist_cabins___n100_building.value, - Building_N100.removing_points_and_erasing_polygons_in_water_features___points_that_do_not_intersect_water_features___n100_building.value, - ], - output=Building_N100.removing_points_and_erasing_polygons_in_water_features___merged_points_and_tourist_cabins___n100_building.value, - ) - - custom_arcpy.apply_symbology( - input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___merged_points_and_tourist_cabins___n100_building.value, - in_symbology_layer=SymbologyN100.building_point.value, - output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___final_points___n100_lyrx.value, # Used in the next file, "removing_overlapping_polygons_and_points.py" - ) - - @timing_decorator def selecting_water_features_close_to_building_polygons(): """ @@ -150,6 +106,31 @@ def erasing_parts_of_building_polygons_in_water_features(): out_feature_class=Building_N100.removing_points_and_erasing_polygons_in_water_features___erased_polygons___n100_building.value, ) + sql_expression_correct_size_polygons = ( + f"Shape_Area >= {N100_Values.minimum_selection_building_polygon_size_m2.value}" + ) + + sql_expression_too_small_polygons = ( + f"Shape_Area < {N100_Values.minimum_selection_building_polygon_size_m2.value}" + ) + + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___erased_polygons___n100_building.value, + expression=sql_expression_correct_size_polygons, + output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___correct_sized_polygons___n100_building.value, + ) + + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___erased_polygons___n100_building.value, + expression=sql_expression_too_small_polygons, + output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___too_small_polygons___n100_building.value, + ) + + arcpy.management.FeatureToPoint( + in_features=Building_N100.removing_points_and_erasing_polygons_in_water_features___too_small_polygons___n100_building.value, + out_feature_class=Building_N100.removing_points_and_erasing_polygons_in_water_features___polygons_to_points___n100_building.value, + ) + @timing_decorator def merge_polygons(): @@ -160,11 +141,66 @@ def merge_polygons(): arcpy.management.Merge( inputs=[ Building_N100.removing_points_and_erasing_polygons_in_water_features___building_polygons_NOT_too_close_to_water_features___n100_building.value, - Building_N100.removing_points_and_erasing_polygons_in_water_features___erased_polygons___n100_building.value, + Building_N100.removing_points_and_erasing_polygons_in_water_features___correct_sized_polygons___n100_building.value, ], output=Building_N100.removing_points_and_erasing_polygons_in_water_features___final_building_polygons_merged___n100_building.value, ) +@timing_decorator +def removing_points_in_water_features(): + """ + Summary: + Selects points that do not intersect with any water features and applies symbology to the filtered points. + """ + + arcpy.management.Merge( + inputs=[ + Building_N100.point_resolve_building_conflicts___POINT_OUTPUT___n100_building.value, + Building_N100.removing_points_and_erasing_polygons_in_water_features___polygons_to_points___n100_building.value, + ], + output=Building_N100.removing_points_and_erasing_polygons_in_water_features___points_polygons_to_points_merged___n100_building.value, + ) + + sql_tourist_cabins = "byggtyp_nbr = 956" + + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___points_polygons_to_points_merged___n100_building.value, + expression=sql_tourist_cabins, + output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___tourist_cabins___n100_building.value, + ) + + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___points_polygons_to_points_merged___n100_building.value, + expression=sql_tourist_cabins, + output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___not_tourist_cabins___n100_building.value, + inverted=True, + ) + + # Select points that DO NOT intersect any waterfeatures + custom_arcpy.select_location_and_make_permanent_feature( + input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___not_tourist_cabins___n100_building.value, + overlap_type=custom_arcpy.OverlapType.INTERSECT, + select_features=Building_N100.removing_points_and_erasing_polygons_in_water_features___water_features___n100_building.value, + output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___points_that_do_not_intersect_water_features___n100_building.value, + inverted=True, + ) + + arcpy.management.Merge( + inputs=[ + Building_N100.removing_points_and_erasing_polygons_in_water_features___tourist_cabins___n100_building.value, + Building_N100.removing_points_and_erasing_polygons_in_water_features___points_that_do_not_intersect_water_features___n100_building.value, + ], + output=Building_N100.removing_points_and_erasing_polygons_in_water_features___merged_points_and_tourist_cabins___n100_building.value, + ) + + custom_arcpy.apply_symbology( + input_layer=Building_N100.removing_points_and_erasing_polygons_in_water_features___merged_points_and_tourist_cabins___n100_building.value, + in_symbology_layer=SymbologyN100.building_point.value, + output_name=Building_N100.removing_points_and_erasing_polygons_in_water_features___final_points___n100_lyrx.value, + # Used in the next file, "removing_overlapping_polygons_and_points.py" + ) + + if __name__ == "__main__": main() From 86428936eecb8e0abe2e7ae37a5b165a03dfd38f Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Tue, 10 Sep 2024 10:14:16 +0200 Subject: [PATCH 02/13] Updating docstring --- .../general_tools/partition_iterator.py | 61 ++++++++----------- documentation/docstring_documentation.py | 2 +- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/custom_tools/general_tools/partition_iterator.py b/custom_tools/general_tools/partition_iterator.py index 8881052b..7b91f5de 100644 --- a/custom_tools/general_tools/partition_iterator.py +++ b/custom_tools/general_tools/partition_iterator.py @@ -135,7 +135,18 @@ def configure_alias_and_type( type_name, type_path, ): - # Check if alias exists, if not, create it + """ + What: + Configures an alias by adding or updating a type with a specified path. + This function checks if the given alias exists within the `nested_alias_type_data` attribute. + If the alias does not exist, it creates a new entry for it. Then, it associates the provided + `type_name` with the given `type_path` under the specified alias. + Args: + alias (str): The alias to be configured. If it does not exist, a new one will be created. + type_name (str): The name of the type to be added or updated under the alias. + type_path (str): The path associated with the specified type. + """ + if alias not in self.nested_alias_type_data: print( f"Alias '{alias}' not found in nested_alias_type_data. Creating new alias." @@ -145,29 +156,11 @@ def configure_alias_and_type( self.nested_alias_type_data[alias][type_name] = type_path print(f"Set path for type '{type_name}' in alias '{alias}' to: {type_path}") - def create_new_alias( - self, - alias, - initial_type_name=None, - initial_type_path=None, - ): - # Check if alias already exists - if alias in self.nested_alias_type_data: - raise ValueError(f"Alias {alias} already exists.") - - # Initialize nested_alias_type_data for alias - if initial_type_name: - # Create alias with initial type and path - self.nested_alias_type_data[alias] = {initial_type_name: initial_type_path} - else: - # Initialize alias as an empty dictionary - self.nested_alias_type_data[alias] = {} - - print( - f"Created new alias '{alias}' in nested_alias_type_data with type '{initial_type_name}' and path: {initial_type_path}" - ) - def create_cartographic_partitions(self): + """ + First deletes existing partitioning feature if it exists. + Then creates partitioning feature using input and context features as the in_features. + """ self.delete_feature_class(self.partition_feature) all_features = [ @@ -204,7 +197,8 @@ def delete_feature_class(feature_class_path, alias=None, output_type=None): @staticmethod def is_safe_to_delete(file_path: str, safe_directory: str) -> bool: """ - Check if the file path is within the specified safe directory. + What: + Check if the file path is within the specified safe directory. Args: file_path (str): The path of the file to check. @@ -246,7 +240,7 @@ def delete_final_outputs(self): ) def delete_iteration_files(self, *file_paths): - """Deletes multiple feature classes or files. Detailed alias and output_type logging is not available here.""" + """Deletes multiple feature classes or files from a list.""" for file_path in file_paths: self.delete_feature_class(file_path) print(f"Deleted file: {file_path}") @@ -266,10 +260,11 @@ def create_feature_class(full_feature_path, template_feature): def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]): """ - Creates dummy features for aliases with specified types. + What: + Creates dummy features for aliases for types specified in types_to_include. Args: - types_to_include (list): Types for which dummy features should be created. + types_to_include (list): A list of types for which dummy features should be created. """ for alias, alias_data in self.nested_alias_type_data.items(): for type_info, path in list(alias_data.items()): @@ -292,6 +287,7 @@ def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]) ) def reset_dummy_used(self): + """Sets the dummy_used to false""" for alias in self.nested_alias_type_data: self.nested_alias_type_data[alias]["dummy_used"] = False @@ -327,13 +323,14 @@ def create_directory_json_documentation( iteration: bool, ) -> str: """ - Creates a directory at the given root_path for the target_dir. + What: + Creates a directory at the given root_path for the target_dir. Args: - root_path: The root directory where initial structure is created + root_path: The root directory where target_dir will be located target_dir: The target where the created directory should be placed iteration: Boolean flag indicating if the iteration_documentation should be added Returns: - A string containing the absolute path of the created directory. + str: A string containing the absolute path of the created directory. """ # Determine base directory @@ -868,10 +865,6 @@ def resolve_param(param_info): else: custom_func["params"][param] = resolved_paths - # print( - # f"Resolved {param_type} path for {param}: {custom_func['params'][param]}" - # ) - def _handle_tuple_param(self, param_info, object_id): """ Handle the resolution of parameters that are tuples of (alias, alias_type). diff --git a/documentation/docstring_documentation.py b/documentation/docstring_documentation.py index 24fbb861..4697f0d1 100644 --- a/documentation/docstring_documentation.py +++ b/documentation/docstring_documentation.py @@ -17,7 +17,7 @@ def example_of_small_function(): """ - What: Does something needing little explanation outside reading the code. + Does something needing little explanation outside reading the code. """ x = 1 + 1 print(x) From 3deb4efb7ed8ed3d9a89ea679e895abdb10a3844 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Wed, 11 Sep 2024 17:07:23 +0200 Subject: [PATCH 03/13] Cleaning up docstring and making class easier to maintain. --- .../general_tools/partition_iterator.py | 244 +++++++++++++----- 1 file changed, 178 insertions(+), 66 deletions(-) diff --git a/custom_tools/general_tools/partition_iterator.py b/custom_tools/general_tools/partition_iterator.py index 7b91f5de..2915f66a 100644 --- a/custom_tools/general_tools/partition_iterator.py +++ b/custom_tools/general_tools/partition_iterator.py @@ -103,31 +103,19 @@ def __init__( self.iteration_times_with_input = [] self.iteration_start_time = None - def unpack_alias_path_data(self, alias_path_data): - """ - Process initial alias_path_data for inputs and outputs. - """ - for alias, info in alias_path_data.items(): - if alias not in self.nested_alias_type_data: - self.nested_alias_type_data[alias] = {} - - for i in range(0, len(info), 2): - type_info = info[i] - path_info = info[i + 1] - self.nested_alias_type_data[alias][type_info] = path_info - - def unpack_alias_path_outputs(self, alias_path_outputs): + @staticmethod + def unpack_alias_path(alias_path, target_dict): """ - Process initial alias_path_outputs for outputs. + Populates target dictionaries with inputs from input parameters. """ - for alias, info in alias_path_outputs.items(): - if alias not in self.nested_final_outputs: - self.nested_final_outputs[alias] = {} + for alias, info in alias_path.items(): + if alias not in target_dict: + target_dict[alias] = {} for i in range(0, len(info), 2): type_info = info[i] path_info = info[i + 1] - self.nested_final_outputs[alias][type_info] = path_info + target_dict[alias][type_info] = path_info def configure_alias_and_type( self, @@ -213,7 +201,9 @@ def is_safe_to_delete(file_path: str, safe_directory: str) -> bool: return file_path.startswith(safe_directory) def delete_final_outputs(self): - """Deletes all existing final output files if they exist and are in the safe directory.""" + """ + Deletes all existing final output files if they exist and are in the safe directory and self.delete_final_outputs_bool is True. + """ # Check if deletion is allowed if not self.delete_final_outputs_bool: @@ -258,7 +248,7 @@ def create_feature_class(full_feature_path, template_feature): ) print(f"Created feature class: {full_feature_path}") - def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]): + def create_dummy_features(self, types_to_include: list = None): """ What: Creates dummy features for aliases for types specified in types_to_include. @@ -266,6 +256,9 @@ def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]) Args: types_to_include (list): A list of types for which dummy features should be created. """ + if types_to_include is None: + types_to_include = ["input_copy", "context_copy"] + for alias, alias_data in self.nested_alias_type_data.items(): for type_info, path in list(alias_data.items()): if type_info in types_to_include and path: @@ -319,22 +312,22 @@ def update_empty_alias_type_with_dummy_file(self, alias, type_info): @staticmethod def create_directory_json_documentation( root_path: str, - target_dir: str, + dir_name: str, iteration: bool, ) -> str: """ What: Creates a directory at the given root_path for the target_dir. Args: - root_path: The root directory where target_dir will be located - target_dir: The target where the created directory should be placed + root_path: The root directory where dir_name will be located + dir_name: The target where the created directory should be placed iteration: Boolean flag indicating if the iteration_documentation should be added Returns: str: A string containing the absolute path of the created directory. """ # Determine base directory - directory_path = os.path.join(root_path, f"{target_dir}") + directory_path = os.path.join(root_path, f"{dir_name}") # Ensure that the directory exists os.makedirs(directory_path, exist_ok=True) @@ -355,13 +348,14 @@ def write_data_to_json( object_id=None, ) -> None: """ - Writes dictionary into a json file. + What: + Writes dictionary into a json file. - Args: - data: The data to write. - file_path: The complete path (directory+file_name) where the file should be created - file_name: The name of the file to create - object_id: If provided, object_id will also be part of the file name. + Args: + data: The data to write. + file_path: The complete path (directory+file_name) where the file should be created + file_name: The name of the file to create + object_id: If provided, object_id will also be part of the file name. """ if object_id: @@ -384,7 +378,8 @@ def export_dictionaries_to_json( object_id=None, ) -> None: """ - Handles the export of alias type data and final outputs into separate json files. + What: + Handles the export of alias type data and final outputs into separate json files. Args: file_path: The complete file path where to create the output directories. @@ -427,7 +422,7 @@ def create_error_log_directory(self): """ return self.create_directory_json_documentation( root_path=self.dictionary_documentation_path, - target_dir="error_log", + dir_name="error_log", iteration=False, ) @@ -436,12 +431,21 @@ def save_error_log(self, error_log): Saves the error log to a JSON file in the error_log directory. """ error_log_directory = self.create_error_log_directory() - self.write_data_to_json( - data=error_log, file_path=error_log_directory, file_name="error_log" - ) + # Check if error log is empty + if not error_log: + self.write_data_to_json( + data=error_log, + file_path=error_log_directory, + file_name="no_errors", + ) + else: + self.write_data_to_json( + data=error_log, file_path=error_log_directory, file_name="error_log" + ) @staticmethod def generate_unique_field_name(input_feature, field_name): + """Generates a unique field name""" existing_field_names = [field.name for field in arcpy.ListFields(input_feature)] unique_field_name = field_name while unique_field_name in existing_field_names: @@ -467,6 +471,12 @@ def find_maximum_object_id(self): print(f"Error in finding max {self.object_id_field}: {e}") def prepare_input_data(self): + """ + Copies the input data, and set the path of the copy to type input_copy in self.nested_alias_type_data. + From now on when the PartitionIterator access the global data for an alias it uses input_copy. + If context_selection bool is True, it will only select context features within the search_distance of + an input_copy feature, then similar to input data set the new context feature as context_copy. + """ for alias, types in self.nested_alias_type_data.items(): if "input" in types: input_data_path = types["input"] @@ -583,9 +593,17 @@ def process_input_features( alias, iteration_partition, object_id, - ): + ) -> bool: """ - Process input features for a given partition. + What: + For an alias makes selection for the partitioning feature being iterated over. + It selects objects with their centerpoint inside the partitioning feature marking it as the objects + being generalized in the partitioning feature, but also the objects within a distance so it is taken into consideration. + The selection path is marked as type input in the self.nested_alias_type_data. + It also counts the objects for input features, if the count is 0 for the input feature the iteration return false. + If there is 0 objects in the iteration it loads in the dummy feature. + Returns: + bool: Returns true or false based if there is an input feature present for the partition. """ if "input_copy" not in self.nested_alias_type_data[alias]: # If there are no inputs to process, return None for the aliases and a flag indicating no input was present. @@ -692,7 +710,19 @@ def process_input_features( # If there are no inputs to process, return None for the aliases and a flag indicating no input was present. return False - def _process_inputs_in_partition(self, aliases, iteration_partition, object_id): + def _process_inputs_in_partition( + self, + aliases, + iteration_partition, + object_id, + ) -> bool: + """ + What: + Process input features using process_input_features function using it on all alias with an input type. + If there are one or more input features present it returns true. + Returns: + bool: Returns true or false based if there are input features present for the partition. + """ inputs_present_in_partition = False for alias in aliases: if "input_copy" in self.nested_alias_type_data[alias]: @@ -700,7 +730,7 @@ def _process_inputs_in_partition(self, aliases, iteration_partition, object_id): input_present = self.process_input_features( alias, iteration_partition, object_id ) - # Sets inputs_present_in_partition as True if any alias in partition has input present. Otherwise it remains False. + # Sets inputs_present_in_partition as True if any alias in partition has input present. Otherwise, it remains False. inputs_present_in_partition = ( inputs_present_in_partition or input_present ) @@ -708,7 +738,8 @@ def _process_inputs_in_partition(self, aliases, iteration_partition, object_id): def process_context_features(self, alias, iteration_partition, object_id): """ - Process context features for a given partition if input features are present. + Selects objects within self.search_distance for a context feature and sets the selection as type context. + If there is no objects within the distance dummy data is loaded in instead. """ if "context_copy" in self.nested_alias_type_data[alias]: context_path = self.nested_alias_type_data[alias]["context_copy"] @@ -749,13 +780,15 @@ def process_context_features(self, alias, iteration_partition, object_id): ) def _process_context_features(self, aliases, iteration_partition, object_id): + """Processes context features fo all alias with a context type using process_context_features""" for alias in aliases: self.process_context_features(alias, iteration_partition, object_id) @staticmethod def format_time(seconds): """ - Convert seconds to a formatted string: HH:MM:SS. + What: + Converts seconds to a formatted string: HH:MM:SS. Args: seconds (float): Time in seconds. @@ -770,7 +803,9 @@ def format_time(seconds): def track_iteration_time(self, object_id, inputs_present_in_partition): """ - Track the iteration time and estimate the remaining time. + What: + Tracks the iteration time and estimate the remaining time. It adds the time of iterations + with input features to a list, using the average time of this list as baseline for remaining time. Args: object_id (int): The ID of the current partition iteration. @@ -805,7 +840,17 @@ def track_iteration_time(self, object_id, inputs_present_in_partition): def find_io_params_custom_logic(self, object_id): """ - Find and resolve the IO parameters for custom logic functions. + What: + Find and resolve the input and output (IO) parameters for custom logic functions. + + How: + This function iterates over a list of custom functions, which could be either standalone + functions or class methods. For each function, it checks if it has partition IO metadata. + Functions needs to be decorated with the partition_io_decorator. In this function + input and output refers to the parameters of the custom_functions and not alias types. + + Args: + object_id: The identifier for the object that the IO parameters will be associated with. """ for custom_func in self.custom_functions: if "class" in custom_func: # Class method @@ -819,34 +864,38 @@ def find_io_params_custom_logic(self, object_id): input_params = metadata.get("inputs", []) output_params = metadata.get("outputs", []) - # print(f"\nResolving IO Params for object_id {object_id}") - # print(f"Before resolving: {custom_func['params']}") - self.resolve_io_params( - param_type="input", params=input_params, custom_func=custom_func, object_id=object_id, ) self.resolve_io_params( - param_type="output", params=output_params, custom_func=custom_func, object_id=object_id, ) - # print(f"After resolving: {custom_func['params']}") - - def resolve_io_params(self, param_type, params, custom_func, object_id): + def resolve_io_params(self, params, custom_func, object_id): """ - Resolve paths for input/output parameters of custom functions. + What: + Resolve the paths for input or output parameters of custom functions. + + How: + This function takes a set of input or output parameters and resolves their paths by iterating + over the parameters and processing them, depending on their type (tuple, dict, list, etc.). + It uses a helper function `resolve_param` to recursively resolve each parameter's path. + + Args: + params: A list of parameters to be resolved. + custom_func: The custom function containing the parameters to resolve. + object_id: The identifier for the object related to these parameters. """ def resolve_param(param_info): if isinstance(param_info, tuple) and len(param_info) == 2: return self._handle_tuple_param(param_info, object_id) elif isinstance(param_info, dict): - return {k: resolve_param(v) for k, v in param_info.items()} + return {key: resolve_param(value) for key, value in param_info.items()} elif isinstance(param_info, list): return [resolve_param(item) for item in param_info] else: @@ -860,14 +909,38 @@ def resolve_param(param_info): resolved_paths = [ resolve_param(param_info) for param_info in param_info_list ] + print(f"Printing param_info_list:\n{param_info_list}") if len(resolved_paths) == 1: custom_func["params"][param] = resolved_paths[0] else: custom_func["params"][param] = resolved_paths - def _handle_tuple_param(self, param_info, object_id): + def _handle_tuple_param(self, param_info, object_id) -> str: """ - Handle the resolution of parameters that are tuples of (alias, alias_type). + What: + Handle the resolution of parameters that are tuples of (alias, alias_type). + + How: + - This method first checks if the `alias_type` needs to be dynamically updated (if it + belongs to `self.types_to_update`). If so, a new path is constructed for the alias and + alias_type during each iteration. + - If the alias and alias_type are static (i.e., they exist in `self.nested_alias_type_data`), + it retrieves and uses the existing path without updating it. + - If the alias_type is neither in `self.types_to_update` nor static, it constructs a new + path and adds the alias_type to `self.types_to_update` for future updates during iterations. + + Why: + - Some alias types are static and only need to be resolved once, which is why their paths + are stored in `self.nested_alias_type_data`. + - Other alias types require dynamic path reconstruction for each iteration, which is why + they are tracked in `self.types_to_update` to ensure they are updated accordingly. + + Args: + param_info: A tuple containing an alias and its associated alias_type. + object_id: The identifier for the object related to these parameters. + + Returns: + str: The resolved path based on the alias and alias_type. """ alias, alias_type = param_info @@ -901,7 +974,7 @@ def _handle_tuple_param(self, param_info, object_id): return resolved_path - def construct_path_for_alias_type(self, alias, alias_type, object_id): + def construct_path_for_alias_type(self, alias, alias_type, object_id) -> str: """ Construct a new path for a given alias and type specific to the current iteration. """ @@ -911,7 +984,17 @@ def construct_path_for_alias_type(self, alias, alias_type, object_id): def execute_custom_functions(self): """ - Execute custom functions with the resolved input and output paths. + What: + Execute custom functions with the resolved input and output paths. + + How: + This function iterates through custom functions and handles them differently based + on whether they are class methods or standalone functions. It extracts the required + parameters, resolves their paths, and logs these parameters before executing the + corresponding class methods or standalone functions. + + For class methods, it separates parameters for class instantiation and method execution. + For standalone functions, it directly prepares and resolves parameters for function execution. """ for custom_func in self.custom_functions: resolved_params = {} # Initialize resolved_params @@ -962,7 +1045,9 @@ def execute_custom_functions(self): def resilient_execute_custom_functions(self, object_id): """ - Helper function to execute custom functions with retry logic to handle potential failures. + What: + Helper function to execute custom functions with retry logic to handle potential failures + caused by unreliable 3rd party logic. Args: object_id (int): The current object_id being processed (for logging). @@ -990,9 +1075,33 @@ def resilient_execute_custom_functions(self, object_id): self.error_log[object_id]["Error Messages"][attempt + 1] = error_message if attempt + 1 == max_retries: - print("Max retries reached. Moving to next iteration.") + print("Max retries reached.") + self.save_error_log(self.error_log) + raise Exception(error_message) def append_iteration_to_final(self, alias, object_id): + """ + What: + Append the result of the current iteration to the final output for a given alias. + + How: + - This method checks if the given `alias` exists in `self.nested_final_outputs`. If it does not, + the function exits. + - For each type associated with the alias, it retrieves the corresponding feature class and appends + the result of the current iteration to the final output path. + - If the alias is marked as a dummy feature (indicated by `dummy_used` in `self.nested_alias_type_data`), + the iteration is skipped. + - It selects the features from the input feature class based on a specific partition field and appends + them to the final output. If the final output does not exist, it creates the output; otherwise, it appends to it. + + Why: + This function is necessary to accumulate the results of each iteration for the given alias in a final output, + ensuring the results from multiple iterations are combined into a single dataset. + + Args: + alias: A string representing the alias whose output data is to be updated. + object_id: The identifier for the current iteration. + """ # Guard clause if alias doesn't exist in nested_final_outputs if alias not in self.nested_final_outputs: return @@ -1126,13 +1235,14 @@ def partition_iteration(self): @timing_decorator def run(self): self.total_start_time = time.time() - self.unpack_alias_path_data(self.raw_input_data) - - if self.raw_output_data is not None: - self.unpack_alias_path_outputs(self.raw_output_data) + self.unpack_alias_path( + alias_path=self.raw_input_data, target_dict=self.nested_alias_type_data + ) + self.unpack_alias_path( + alias_path=self.raw_output_data, target_dict=self.nested_final_outputs + ) self.export_dictionaries_to_json(file_name="post_initialization") - print("Initialization done\n") print("\nStarting Data Preparation...") self.delete_final_outputs() @@ -1309,12 +1419,14 @@ def run(self): root_file_partition_iterator=Building_N100.iteration__partition_iterator__n100.value, scale=env_setup.global_config.scale_n100, dictionary_documentation_path=Building_N100.iteration___json_documentation___building_n100.value, - feature_count="400000", + feature_count="800000", ) # Run the partition iterator partition_iterator.run() + # partition_iterator.find_io_params_custom_logic(3) + # Instantiate PartitionIterator with necessary parameters partition_iterator_2 = PartitionIterator( alias_path_data=inputs3, From 03613fc1d4e3804e721e8cef141169692f3c3772 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Wed, 11 Sep 2024 17:08:58 +0200 Subject: [PATCH 04/13] Fixed issue where the wrong io's where used in point_propogate_displacement.py --- generalization/n100/building/point_propogate_displacement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generalization/n100/building/point_propogate_displacement.py b/generalization/n100/building/point_propogate_displacement.py index 944c307d..649330a0 100644 --- a/generalization/n100/building/point_propogate_displacement.py +++ b/generalization/n100/building/point_propogate_displacement.py @@ -47,7 +47,7 @@ def propagate_displacement_building_points(): ) arcpy.cartography.PropagateDisplacement( - in_features=Building_N100.calculate_point_values___points_going_into_propagate_displacement___n100_building.value, + in_features=Building_N100.point_propagate_displacement___points_after_propagate_displacement___n100_building.value, displacement_features=Building_N100.data_selection___displacement_feature___n100_building.value, adjustment_style="SOLID", ) From dcdd373619910b63fc1faa9f04123b383291c21f Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Thu, 12 Sep 2024 14:47:43 +0200 Subject: [PATCH 05/13] Updating docstring for partition_iterator.py and docstring_documentation.py --- .../general_tools/partition_iterator.py | 324 ++++++++++++++++-- documentation/docstring_documentation.py | 16 +- 2 files changed, 295 insertions(+), 45 deletions(-) diff --git a/custom_tools/general_tools/partition_iterator.py b/custom_tools/general_tools/partition_iterator.py index 2915f66a..7a031543 100644 --- a/custom_tools/general_tools/partition_iterator.py +++ b/custom_tools/general_tools/partition_iterator.py @@ -27,7 +27,213 @@ class PartitionIterator: - """THIS IS WORK IN PROGRESS NOT READY FOR USE YET""" + """ + A class designed to manage and execute custom partition-based logic on geospatial datasets. + + This class supports processing partitioned geospatial datasets, executing custom functions, and handling + input/output data across multiple iterations. Each function configured in `custom_functions` is expected + to work with specific input and output parameters, which are defined using tuples of (alias, type). These + parameters allow the class to automatically handle paths to input and output datasets. + + **Alias and Type Terminology:** + + - **Alias:** A named reference to a dataset. Each alias represents a specific dataset that will be used + as an input during partitioning. Aliases allow the class to organize and reference datasets + consistently across different logics. + + - **Type:** Each alias can have multiple types, indicating different versions the dataset can be accessed during + partitioning process. A type holds a path. This makes it so that if you run multiple logics in partition iterator + you can still access any of the outputs at any point for each alias. + + **Setting Up `alias_path_data` and `alias_path_outputs`:** + + - `alias_path_data`: A dictionary used to define the input datasets for the class. Each key is an alias, + and its value is a list of tuples where the first element is a type, and the second element is the path to the dataset. + The type of must be either 'input', 'context', 'reference' in `alias_path_data`. In a sense alias_path_data + works to load in all the data you are going to use in an instance of partition iterator. + So if you have different logics using inputs all the data used by all logics are entered in alias_path_data. + + Example: + ```python + alias_path_data = { + "building_points": [("input", "path_to_building_points")], + "river": [("reference", "path_to_river")] + } + ``` + + - `alias_path_outputs`: A dictionary used to define the output datasets for the class. Each key is an alias, + and its value is a list of tuples where the first element is the output type (e.g., 'processed_output') and + the second element is the path where the output should be saved. This means that you can have multiple outputs + for each alias. + + Example: + ```python + alias_path_outputs = { + "building_points": [("processed_output", "path_to_processed_building_points")] + } + ``` + + **Important: Reserved Types for Alias:** + + The following types are reserved for use by the class and should not be used to create new output types in logic configs: + - **"input"**: Used for input datasets provided to the class. Is the focus of the processing. If you want + - **"context"**: Represents context data used during processing, data of this type is + - **"reference"**: Represents reference datasets that are not directly processed but are required for context. + - **"input_copy"**: Internal type used to represent a copy of the input data. + - **"context_copy"**: Internal type used to represent a copy of the context data. + + New types should only be created during the partitioning process for outputs, and they should not use any + of the reserved types listed above. + + **Custom Function Configuration:** + + - The `custom_functions` parameter is a list of function configurations, where each configuration describes + a custom function (standalone or a method of a class) to be executed during the partitioning process. + - The input and output parameters for each function are defined as tuples of `(alias, type)`. The alias refers + to a named dataset, while the type specifies whether the dataset is used as an input or an output. + - Custom functions must be decorated with the `partition_io_decorator` to mark which parameters are inputs + and which are outputs. Only parameters marked as inputs or outputs will be managed by this system. + - Outputs can create new types associated with an alias, and these new types will have paths dynamically + generated by the class. These new types can then be used as inputs for other functions in subsequent + iterations. + - The class itself can write any existing alias to its final output, but it cannot create new types for + the alias. + + **Important Notes:** + + - Any function (or class method) included in `custom_functions` must be decorated with the `partition_io_decorator`. + For class methods, this means the method being used in `custom_functions`, not the entire class, must be decorated. + - During processing, the class dynamically resolves the paths for both input and output datasets based on the + `(alias, type)` tuples. New paths for outputs will be created automatically, and these outputs can then be used + in future iterations as inputs with the corresponding alias and type. + - Multiple outputs for a single alias can exist, but only previously defined types can be used as inputs for + the class’s output. + + Args: + alias_path_data (Dict[str, Tuple[str, str]]): + A dictionary where the key is an alias (representing a dataset), and the value is a tuple containing + the type of data (e.g., 'input', 'context') and the path to the dataset. + alias_path_outputs (Dict[str, Tuple[str, str]]): + A dictionary where the key is an alias (representing a dataset), and the value is a tuple containing + the type of output and the path where the results should be saved. + root_file_partition_iterator (str): + The base path for intermediate outputs generated during the partitioning process. + custom_functions (list, optional): + A list of configurations for the custom functions that will be executed during the partitioning process. + Each function must be configured with the `partition_io_decorator` and have its input/output parameters + specified. + dictionary_documentation_path (str, optional): + The path where documentation related to the partitioning process (e.g., JSON logs) will be stored. + feature_count (str, optional): + The maximum number of features allowed in each partition. Default is "15000". + partition_method (Literal['FEATURES', 'VERTICES'], optional): + The method used to create partitions, either by the number of features ('FEATURES') or vertices ('VERTICES'). + Default is 'FEATURES'. + search_distance (str, optional): + The distance within which context features are selected relative to input features. Default is '500 Meters'. + context_selection (bool, optional): + Whether to enable context feature selection based on proximity to input features. Default is True. + delete_final_outputs (bool, optional): + Whether to delete existing final outputs before starting the partitioning process. Default is True. + safe_output_final_cleanup (bool, optional): + Whether to enable safe deletion of outputs during cleanup. Default is True. + object_id_field (str, optional): + The field representing the object ID used during partitioning. Default is "OBJECTID". + + Methods: + run(): + Orchestrates the entire partitioning process, including input data preparation, partition creation, + custom function execution, and final output generation. + partition_iteration(): + Handles the iteration over data partitions, executing custom logic for each partition. + execute_custom_functions(): + Executes the custom functions with resolved input/output paths for the current iteration. + find_io_params_custom_logic(object_id: int): + Resolves input/output paths for custom functions based on the current partition. + ... + + Examples: + Example 1: + + ```python + inputs = { + "building_points": [ + "input", + "path_to_building_points" + ], + "river": [ + "reference", + "path_to_river" + ] + } + + outputs = { + "building_points": [ + "processed_points", + "path_to_output_points" + ] + } + + select_hospitals_config = { + "func": custom_arcpy.select_attribute_and_make_permanent_feature, + "params": { + "input_layer": ("building_points", "input"), + "output_name": ("building_points", "hospitals_selection"), + "expression": "symbol_val IN (1, 2, 3)", + } + } + + partition_iterator = PartitionIterator( + alias_path_data=inputs, + alias_path_outputs=outputs, + custom_functions=[select_hospitals_config], + root_file_partition_iterator="root_path", + scale="1:50000", + dictionary_documentation_path="documentation_path", + ) + + partition_iterator.run() + ``` + + Example 2: + + ```python + inputs = { + "land_cover": [ + "input", + "path_to_land_cover" + ] + } + + outputs = { + "land_cover": [ + "processed_land_cover", + "path_to_processed_land_cover" + ] + } + + process_land_cover = { + "class": LandCoverProcessor, + "method": "run", + "params": { + "input_land_cover": ("land_cover", "input"), + "output_processed_land_cover": ("land_cover", "processed_land_cover"), + } + } + + partition_iterator = PartitionIterator( + alias_path_data=inputs, + alias_path_outputs=outputs, + custom_functions=[process_land_cover], + root_file_partition_iterator="root_path", + scale="1:50000", + dictionary_documentation_path="documentation_path", + ) + + partition_iterator.run() + ``` + + """ # Class-level constants PARTITION_FIELD = "partition_select" @@ -40,7 +246,6 @@ def __init__( ], alias_path_outputs: Dict[str, Tuple[str, str]], root_file_partition_iterator: str, - scale: str, custom_functions=None, dictionary_documentation_path: str = None, feature_count: str = "15000", @@ -319,9 +524,9 @@ def create_directory_json_documentation( What: Creates a directory at the given root_path for the target_dir. Args: - root_path: The root directory where dir_name will be located - dir_name: The target where the created directory should be placed - iteration: Boolean flag indicating if the iteration_documentation should be added + root_path (str): The root directory where dir_name will be located + dir_name (str): The target where the created directory should be placed + iteration (bool): Boolean flag indicating if the iteration_documentation should be added Returns: str: A string containing the absolute path of the created directory. """ @@ -345,17 +550,17 @@ def write_data_to_json( data: dict, file_path: str, file_name: str, - object_id=None, + object_id: int = None, ) -> None: """ What: Writes dictionary into a json file. Args: - data: The data to write. - file_path: The complete path (directory+file_name) where the file should be created - file_name: The name of the file to create - object_id: If provided, object_id will also be part of the file name. + data (dict): The data to write. + file_path (str): The complete path (directory+file_name) where the file should be created + file_name (str): The name of the file to create + object_id (int): If provided, object_id will also be part of the file name. """ if object_id: @@ -375,19 +580,19 @@ def export_dictionaries_to_json( final_outputs: dict = None, file_name: str = None, iteration: bool = False, - object_id=None, + object_id: int = None, ) -> None: """ What: Handles the export of alias type data and final outputs into separate json files. Args: - file_path: The complete file path where to create the output directories. - alias_type_data: The alias type data to export. - final_outputs: The final outputs data to export. - file_name: The name of the file to create - iteration: Boolean flag indicating if the iteration_documentation should be added - object_id: Object ID to be included in the file name if it's an iteration (`iteration==True`). If `None`, will not be used. + file_path (str): The complete file path where to create the output directories. + alias_type_data (dict): The alias type data to export. + final_outputs (dict): The final outputs data to export. + file_name (str): The name of the file to create + iteration (bool): Boolean flag indicating if the iteration_documentation should be added + object_id (int): Object ID to be included in the file name if it's an iteration (`iteration==True`). If `None`, will not be used. """ if file_path is None: @@ -838,7 +1043,7 @@ def track_iteration_time(self, object_id, inputs_present_in_partition): print(f"Current runtime: {formatted_total_runtime}") print(f"Estimated remaining time: {formatted_estimated_remaining_time}") - def find_io_params_custom_logic(self, object_id): + def find_io_params_custom_logic(self, object_id: int): """ What: Find and resolve the input and output (IO) parameters for custom logic functions. @@ -850,7 +1055,7 @@ def find_io_params_custom_logic(self, object_id): input and output refers to the parameters of the custom_functions and not alias types. Args: - object_id: The identifier for the object that the IO parameters will be associated with. + object_id (int): The identifier for the object that the IO parameters will be associated with. """ for custom_func in self.custom_functions: if "class" in custom_func: # Class method @@ -875,7 +1080,7 @@ def find_io_params_custom_logic(self, object_id): object_id=object_id, ) - def resolve_io_params(self, params, custom_func, object_id): + def resolve_io_params(self, params: list, custom_func: dict, object_id: int): """ What: Resolve the paths for input or output parameters of custom functions. @@ -886,9 +1091,9 @@ def resolve_io_params(self, params, custom_func, object_id): It uses a helper function `resolve_param` to recursively resolve each parameter's path. Args: - params: A list of parameters to be resolved. - custom_func: The custom function containing the parameters to resolve. - object_id: The identifier for the object related to these parameters. + params (list): A list of parameters to be resolved. + custom_func (dict): The custom function containing the parameters to resolve. + object_id (int): The identifier for the object related to these parameters. """ def resolve_param(param_info): @@ -915,7 +1120,7 @@ def resolve_param(param_info): else: custom_func["params"][param] = resolved_paths - def _handle_tuple_param(self, param_info, object_id) -> str: + def _handle_tuple_param(self, param_info: tuple, object_id: int) -> str: """ What: Handle the resolution of parameters that are tuples of (alias, alias_type). @@ -936,12 +1141,13 @@ def _handle_tuple_param(self, param_info, object_id) -> str: they are tracked in `self.types_to_update` to ensure they are updated accordingly. Args: - param_info: A tuple containing an alias and its associated alias_type. - object_id: The identifier for the object related to these parameters. + param_info (tuple): A tuple containing an alias and its associated alias_type. + object_id (int): The identifier for the object related to these parameters. Returns: str: The resolved path based on the alias and alias_type. """ + alias, alias_type = param_info if alias_type in self.types_to_update: @@ -996,6 +1202,7 @@ def execute_custom_functions(self): For class methods, it separates parameters for class instantiation and method execution. For standalone functions, it directly prepares and resolves parameters for function execution. """ + for custom_func in self.custom_functions: resolved_params = {} # Initialize resolved_params @@ -1043,7 +1250,7 @@ def execute_custom_functions(self): # Execute the function with resolved parameters method(**resolved_params) - def resilient_execute_custom_functions(self, object_id): + def resilient_execute_custom_functions(self, object_id: int): """ What: Helper function to execute custom functions with retry logic to handle potential failures @@ -1051,7 +1258,6 @@ def resilient_execute_custom_functions(self, object_id): Args: object_id (int): The current object_id being processed (for logging). - error_log (dict): A dictionary to store error information for each iteration. """ max_retries = 50 @@ -1079,7 +1285,7 @@ def resilient_execute_custom_functions(self, object_id): self.save_error_log(self.error_log) raise Exception(error_message) - def append_iteration_to_final(self, alias, object_id): + def append_iteration_to_final(self, alias: str, object_id: int): """ What: Append the result of the current iteration to the final output for a given alias. @@ -1099,9 +1305,10 @@ def append_iteration_to_final(self, alias, object_id): ensuring the results from multiple iterations are combined into a single dataset. Args: - alias: A string representing the alias whose output data is to be updated. - object_id: The identifier for the current iteration. + alias (str): A string representing the alias whose output data is to be updated. + object_id (int): The identifier for the current iteration. """ + # Guard clause if alias doesn't exist in nested_final_outputs if alias not in self.nested_final_outputs: return @@ -1150,12 +1357,14 @@ def append_iteration_to_final(self, alias, object_id): ) @staticmethod - def delete_fields(feature_class_path, fields_to_delete): + def delete_fields(feature_class_path: str, fields_to_delete: list): """ - Deletes specified fields from the given feature class if they exist. + What: + Deletes specified fields from the given feature class if they exist. - :param feature_class_path: The path to the feature class. - :param fields_to_delete: A list of field names to delete. + Args: + feature_class_path (str): The path to the feature class. + fields_to_delete (list): A list of field names to delete. """ for field_name in fields_to_delete: try: @@ -1181,6 +1390,31 @@ def cleanup_final_outputs(self): self.delete_fields(feature_class_path, fields_to_delete) def partition_iteration(self): + """ + What: + Processes each data partition in multiple iterations. + + How: + - Iterates over partitions based on `object_id`, from 1 to `max_object_id`. + - For each iteration: + - Creates and resets dummy features. + - Deletes files from the previous iteration and clears the file path list. + - Backs up the original parameters for custom functions. + - Selects a partition feature and processes the inputs for that partition. + - If inputs are present: + - Processes context features. + - Finds input/output parameters for custom logic. + - Exports data from the iteration to JSON. + - Executes custom functions. + - Appends the output of the current iteration to the final outputs. + - Tracks and logs the time taken for each iteration. + + Why: + This method controls the process of iterating through partitions, executing custom logic, and + accumulating results. It ensures that each partition is processed individually and combined + into final outputs. + """ + aliases = self.nested_alias_type_data.keys() self.find_maximum_object_id() @@ -1234,6 +1468,23 @@ def partition_iteration(self): @timing_decorator def run(self): + """ + What: + Orchestrates the entire workflow for the class, from data preparation to final output generation. + + How: + - Initializes by unpacking input and output paths into `nested_alias_type_data` and `nested_final_outputs`. + - Exports the initialized dictionaries to JSON for future reference. + - Prepares the input data, including deleting old final outputs and processing the raw input data. + - Creates cartographic partitions to organize the data. + - Runs the partition iteration process by calling `partition_iteration`. + - Once iterations are complete, it cleans up final outputs, and logs any errors that occurred. + + Why: + This function acts as the main entry point for running the class. It handles the entire process from + initializing the data to running the partitioning iterations and finalizing the output. + """ + self.total_start_time = time.time() self.unpack_alias_path( alias_path=self.raw_input_data, target_dict=self.nested_alias_type_data @@ -1417,13 +1668,13 @@ def run(self): alias_path_outputs=outputs, custom_functions=[select_hospitals_config, polygon_processor_config], root_file_partition_iterator=Building_N100.iteration__partition_iterator__n100.value, - scale=env_setup.global_config.scale_n100, dictionary_documentation_path=Building_N100.iteration___json_documentation___building_n100.value, feature_count="800000", ) # Run the partition iterator partition_iterator.run() + # partition_iterator.find_io_params_custom_logic(1) # partition_iterator.find_io_params_custom_logic(3) @@ -1433,7 +1684,6 @@ def run(self): alias_path_outputs=outputs3, custom_functions=[buffer_displacement_config], root_file_partition_iterator=Building_N100.iteration__partition_iterator__n100.value, - scale=env_setup.global_config.scale_n100, dictionary_documentation_path=Building_N100.iteration___json_documentation___building_n100.value, feature_count="33000", ) diff --git a/documentation/docstring_documentation.py b/documentation/docstring_documentation.py index 4697f0d1..7fa5a323 100644 --- a/documentation/docstring_documentation.py +++ b/documentation/docstring_documentation.py @@ -31,7 +31,7 @@ def example_small_function_taking_args_with_returns(parameter: str) -> str: is what is required from the parameter, and what it returns. Args: - parameter: The string used in something. Important information regarding the parameter is informed about. + parameter (str): The string used in something. Important information regarding the parameter is informed about. Returns: str: A modified string used for something else. @@ -75,9 +75,9 @@ def example_of_callable_function( it is important to put focus on how the function is supposed to be used since it is intended to be called. Args: - parameter: A string that will be included in the printed or constructed message. - parameter_2: An integer representing the number of iterations. - parameter_3: A boolean that determines whether to print the message or construct it silently. + parameter (str): A string that will be included in the printed or constructed message. + parameter_2 (int): An integer representing the number of iterations. + parameter_3 (bool): A boolean that determines whether to print the message or construct it silently. Returns: str: The final value constructed in the loop. @@ -146,9 +146,9 @@ def __init__( Initializes the class with three parameters that will be processed in various utility methods. Args: - parameter: A string that will be processed in various class methods. - parameter_2: An integer used to control the number of iterations in loops. - parameter_3: A boolean controlling how parameters are processed or printed. + parameter (str): A string that will be processed in various class methods. + parameter_2 (int): An integer used to control the number of iterations in loops. + parameter_3 (bool): A boolean controlling how parameters are processed or printed. """ self.parameter = parameter self.parameter_2 = parameter_2 @@ -171,7 +171,7 @@ def example_small_function_taking_args_with_returns( and returns a modified string based on the current state of the class. Args: - additional_param: An additional string that will be combined with `self.parameter`. + additional_param (str): An additional string that will be combined with `self.parameter`. Returns: str: A modified string that combines `self.parameter` and `additional_param`. From 9b68c098a33e2690108024fe0ba970a0b0b625e9 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Thu, 12 Sep 2024 14:48:52 +0200 Subject: [PATCH 06/13] Removed unused param scale from partition_iterator.py --- custom_tools/general_tools/partition_iterator.py | 2 -- generalization/n100/building/data_preparation.py | 2 -- generalization/n100/building/point_displacement_with_buffer.py | 1 - .../n100/building/point_resolve_building_conflicts.py | 1 - 4 files changed, 6 deletions(-) diff --git a/custom_tools/general_tools/partition_iterator.py b/custom_tools/general_tools/partition_iterator.py index 7a031543..24016c84 100644 --- a/custom_tools/general_tools/partition_iterator.py +++ b/custom_tools/general_tools/partition_iterator.py @@ -261,7 +261,6 @@ def __init__( :param alias_path_data: A nested dictionary of input feature class paths with their aliases. :param root_file_partition_iterator: Base path for in progress outputs. - :param scale: Scale for the partitions. :param alias_path_outputs: A nested dictionary of output feature class for final results. :param feature_count: Feature count for cartographic partitioning. :param partition_method: Method used for creating cartographic partitions. @@ -278,7 +277,6 @@ def __init__( else: self.dictionary_documentation_path = dictionary_documentation_path - self.scale = scale self.search_distance = search_distance self.feature_count = feature_count self.partition_method = partition_method diff --git a/generalization/n100/building/data_preparation.py b/generalization/n100/building/data_preparation.py index 6e67a6c2..ca7dfa63 100644 --- a/generalization/n100/building/data_preparation.py +++ b/generalization/n100/building/data_preparation.py @@ -174,7 +174,6 @@ def data_selection(): alias_path_outputs=outputs, custom_functions=[process_data_validation], root_file_partition_iterator=Building_N100.data_preparation___begrensningskurve_base___n100_building.value, - scale=env_setup.global_config.scale_n100, dictionary_documentation_path=Building_N100.data_preparation___begrensingskurve_docu___building_n100.value, feature_count="5000", delete_final_outputs=False, @@ -238,7 +237,6 @@ def begrensningskurve_land_and_water_bodies(): alias_path_outputs=outputs, custom_functions=[process_begrensningskurve], root_file_partition_iterator=Building_N100.data_preparation___begrensningskurve_base___n100_building.value, - scale=env_setup.global_config.scale_n100, dictionary_documentation_path=Building_N100.data_preparation___begrensingskurve_docu___building_n100.value, feature_count="800000", ) diff --git a/generalization/n100/building/point_displacement_with_buffer.py b/generalization/n100/building/point_displacement_with_buffer.py index 56f6cbaf..8842b0a5 100644 --- a/generalization/n100/building/point_displacement_with_buffer.py +++ b/generalization/n100/building/point_displacement_with_buffer.py @@ -154,7 +154,6 @@ def buffer_displacement(): alias_path_outputs=outputs, custom_functions=[buffer_displacement_config], root_file_partition_iterator=Building_N100.point_displacement_with_buffer___root_file___n100_building.value, - scale=env_setup.global_config.scale_n100, dictionary_documentation_path=Building_N100.point_displacement_with_buffer___documentation___building_n100.value, feature_count="1400000", ) diff --git a/generalization/n100/building/point_resolve_building_conflicts.py b/generalization/n100/building/point_resolve_building_conflicts.py index 6ab6f21a..63b217a6 100644 --- a/generalization/n100/building/point_resolve_building_conflicts.py +++ b/generalization/n100/building/point_resolve_building_conflicts.py @@ -229,7 +229,6 @@ def resolve_building_conflicts(): alias_path_outputs=outputs, custom_functions=[resolve_building_conflicts_config], root_file_partition_iterator=Building_N100.point_resolve_building_conflicts___root_file___n100_building.value, - scale=env_setup.global_config.scale_n100, dictionary_documentation_path=Building_N100.point_resolve_building_conflicts___documentation___building_n100.value, feature_count="100000", ) From 6ecfde9179dcdbd585dd2d7e5ed1f20959676522 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Fri, 13 Sep 2024 14:41:09 +0200 Subject: [PATCH 07/13] Updated the docstring of partition_iterator.py --- .../general_tools/partition_iterator.py | 214 ++++++------------ 1 file changed, 73 insertions(+), 141 deletions(-) diff --git a/custom_tools/general_tools/partition_iterator.py b/custom_tools/general_tools/partition_iterator.py index 24016c84..a3330c49 100644 --- a/custom_tools/general_tools/partition_iterator.py +++ b/custom_tools/general_tools/partition_iterator.py @@ -28,12 +28,10 @@ class PartitionIterator: """ - A class designed to manage and execute custom partition-based logic on geospatial datasets. - - This class supports processing partitioned geospatial datasets, executing custom functions, and handling - input/output data across multiple iterations. Each function configured in `custom_functions` is expected - to work with specific input and output parameters, which are defined using tuples of (alias, type). These - parameters allow the class to automatically handle paths to input and output datasets. + This class handles processing of processing intense operations for large data sources using partitions. + Differentiating between which data is important and which are needed for context it processes as little + data as possible saving time. It then iterates over the partitions selecting data, and doing any amount + of logic on the selections, finally appending the defined result to an output file. **Alias and Type Terminology:** @@ -47,49 +45,47 @@ class PartitionIterator: **Setting Up `alias_path_data` and `alias_path_outputs`:** - - `alias_path_data`: A dictionary used to define the input datasets for the class. Each key is an alias, + - **`alias_path_data`**: A dictionary used to define the input datasets for the class. Each key is an alias, and its value is a list of tuples where the first element is a type, and the second element is the path to the dataset. The type of must be either 'input', 'context', 'reference' in `alias_path_data`. In a sense alias_path_data works to load in all the data you are going to use in an instance of partition iterator. So if you have different logics using inputs all the data used by all logics are entered in alias_path_data. - Example: - ```python - alias_path_data = { - "building_points": [("input", "path_to_building_points")], - "river": [("reference", "path_to_river")] - } - ``` - - `alias_path_outputs`: A dictionary used to define the output datasets for the class. Each key is an alias, + - **`alias_path_outputs`**: A dictionary used to define the output datasets for the class. Each key is an alias, and its value is a list of tuples where the first element is the output type (e.g., 'processed_output') and the second element is the path where the output should be saved. This means that you can have multiple outputs for each alias. - Example: - ```python - alias_path_outputs = { - "building_points": [("processed_output", "path_to_processed_building_points")] - } - ``` - **Important: Reserved Types for Alias:** The following types are reserved for use by the class and should not be used to create new output types in logic configs: - - **"input"**: Used for input datasets provided to the class. Is the focus of the processing. If you want - - **"context"**: Represents context data used during processing, data of this type is - - **"reference"**: Represents reference datasets that are not directly processed but are required for context. - - **"input_copy"**: Internal type used to represent a copy of the input data. - - **"context_copy"**: Internal type used to represent a copy of the context data. - - New types should only be created during the partitioning process for outputs, and they should not use any - of the reserved types listed above. + - **"input"**: + Used for input datasets provided to the class. Is the focus of the processing. If you in a config want to use + the partition selection of the original input data as an input this is the type which should be used. + - **"context"**: + Represents context data used during processing. This data is not central and will be selected based on proximity + to input data. If you in a config want to use the partition selection of the original context data as an input + this is the type which should be used. + - **"reference"**: + Represents reference datasets that are completely static and will not be processed in any way. + An example could be a lyrx file. + - **"input_copy"**: + Internal type used to hold a copy of the global input data. Should not be used in configs. + - **"context_copy"**: + Internal type used to hold a copy of the global context data. Should not be used in configs. + + New types should only be created during the outputs from configs in custom_functions, and they should not use any + of the reserved types listed above if you intend it to be a new output. So for instance an operation doing a buffer + on the "input" type of alias should not have an output using the "input" type, but for instance "buffer". If you + in the next config want to use the buffer output, use the "buffer" type for the alias as an input for the next config. **Custom Function Configuration:** - The `custom_functions` parameter is a list of function configurations, where each configuration describes a custom function (standalone or a method of a class) to be executed during the partitioning process. - - The input and output parameters for each function are defined as tuples of `(alias, type)`. The alias refers + - The input and output parameters needs to be defined using the partition_io_decorator. A logic can have multiple + input and output parameter. for each function are defined as tuples of `(alias, type)`. The alias refers to a named dataset, while the type specifies whether the dataset is used as an input or an output. - Custom functions must be decorated with the `partition_io_decorator` to mark which parameters are inputs and which are outputs. Only parameters marked as inputs or outputs will be managed by this system. @@ -139,100 +135,6 @@ class PartitionIterator: Whether to enable safe deletion of outputs during cleanup. Default is True. object_id_field (str, optional): The field representing the object ID used during partitioning. Default is "OBJECTID". - - Methods: - run(): - Orchestrates the entire partitioning process, including input data preparation, partition creation, - custom function execution, and final output generation. - partition_iteration(): - Handles the iteration over data partitions, executing custom logic for each partition. - execute_custom_functions(): - Executes the custom functions with resolved input/output paths for the current iteration. - find_io_params_custom_logic(object_id: int): - Resolves input/output paths for custom functions based on the current partition. - ... - - Examples: - Example 1: - - ```python - inputs = { - "building_points": [ - "input", - "path_to_building_points" - ], - "river": [ - "reference", - "path_to_river" - ] - } - - outputs = { - "building_points": [ - "processed_points", - "path_to_output_points" - ] - } - - select_hospitals_config = { - "func": custom_arcpy.select_attribute_and_make_permanent_feature, - "params": { - "input_layer": ("building_points", "input"), - "output_name": ("building_points", "hospitals_selection"), - "expression": "symbol_val IN (1, 2, 3)", - } - } - - partition_iterator = PartitionIterator( - alias_path_data=inputs, - alias_path_outputs=outputs, - custom_functions=[select_hospitals_config], - root_file_partition_iterator="root_path", - scale="1:50000", - dictionary_documentation_path="documentation_path", - ) - - partition_iterator.run() - ``` - - Example 2: - - ```python - inputs = { - "land_cover": [ - "input", - "path_to_land_cover" - ] - } - - outputs = { - "land_cover": [ - "processed_land_cover", - "path_to_processed_land_cover" - ] - } - - process_land_cover = { - "class": LandCoverProcessor, - "method": "run", - "params": { - "input_land_cover": ("land_cover", "input"), - "output_processed_land_cover": ("land_cover", "processed_land_cover"), - } - } - - partition_iterator = PartitionIterator( - alias_path_data=inputs, - alias_path_outputs=outputs, - custom_functions=[process_land_cover], - root_file_partition_iterator="root_path", - scale="1:50000", - dictionary_documentation_path="documentation_path", - ) - - partition_iterator.run() - ``` - """ # Class-level constants @@ -257,13 +159,53 @@ def __init__( object_id_field: str = "OBJECTID", ): """ - Initialize the PartitionIterator with input datasets for partitioning and processing. + Initializes the PartitionIterator with input and output datasets, custom functions, and configuration + for partitioning and processing. + + Args: + alias_path_data (Dict[str, Tuple[str, str]]): + A dictionary mapping aliases (names for datasets) to their type and dataset path. + Types should be either 'input', 'context', or 'reference'. This parameter sets up the data + that will be used across all iterations and logics. + + alias_path_outputs (Dict[str, Tuple[str, str]]): + A dictionary mapping aliases to their output type and path where results will be saved. + This defines how and where outputs are stored after each iteration of partitioning. + + root_file_partition_iterator (str): + The base path for storing intermediate outputs during partitioning. + + custom_functions (list, optional): + A list of configurations for custom functions that will be executed during the partitioning process. + Each function must be decorated with `partition_io_decorator` and have its input/output parameters + specified. Defaults to None. + + dictionary_documentation_path (str, optional): + The path where documentation related to the partitioning process (e.g., JSON logs) will be stored. + Defaults to None. - :param alias_path_data: A nested dictionary of input feature class paths with their aliases. - :param root_file_partition_iterator: Base path for in progress outputs. - :param alias_path_outputs: A nested dictionary of output feature class for final results. - :param feature_count: Feature count for cartographic partitioning. - :param partition_method: Method used for creating cartographic partitions. + feature_count (str, optional): + The maximum number of features allowed in each partition. Defaults to "15000". + + partition_method (Literal['FEATURES', 'VERTICES'], optional): + The method used to create partitions, either by feature count ('FEATURES') or vertices ('VERTICES'). + Defaults to 'FEATURES'. + + search_distance (str, optional): + The search distance used to select context features relative to the input features. Defaults to "500 Meters". + + context_selection (bool, optional): + Whether to enable context feature selection based on proximity to input features. Defaults to True. + + delete_final_outputs (bool, optional): + Whether to delete existing final outputs before starting the partitioning process. Defaults to True. + + safe_output_final_cleanup (bool, optional): + Whether to ensure outputs are deleted safely during cleanup by verifying their directory. + Defaults to True. + + object_id_field (str, optional): + The field representing the object ID used during partitioning. Defaults to "OBJECTID". """ # Raw inputs and initial setup @@ -1405,12 +1347,6 @@ def partition_iteration(self): - Exports data from the iteration to JSON. - Executes custom functions. - Appends the output of the current iteration to the final outputs. - - Tracks and logs the time taken for each iteration. - - Why: - This method controls the process of iterating through partitions, executing custom logic, and - accumulating results. It ensures that each partition is processed individually and combined - into final outputs. """ aliases = self.nested_alias_type_data.keys() @@ -1477,10 +1413,6 @@ def run(self): - Creates cartographic partitions to organize the data. - Runs the partition iteration process by calling `partition_iteration`. - Once iterations are complete, it cleans up final outputs, and logs any errors that occurred. - - Why: - This function acts as the main entry point for running the class. It handles the entire process from - initializing the data to running the partitioning iterations and finalizing the output. """ self.total_start_time = time.time() From 26cfdb57bac3734042a6154edea0f2e6e10e3098 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 14 Sep 2024 15:39:11 +0200 Subject: [PATCH 08/13] Fixed use of wrong input file from removing_overlapping_polygons_and_points logic for building polygons. --- generalization/n100/building/finalizing_buildings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generalization/n100/building/finalizing_buildings.py b/generalization/n100/building/finalizing_buildings.py index 2635e3b9..d4493939 100644 --- a/generalization/n100/building/finalizing_buildings.py +++ b/generalization/n100/building/finalizing_buildings.py @@ -101,14 +101,14 @@ def building_polygons_to_line(): Converts building polygons to lines """ arcpy.management.PolygonToLine( - in_features=Building_N100.removing_overlapping_polygons_and_points___building_polygons_not_intersecting_church_hospitals___n100_building.value, + in_features=Building_N100.removing_overlapping_polygons_and_points___polygons_NOT_intersecting_road_buffers___n100_building.value, out_feature_class=Building_N100.finalizing_buildings___polygon_to_line___n100_building.value, neighbor_option="IDENTIFY_NEIGHBORS", ) arcpy.analysis.SpatialJoin( target_features=Building_N100.finalizing_buildings___polygon_to_line___n100_building.value, - join_features=Building_N100.removing_overlapping_polygons_and_points___building_polygons_not_intersecting_church_hospitals___n100_building.value, + join_features=Building_N100.removing_overlapping_polygons_and_points___polygons_NOT_intersecting_road_buffers___n100_building.value, out_feature_class=Building_N100.finalizing_buildings___polygon_to_line_joined_fields___n100_building.value, match_option="SHARE_A_LINE_SEGMENT_WITH", ) @@ -150,7 +150,7 @@ def assigning_final_file_names(): ) arcpy.management.CopyFeatures( - Building_N100.removing_overlapping_polygons_and_points___building_polygons_not_intersecting_church_hospitals___n100_building.value, + Building_N100.removing_overlapping_polygons_and_points___polygons_NOT_intersecting_road_buffers___n100_building.value, Building_N100.Grunnriss.value, ) From 6377d8fbd6b015f5a0455181914440520e2f1a3c Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Mon, 16 Sep 2024 08:50:34 +0200 Subject: [PATCH 09/13] Update to docstring. --- .../general_tools/partition_iterator.py | 44 +------------------ 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/custom_tools/general_tools/partition_iterator.py b/custom_tools/general_tools/partition_iterator.py index a3330c49..4b16fae1 100644 --- a/custom_tools/general_tools/partition_iterator.py +++ b/custom_tools/general_tools/partition_iterator.py @@ -163,49 +163,7 @@ def __init__( for partitioning and processing. Args: - alias_path_data (Dict[str, Tuple[str, str]]): - A dictionary mapping aliases (names for datasets) to their type and dataset path. - Types should be either 'input', 'context', or 'reference'. This parameter sets up the data - that will be used across all iterations and logics. - - alias_path_outputs (Dict[str, Tuple[str, str]]): - A dictionary mapping aliases to their output type and path where results will be saved. - This defines how and where outputs are stored after each iteration of partitioning. - - root_file_partition_iterator (str): - The base path for storing intermediate outputs during partitioning. - - custom_functions (list, optional): - A list of configurations for custom functions that will be executed during the partitioning process. - Each function must be decorated with `partition_io_decorator` and have its input/output parameters - specified. Defaults to None. - - dictionary_documentation_path (str, optional): - The path where documentation related to the partitioning process (e.g., JSON logs) will be stored. - Defaults to None. - - feature_count (str, optional): - The maximum number of features allowed in each partition. Defaults to "15000". - - partition_method (Literal['FEATURES', 'VERTICES'], optional): - The method used to create partitions, either by feature count ('FEATURES') or vertices ('VERTICES'). - Defaults to 'FEATURES'. - - search_distance (str, optional): - The search distance used to select context features relative to the input features. Defaults to "500 Meters". - - context_selection (bool, optional): - Whether to enable context feature selection based on proximity to input features. Defaults to True. - - delete_final_outputs (bool, optional): - Whether to delete existing final outputs before starting the partitioning process. Defaults to True. - - safe_output_final_cleanup (bool, optional): - Whether to ensure outputs are deleted safely during cleanup by verifying their directory. - Defaults to True. - - object_id_field (str, optional): - The field representing the object ID used during partitioning. Defaults to "OBJECTID". + See class docstring. """ # Raw inputs and initial setup From 8c1642f622085f375f5a21eebba98ee0782a5188 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Mon, 16 Sep 2024 08:50:46 +0200 Subject: [PATCH 10/13] Added docstring --- .../general_tools/line_to_buffer_symbology.py | 22 +++-- .../building/buffer_displacement.py | 98 +++++++++++++++++-- 2 files changed, 104 insertions(+), 16 deletions(-) diff --git a/custom_tools/general_tools/line_to_buffer_symbology.py b/custom_tools/general_tools/line_to_buffer_symbology.py index b7379ca8..d7843f65 100644 --- a/custom_tools/general_tools/line_to_buffer_symbology.py +++ b/custom_tools/general_tools/line_to_buffer_symbology.py @@ -21,13 +21,11 @@ def __init__( root_file: str = None, ): """ - Initializes the LineToBufferSymbology class with the specified parameters. + What: + Initializes the LineToBufferSymbology class with the specified parameters. - :param input_road_lines: Path to the input road lines. - :param sql_selection_query: Dictionary containing SQL queries and associated buffer widths. - :param output_road_buffer: Path to save the output road buffer. - :param buffer_factor: Multiplicative factor to adjust buffer widths, avoid using 0. - :param fixed_buffer_addition: Additional fixed width to add to buffer widths. + Args: + See class docstring. """ self.input_road_lines = input_road_lines self.sql_selection_query = sql_selection_query @@ -50,9 +48,17 @@ def __init__( "buffer_factor should not be 0 to avoid non-buffer creation." ) - def selecting_different_road_lines(self, sql_query, selection_output_name): + def selecting_different_road_lines( + self, sql_query: str, selection_output_name: str + ): """ - Selects road lines based on the provided SQL query and creates a feature layer. + What: + Selects road lines based on the provided SQL query and creates a feature layer or a permanent feature class + depending on the `write_work_files_to_memory` flag. + + Args: + sql_query (str): The SQL query string used to select the road lines. + selection_output_name (str): The name for the output feature layer or file. """ if self.write_work_files_to_memory: diff --git a/custom_tools/generalization_tools/building/buffer_displacement.py b/custom_tools/generalization_tools/building/buffer_displacement.py index 1fdfca76..18670788 100644 --- a/custom_tools/generalization_tools/building/buffer_displacement.py +++ b/custom_tools/generalization_tools/building/buffer_displacement.py @@ -5,14 +5,58 @@ from env_setup import environment_setup from constants.n100_constants import N100_Symbology, N100_SQLResources, N100_Values from file_manager.n100.file_manager_buildings import Building_N100 -from input_data import input_n100 -from custom_tools.general_tools import custom_arcpy from custom_tools.general_tools.line_to_buffer_symbology import LineToBufferSymbology from custom_tools.general_tools.polygon_processor import PolygonProcessor from custom_tools.decorators.partition_io_decorator import partition_io_decorator class BufferDisplacement: + """ + This class handles the displacement of building points relative to road buffers based on specified buffer increments. + It processes multiple features, mainly focusing on roads taking into account varied symbology width for roads, + displacing building points away from roads and other barriers, while iteratively calculating buffer increments. + + **Buffer Displacement Logic:** + + - **Road Buffers:** Buffers are created for road features based on a factor and a fixed buffer addition value. + - **Building Points:** Building points are processed by converting them into polygons, erasing any that overlap + with the generated road buffers or other barrier features, and then converting them back into points. + - **Miscellaneous Objects:** Optional miscellaneous features can be buffered along with roads, and merged into a + unified barrier layer to control building point displacement. + + **Increment Calculation:** + The buffer increments are calculated based on the largest road dimension and building symbol dimensions. The process + ensures that buffers gradually increase up to the target displacement value, while adhering to a set tolerance level. + + **Work File Management:** + The class can optionally store working files either in memory or on disk, depending on the parameters provided. It + can also automatically clean up working files if unless keep_work_files is set to False. + + Args: + input_road_lines (str): + Path to the input road line features to be buffered. + input_building_points (str): + Path to the input building points that will be displaced. + output_building_points (str): + Path where the final displaced building points will be stored. + sql_selection_query (dict): + A dictionary where the keys are SQL queries to select from the input road features based on attribute values, + and the values are the corresponding buffer widths representing the road symbology. + root_file (str): + The base path for storing work files, required if `write_work_files_to_memory` is False or + `keep_work_files` is True. + buffer_displacement_meter (int, optional): + The buffer displacement distance in meters. Default is 30 meters. + building_symbol_dimensions (Dict[int, Tuple[int, int]], optional): + A dictionary mapping building symbols to their dimensions, used to ensure displacement calculations account for building size. + input_misc_objects (Dict[str, List[Union[str, int]]], optional): + A dictionary of miscellaneous objects that will also be buffered, where each entry includes the feature name and a buffer width. + write_work_files_to_memory (bool, optional): + If True, work files are written to memory. Default is True. + keep_work_files (bool, optional): + If True, work files are retained after the process is complete. Default is False. + """ + def __init__( self, input_road_lines: str, @@ -26,6 +70,13 @@ def __init__( write_work_files_to_memory: bool = True, keep_work_files: bool = False, ): + """ + Initialize the BufferDisplacement class with the necessary input data and configuration. + + Args: + See class docstring. + """ + self.input_road_lines = input_road_lines self.input_building_points = input_building_points self.sql_selection_query = sql_selection_query @@ -66,7 +117,9 @@ def __init__( self.working_files_list_2 = [] def initialize_work_file_location(self): - """Reset temporary file attributes.""" + """ + Determines the file location for temporary work files, either in memory or on disk, based on class parameters. + """ temporary_file = "in_memory\\" permanent_file = f"{self.root_file}_" if self.root_file is None: @@ -84,9 +137,10 @@ def initialize_work_file_location(self): else: self.file_location = permanent_file - def finding_dimensions(self, buffer_displacement_meter): + def finding_dimensions(self, buffer_displacement_meter: int): """ - Finds the smallest building symbol dimension and the largest road dimension. + Finds the smallest building symbol dimension and the largest road dimension to calculate the maximum + buffer tolerance and target displacement value to prevent loosing buildings due to large increases. """ if not self.building_symbol_dimensions: raise ValueError("building_symbol_dimensions is required.") @@ -104,7 +158,15 @@ def finding_dimensions(self, buffer_displacement_meter): self.target_value = self.largest_road_dimension + buffer_displacement_meter - def calculate_buffer_increments(self): + def calculate_buffer_increments(self) -> list: + """ + What: + Calculates incremental buffer steps based on the road and building dimensions and tolerance, ensuring that + buffer increments increase gradually until the target displacement value is reached. + + Returns: + list: A list of tuples where each tuple contains a buffer factor and the corresponding buffer addition. + """ iteration_buffer_factor = 0 found_valid_increment = False @@ -172,7 +234,13 @@ def process_buffer_factor( self, factor: Union[int, float], fixed_addition: Union[int, float] ): """ - Processes a single buffer factor by creating buffers and displacing points. + What: + Processes a single buffer factor, creating buffers for the roads and any miscellaneous features, and then + displaces the building points based on the calculated buffers. + + Args: + factor (Union[int, float]): The buffer factor to be applied. + fixed_addition (Union[int, float]): The fixed buffer addition value to be applied. """ factor_name = str(factor).replace(".", "_") fixed_addition_name = str(fixed_addition).replace(".", "_") @@ -257,15 +325,21 @@ def process_buffer_factor( def delete_working_files(self, *file_paths): """ Deletes multiple feature classes or files. + + Args: + *file_paths: Paths of files to be deleted. """ for file_path in file_paths: self.delete_feature_class(file_path) print(f"Deleted file: {file_path}") @staticmethod - def delete_feature_class(feature_class_path): + def delete_feature_class(feature_class_path: str): """ Deletes a feature class if it exists. + + Args: + feature_class_path (str): Path to the feature class to be deleted. """ if arcpy.Exists(feature_class_path): arcpy.management.Delete(feature_class_path) @@ -279,6 +353,10 @@ def delete_feature_class(feature_class_path): output_param_names=["output_building_points"], ) def run(self): + """ + Executes the buffer displacement process, running the calculations for buffer increments, applying buffers, + displacing building points, and writing the final output. + """ self.initialize_work_file_location() self.finding_dimensions(self.buffer_displacement_meter) self.calculate_buffer_increments() @@ -332,3 +410,7 @@ def run(self): keep_work_files=False, ) point_displacement.run() + + point_displacement.finding_dimensions(point_displacement.buffer_displacement_meter) + point_displacement.calculate_buffer_increments() + print(type(point_displacement.increments)) From e7520677ed3ed3760f9481fdfbf069d1feb27e9e Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Tue, 17 Sep 2024 13:50:41 +0200 Subject: [PATCH 11/13] Updated docstring and added typehint for polygon_processor.py --- .../general_tools/polygon_processor.py | 184 +++++++++++------- 1 file changed, 111 insertions(+), 73 deletions(-) diff --git a/custom_tools/general_tools/polygon_processor.py b/custom_tools/general_tools/polygon_processor.py index 1574da68..cedbc928 100644 --- a/custom_tools/general_tools/polygon_processor.py +++ b/custom_tools/general_tools/polygon_processor.py @@ -14,64 +14,32 @@ class PolygonProcessor: """ - Example - ------- - To use PolygonProcessor to convert building points to polygons: - - >>> building_symbol_dimensions = {1: (145, 145), 2: (195, 145)} - >>> polygon_processor = PolygonProcessor( - ... input_building_points="path/to/input/building_points", - ... output_polygon_feature_class="path/to/output/polygon_feature_class", - ... building_symbol_dimensions=building_symbol_dimensions, - ... symbol_field_name="symbol_type_field", - ... index_field_name="object_id_field" - ... ) - >>> polygon_processor.run() - - Processes building point data to generate polygon feature classes keeping the attributes. - - This class converts point geometries into polygon geometries based on given symbol dimensions. - It handles batch processing, joining additional fields, and cleaning up temporary data. - - Parameters - ---------- - input_building_points : str - The path to the input building points feature class. - output_polygon_feature_class : str - The path to the output polygon feature class. - building_symbol_dimensions : dict - A dictionary mapping symbol types to their width and height dimensions. - symbol_field_name : str - The name of the field in the input feature class that contains symbol type information. - index_field_name : str - The name of the field in the input feature class used for indexing during join operations. - - Attributes - ---------- - input_building_points : str - The input building points feature class. - output_polygon_feature_class : str - The output polygon feature class. - spatial_reference_system : SpatialReference - The spatial reference system used for the feature classes. - building_symbol_dimensions : dict - The dimensions for each symbol type. - symbol_field_name : str - The field name for symbol type. - index_field_name : str - The field name for indexing. - origin_id_field : str - The generated unique field name for joining operations. - IN_MEMORY_WORKSPACE : str - The in-memory workspace used for temporary storage. - TEMPORARY_FEATURE_CLASS_NAME : str - The name of the temporary feature class. - BATCH_PERCENTAGE : float - The percentage of the total data processed in each batch. - NUMBER_OF_SUBSETS : int - The number of subsets to divide the data into for processing. - PERCENTAGE_OF_CPU_CORES : float - The percentage of CPU cores to use for parallel processing. + What: + This class processes point data representing building locations to generate polygon feature classes, + using specified dimensions for each building symbol type. The result is a polygon feature class + with polygons sized according to the building's symbol type. + + How: + The class takes building points as input and using a dictionary where the key is the symbol_val and values + are the dimensions of the building symbology. This is used to create polygon geometries for each point. + Depending on the number of objects it is either run single process or parallel processing using batches. + The table information from the original data is kept using a join field to the output data. + + Why: + For some operations the geometric representation of building point symbology is needed. This class transforms + input building points to building polygons so that such operations can be done. + + Args: + input_building_points (str): + The path to the input feature class containing the building points. + output_polygon_feature_class (str): + The path where the output polygon feature class will be saved. + building_symbol_dimensions (dict): + A dictionary the key is the symbol_val and values are the dimensions of the building symbology. + symbol_field_name (str): + The field in the input feature class that contains the building symbol type. + index_field_name (str): + The field in the input feature class used for indexing during join operations. """ # Initialization @@ -83,6 +51,13 @@ def __init__( symbol_field_name, index_field_name, ): + """ + Initializes the PolygonProcessor with required inputs, including paths to the input and output feature + classes, symbol dimensions, and field names used in the process. + + Args: + See class docstring. + """ self.input_building_points = input_building_points self.output_polygon_feature_class = output_polygon_feature_class self.building_symbol_dimensions = building_symbol_dimensions @@ -101,7 +76,19 @@ def __init__( self.calculate_batch_params(input_building_points) # Utility Functions - def calculate_batch_params(self, input_building_points): + def calculate_batch_params(self, input_building_points: str): + """ + What: + Calculates the batch size and the number of subsets for processing based on the number of + input building points. + + How: + If the input dataset is small, processing is done in a single batch. For larger datasets, + it divides the data into subsets for more efficient parallel processing. + + Args: + input_building_points (str): The input dataset. + """ total_data_points = int( arcpy.GetCount_management(input_building_points).getOutput(0) ) @@ -115,7 +102,15 @@ def calculate_batch_params(self, input_building_points): self.BATCH_PERCENTAGE = 0.02 self.NUMBER_OF_SUBSETS = 5 - def generate_unique_field_name(self, dataset, base_name): + @staticmethod + def generate_unique_field_name(dataset: str, base_name: str): + """ + What: + Retrieves the existing field names, and makes sure the added field is unique. + Args: + dataset (str): The path to the dataset. + base_name (str): the base name of the added field + """ existing_field_names = [field.name for field in arcpy.ListFields(dataset)] unique_name = base_name while unique_name in existing_field_names: @@ -123,17 +118,22 @@ def generate_unique_field_name(self, dataset, base_name): return unique_name def setup_spatial_reference_and_origin_id(self): + """ + Sets up the spatial reference system and generates a unique field name for the origin ID, + which is used in join operations. + """ self.spatial_reference_system = arcpy.SpatialReference( environment_setup.project_spatial_reference ) self.origin_id_field = self.generate_unique_field_name( - self.input_building_points, "match_id" + dataset=self.input_building_points, base_name="match_id" ) @staticmethod - def convert_corners_to_wkt(polygon_corners): + def convert_corners_to_wkt(polygon_corners: list[tuple[float, float]]) -> str: """ - Converts a list of polygon corner coordinates to a Well-Known Text (WKT) string. + What: + Converts a list of polygon corner coordinates to a Well-Known Text (WKT) string. Args: polygon_corners (list): A list of tuples representing the coordinates of the polygon corners. Returns: @@ -143,9 +143,12 @@ def convert_corners_to_wkt(polygon_corners): return f"POLYGON (({coordinate_strings}))" # Core Processing Functions - def calculate_well_known_text_polygon(self, arguments): + def calculate_well_known_text_polygon( + self, arguments: tuple[int, float, float, int, int] + ) -> tuple[int, str]: """ - Generates the Well-Known Text (WKT) representation of a polygon based on input arguments. + What: + Generates the Well-Known Text (WKT) representation of a polygon based on input arguments. Args: arguments (tuple): A tuple containing index, x-coordinate, y-coordinate, object ID, and symbol type. Returns: @@ -172,8 +175,12 @@ def calculate_well_known_text_polygon(self, arguments): # Data Handling and Batch Processing def create_output_feature_class_if_not_exists(self): """ - Deletes and creates an output feature class. - This ensures that a fresh feature class is created each time the script runs. + What: + Creates a polygon feature output. + + How: + If an existing output feature class is found, it is deleted. A new feature class is then created + with the appropriate schema and spatial reference. """ if arcpy.Exists(self.output_polygon_feature_class): arcpy.management.Delete(self.output_polygon_feature_class) @@ -194,9 +201,16 @@ def create_output_feature_class_if_not_exists(self): field_type="LONG", ) - def process_data_in_batches(self, well_known_text_data): + def process_data_in_batches(self, well_known_text_data: list[tuple[int, str]]): """ - Processes data in batches and appends the results to the output feature class. + What: + Processes the data in batches, creating polygons from the input points and appending them + to the output feature class. + + How: + The input data is divided into subsets and processed in batches to avoid memory overload. + Temporary feature classes are created in-memory for intermediate results, which are appended + to the output feature class. Args: well_known_text_data (list): A list of tuples containing object IDs and their WKT polygons. """ @@ -239,7 +253,15 @@ def process_data_in_batches(self, well_known_text_data): ) arcpy.DeleteRows_management(temporary_feature_class) - def prepare_data_for_processing(self): + def prepare_data_for_processing(self) -> list[tuple[int, float, float, int, int]]: + """ + What: + Extracts the necessary fields from the input building points and prepares the data for processing. + + How: + Converts the input data into a NumPy array, which is easier to work with for batch processing. + Each record contains coordinates, object IDs, and symbol type information needed to create the polygons. + """ input_data_array = arcpy.da.FeatureClassToNumPyArray( self.input_building_points, ["SHAPE@X", "SHAPE@Y", self.index_field_name, self.symbol_field_name], @@ -256,7 +278,17 @@ def prepare_data_for_processing(self): ] return data_to_be_processed - def process_data(self, data_to_be_processed): + def process_data( + self, data_to_be_processed: list[tuple[int, float, float, int, int]] + ) -> list[tuple[int, str]]: + """ + What: + Processes the input data to generate the corresponding Well-Known Text (WKT) polygons. + + How: + The method uses either multiprocessing (for large datasets) or sequential processing (for smaller + datasets) to convert the building points into polygons based on their symbol dimensions. + """ total_data_points = len(data_to_be_processed) if total_data_points >= 10000: number_of_cores = int(cpu_count() * self.PERCENTAGE_OF_CPU_CORES) @@ -274,7 +306,7 @@ def process_data(self, data_to_be_processed): # Field Management and Cleanup def add_fields_with_join(self): """ - Joins fields from the input building points to the output polygon feature class. + Joins fields from the input building points to the generated polygon feature class using the unique ID field. """ # Add index to the join field in the output feature class arcpy.management.AddIndex( @@ -308,7 +340,13 @@ def delete_origin_id_field(self): ) def run(self): """ - Orchestrates the process of converting building points to polygons. + What: + Executes the entire process of converting building points into polygons, from spatial reference + setup to final field joins. + + How: + The method orchestrates all the other methods in the correct order, handling data preparation, + batch processing, and cleanup to generate the final polygon feature class. """ self.setup_spatial_reference_and_origin_id() From a6adb713cffa95fe1fd59559b1ce6939c4a5e2ee Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Tue, 17 Sep 2024 13:51:04 +0200 Subject: [PATCH 12/13] Updates to docstring --- .../decorators/partition_io_decorator.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/custom_tools/decorators/partition_io_decorator.py b/custom_tools/decorators/partition_io_decorator.py index 8e233726..d702efec 100644 --- a/custom_tools/decorators/partition_io_decorator.py +++ b/custom_tools/decorators/partition_io_decorator.py @@ -1,4 +1,22 @@ -def partition_io_decorator(input_param_names=None, output_param_names=None): +from typing import Callable, List, Optional + + +def partition_io_decorator( + input_param_names: Optional[List[str]] = None, + output_param_names: Optional[List[str]] = None, +) -> Callable: + """ + What: + A decorator that adds metadata about input and output parameters for partitioning logic functions. + + Args: + input_param_names (Optional[List[str]]): A list of input parameter names for the decorated function. + output_param_names (Optional[List[str]]): A list of output parameter names for the decorated function. + + Returns: + Callable: The decorated function with added metadata attributes for inputs and outputs. + """ + def decorator(func): setattr( func, From a40def5d55ff95ffc3075aa6a76e519cc01310cb Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Tue, 17 Sep 2024 13:51:32 +0200 Subject: [PATCH 13/13] Added run func --- input_data/input_n50.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/input_data/input_n50.py b/input_data/input_n50.py index cc15e4be..df32e96e 100644 --- a/input_data/input_n50.py +++ b/input_data/input_n50.py @@ -83,3 +83,7 @@ def check_paths(): print(f"Failed on {dataset}: {e}") else: print(f"Success on {dataset}") + + +if __name__ == "__main__": + check_paths()