From 957d8c2cba7b730360632daebc647e1cb6582865 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Mon, 18 Mar 2024 16:32:24 +0100 Subject: [PATCH 01/19] Introduce JSON output and revise building data preparation algorithm. The commit includes functionalities that allow saving of data to JSON files. Other changes include optimizing the function for creating dummy features and enhancing the run function with timing decorator and writing data to JSON before and after partition iteration. Additionally, the "input" field within the Building_N100 class was modified in several instances. --- .../partition_iterator_state_based.py | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 45740ef5..5e29bb69 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -1,11 +1,13 @@ import arcpy import os import random -from enum import Enum +import json import env_setup.global_config from env_setup import environment_setup from custom_tools import custom_arcpy +from custom_tools.timing_decorator import timing_decorator + from input_data import input_n50 from file_manager.n100.file_manager_buildings import Building_N100 @@ -186,6 +188,10 @@ def create_dummy_features(self, types_to_include=["input", "context"]): path=dummy_feature_path, ) + def write_data_to_json(self, file_name): + with open(file_name, "w") as file: + json.dump(self.data, file, indent=4) + def pre_iteration(self): """ Determine the maximum OBJECTID for partitioning. @@ -422,12 +428,7 @@ def partition_iteration(self): max_object_id = self.pre_iteration() self.delete_existing_outputs() - self.create_dummy_features( - types_to_include=[ - "input", - "context", - ] - ) + self.create_dummy_features(types_to_include=["input", "context"]) for alias in aliases: self.delete_iteration_files(*self.iteration_file_paths) self.iteration_file_paths.clear() @@ -491,12 +492,21 @@ def partition_iteration(self): # self.delete_iteration_files(*self.iteration_file_paths) # print(f"Finished iteration {object_id}") + @timing_decorator def run(self): self.prepare_input_data() self.create_cartographic_partitions() + self.write_data_to_json( + Building_N100.iteration___json_documentation_before___building_n100.value + ) + self.partition_iteration() + self.write_data_to_json( + Building_N100.iteration___json_documentation_after___building_n100.value + ) + if __name__ == "__main__": environment_setup.main() @@ -510,7 +520,7 @@ def run(self): Building_N100.data_preparation___matrikkel_bygningspunkt___n100_building.value, ], building_polygons: [ - "context", + "input", input_n50.Grunnriss, ], } @@ -520,10 +530,10 @@ def run(self): "input", Building_N100.iteration__partition_iterator_final_output_points__n100.value, ], - # building_polygons: [ - # "output", - # Building_N100.iteration__partition_iterator_final_output_polygons__n100.value, - # ], + building_polygons: [ + "input", + Building_N100.iteration__partition_iterator_final_output_polygons__n100.value, + ], } # Instantiate PartitionIterator with necessary parameters From be04f6e8ce5a68be7100a1c1423b9e812ca4800c Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Wed, 20 Mar 2024 09:22:58 +0100 Subject: [PATCH 02/19] Refactor partition_iterator_state_based.py for data integration The changes remove redundant code in the constructor of the partition iterator and streamline initial data integration. The alias_path_data is now processed once during class initialization, instead of multiple times in different methods. The custom function specification processing was also commented out for further review. --- .../partition_iterator_state_based.py | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 5e29bb69..a812de25 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -44,12 +44,7 @@ def __init__( :param partition_method: Method used for creating cartographic partitions. """ self.data = {} - for alias, info in alias_path_data.items(): - type_info, path_info = info - if alias not in self.data: - self.data[alias] = {} - self.data[alias][type_info] = path_info - + self.alias_path_data = alias_path_data self.root_file_partition_iterator = root_file_partition_iterator self.scale = scale self.output_feature_class = alias_path_outputs @@ -67,23 +62,22 @@ def __init__( self.max_object_id = None self.final_append_feature = None - def integrate_initial_data(self, alias_path_data, custom_function_specs): + def integrate_initial_data(self, alias_path_data): # Process initial alias_path_data for inputs and outputs - for alias, (type_info, path_info) in alias_path_data.items(): - self.update_alias_state( - alias=alias, - type_info=type_info, - path=path_info, - ) + for alias, info in alias_path_data.items(): + type_info, path_info = info + if alias not in self.data: + self.data[alias] = {} + self.data[alias][type_info] = path_info - for func_name, specs in custom_function_specs.items(): - for alias, types in specs.items(): - for type_info in types: - self.update_alias_state( - alias=alias, - type_info=type_info, - path=None, - ) + # for func_name, specs in custom_function_specs.items(): + # for alias, types in specs.items(): + # for type_info in types: + # self.update_alias_state( + # alias=alias, + # type_info=type_info, + # path=None, + # ) def update_alias_state(self, alias, type_info, path=None): if alias not in self.data: @@ -494,6 +488,7 @@ def partition_iteration(self): @timing_decorator def run(self): + self.integrate_initial_data(self.alias_path_data) self.prepare_input_data() self.create_cartographic_partitions() From 90b61f5ee5adfe29dd5d352c41f676023de873be Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Fri, 22 Mar 2024 10:07:07 +0100 Subject: [PATCH 03/19] Implement unpacking and append functionalities in partition iterator The commit integrates unpacking `alias_path_outputs` into the partition iterator's function. The code also now favors the appending of each iteration result to the 'final_outputs'. The iteration results are stored per alias and type for easier data retrieval. --- .../partition_iterator_state_based.py | 158 ++++++++++++------ 1 file changed, 106 insertions(+), 52 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index a812de25..14681f4c 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -43,8 +43,11 @@ def __init__( :param feature_count: Feature count for cartographic partitioning. :param partition_method: Method used for creating cartographic partitions. """ + self.data = {} self.alias_path_data = alias_path_data + self.alias_path_outputs = alias_path_outputs or {} + print("\nInitializing with alias_path_outputs = ", alias_path_outputs) self.root_file_partition_iterator = root_file_partition_iterator self.scale = scale self.output_feature_class = alias_path_outputs @@ -55,9 +58,10 @@ def __init__( ) self.custom_functions = custom_functions or [] self.iteration_file_paths = [] - self.final_append_features = {} + self.final_outputs = {} self.search_distance = search_distance self.object_id_field = object_id_field + self.input_data_copy = None self.max_object_id = None self.final_append_feature = None @@ -79,6 +83,20 @@ def integrate_initial_data(self, alias_path_data): # path=None, # ) + def unpack_alias_path_outputs(self, alias_path_outputs): + self.final_outputs = {} + for alias, info in alias_path_outputs.items(): + type_info, path_info = info + if alias not in self.final_outputs: + self.final_outputs[alias] = {} + self.final_outputs[alias][type_info] = path_info + print("\nUnpacking alias_path_outputs = ", alias_path_outputs) + + def integrate_results(self): + for alias, types in self.final_outputs.items(): + for type_info, final_output_path in types.items(): + iteration_output_path = self.data[alias][type_info] + def update_alias_state(self, alias, type_info, path=None): if alias not in self.data: self.data[alias] = {} @@ -124,6 +142,14 @@ def delete_feature_class(self, feature_class_path, alias=None, output_type=None) else: print(f"Deleted feature class: {feature_class_path}") + def delete_final_outputs(self): + """Deletes all final output files if they exist.""" + for alias in self.final_outputs: + for _, output_file_path in self.final_outputs[alias].items(): + if arcpy.Exists(output_file_path): + arcpy.management.Delete(output_file_path) + print(f"Deleted file: {output_file_path}") + def delete_existing_outputs(self): for alias, output_info in self.output_feature_class.items(): output_type, output_path = output_info @@ -143,6 +169,7 @@ def delete_iteration_files(self, *file_paths): """Deletes multiple feature classes or files. Detailed alias and output_type logging is not available here.""" for file_path in file_paths: self.delete_feature_class(file_path) + print(f"Deleted file: {file_path}") @staticmethod def create_feature_class(full_feature_path, template_feature): @@ -288,6 +315,13 @@ def process_input_features( return None, False if "input" in self.data[alias]: + input_data_copy = f"{self.root_file_partition_iterator}_{alias}_input_copy" + self.update_alias_state( + alias=alias, + type_info="input", + path=input_data_copy, + ) + input_path = self.data[alias]["input"] input_features_partition_selection = ( f"in_memory/{alias}_partition_base_select_{self.scale}" @@ -366,6 +400,12 @@ def process_input_features( schema_type="NO_TEST", ) + self.update_alias_state( + alias=alias, + type_info="input", + path=iteration_append_feature, + ) + print( f"iteration partition {input_features_partition_context_selection} appended to {iteration_append_feature}" ) @@ -417,14 +457,66 @@ def _process_context_features_and_others( else: self.process_context_features(alias, iteration_partition) + def append_iteration_to_final(self, alias): + # Guard clause if alias doesn't exist in final_outputs + if alias not in self.final_outputs: + return + + # For each type under current alias, append the result of the current iteration + for type_info, final_output_path in self.final_outputs[alias].items(): + input_feature_class = self.data[alias][type_info] + + partition_target_selection = ( + f"in_memory/{alias}_{type_info}_partition_target_selection_{self.scale}" + ) + self.iteration_file_paths.append(partition_target_selection) + self.iteration_file_paths.append(input_feature_class) + + # Apply feature selection + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=input_feature_class, + expression=f"{self.PARTITION_FIELD} = 1", + output_name=partition_target_selection, + ) + + # Number of features before append/copy + orig_num_features = ( + int(arcpy.GetCount_management(final_output_path).getOutput(0)) + if arcpy.Exists(final_output_path) + else 0 + ) + print(f"\nNumber of features originally in the file: {orig_num_features}") + + if not arcpy.Exists(final_output_path): + arcpy.management.CopyFeatures( + in_features=partition_target_selection, + out_feature_class=final_output_path, + ) + + else: + arcpy.management.Append( + inputs=partition_target_selection, + target=final_output_path, + schema_type="NO_TEST", + ) + + # Number of features after append/copy + new_num_features = int( + arcpy.GetCount_management(final_output_path).getOutput(0) + ) + print(f"\nNumber of features after append/copy: {new_num_features}") + + def _append_iteration_to_final_and_others(self, alias): + self.append_iteration_to_final(alias) + def partition_iteration(self): aliases = self.data.keys() max_object_id = self.pre_iteration() - self.delete_existing_outputs() + # self.delete_existing_outputs() self.create_dummy_features(types_to_include=["input", "context"]) - for alias in aliases: - self.delete_iteration_files(*self.iteration_file_paths) + + self.delete_iteration_files(*self.iteration_file_paths) self.iteration_file_paths.clear() for object_id in range(1, max_object_id + 1): @@ -439,56 +531,20 @@ def partition_iteration(self): self._process_context_features_and_others( aliases, iteration_partition, object_id ) - else: + if inputs_present_in_partition: for alias in aliases: - self.delete_iteration_files(*self.iteration_file_paths) - - # # Process each alias after custom functions - # for alias in self.data.keys(): - # if inputs_present_in_partition: - # # Retrieve the output path for the current alias - # output_path = self.outputs.get(alias) - # iteration_append_feature = f"{self.root_file_partition_iterator}_{alias}_iteration_append_feature_{scale}" - # - # if not arcpy.Exists(output_path): - # self.create_feature_class( - # out_path=os.path.dirname(output_path), - # out_name=os.path.basename(output_path), - # template_feature=iteration_append_feature, - # ) - # - # partition_target_selection = ( - # f"in_memory/{alias}_partition_target_selection_{self.scale}" - # ) - # self.iteration_file_paths.append(partition_target_selection) - # - # custom_arcpy.select_attribute_and_make_permanent_feature( - # input_layer=iteration_append_feature, - # expression=f"{self.PARTITION_FIELD} = 1", - # output_name=partition_target_selection, - # ) - # - # print( - # f"for {alias} in {iteration_append_feature} \nThe input is: {partition_target_selection}\nAppending to {output_path}" - # ) - # - # arcpy.management.Append( - # inputs=partition_target_selection, - # target=output_path, - # schema_type="NO_TEST", - # ) - # else: - # print( - # f"No features found in {alias} for {self.object_id_field} {object_id} to append to {output_path}" - # ) - - # for alias in self.alias: - # self.delete_iteration_files(*self.iteration_file_paths) - # print(f"Finished iteration {object_id}") + self.append_iteration_to_final(alias) + self.delete_iteration_files(*self.iteration_file_paths) + else: + self.delete_iteration_files(*self.iteration_file_paths) @timing_decorator def run(self): self.integrate_initial_data(self.alias_path_data) + if self.alias_path_outputs is not None: + self.unpack_alias_path_outputs(self.alias_path_outputs) + print("\nAfter unpacking, final_outputs = ", self.final_outputs) + self.delete_final_outputs() self.prepare_input_data() self.create_cartographic_partitions() @@ -620,8 +676,6 @@ def run(self): # Thoughts on PartitionIterator: """ - -Need to decide if I will use class based variables or if I will implemented having a nested class -handling class based variables and file_path definitions inside the function. +Working on append_iteration_to_final need it to select the correct file from nested dictionary based on type for alias """ From 2096c1bdee01d53d447e10dac73258f2b6c01c7b Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Fri, 22 Mar 2024 15:00:55 +0100 Subject: [PATCH 04/19] fixed partition iterator bug Fixed logic to prevent incremental duplication for append features --- .../partition_iterator_state_based.py | 78 ++-- .../not_in_use_models/testing_buildings.py | 333 ++++++++++++++++++ 2 files changed, 389 insertions(+), 22 deletions(-) create mode 100644 generalization/n100/building/not_in_use_models/testing_buildings.py diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 14681f4c..2cabd7cd 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -102,6 +102,23 @@ def update_alias_state(self, alias, type_info, path=None): self.data[alias] = {} self.data[alias][type_info] = path + def add_type_to_alias(self, alias, new_type, new_type_path=None): + """Adds a new type with optional file path to an existing alias.""" + if alias not in self.data: + print(f"Alias '{alias}' not found in data.") + return + + if new_type in self.data[alias]: + print( + f"Type '{new_type}' already exists for alias '{alias}'. Current path: {self.data[alias][new_type]}" + ) + return + + self.data[alias][new_type] = new_type_path + print( + f"Added type '{new_type}' to alias '{alias}' in data with path: {new_type_path}" + ) + def create_new_alias(self, alias, initial_states): if alias in self.data: raise ValueError(f"Alias {alias} already exists.") @@ -116,7 +133,7 @@ def create_cartographic_partitions(self): path for alias, types in self.data.items() for type_key, path in types.items() - if type_key in ["input", "context"] and path is not None + if type_key in ["input_copy", "context_copy"] and path is not None ] print(f"all_features: {all_features}") @@ -184,7 +201,7 @@ def create_feature_class(full_feature_path, template_feature): ) print(f"Created feature class: {full_feature_path}") - def create_dummy_features(self, types_to_include=["input", "context"]): + def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]): """ Creates dummy features for aliases with specified types. @@ -246,11 +263,11 @@ def prepare_input_data(self): ) print(f"Copied input data for: {alias}") - # Update the path for 'input' type to the new copied path - self.update_alias_state( + # Add a new type for the alias the copied input data + self.add_type_to_alias( alias=alias, - type_info="input", - path=input_data_copy, + new_type="input_copy", + new_type_path=input_data_copy, ) arcpy.AddField_management( @@ -286,6 +303,24 @@ def prepare_input_data(self): # Update the instance variable if a new unique field name was created self.ORIGINAL_ID_FIELD = unique_orig_id_field + if "context" in types: + context_data_path = types["context"] + context_data_copy = ( + f"{self.root_file_partition_iterator}_{alias}_context_copy" + ) + # self.delete_feature_class(input_data_copy) + arcpy.management.Copy( + in_data=context_data_path, + out_data=context_data_copy, + ) + print(f"Copied context data for: {alias}") + + self.add_type_to_alias( + alias=alias, + new_type="context_copy", + new_type_path=context_data_copy, + ) + def custom_function(inputs): outputs = [] return outputs @@ -311,18 +346,11 @@ def process_input_features( """ Process input features for a given partition. """ - if "input" not in self.data[alias]: + if "input_copy" not in self.data[alias]: return None, False - if "input" in self.data[alias]: - input_data_copy = f"{self.root_file_partition_iterator}_{alias}_input_copy" - self.update_alias_state( - alias=alias, - type_info="input", - path=input_data_copy, - ) - - input_path = self.data[alias]["input"] + if "input_copy" in self.data[alias]: + input_path = self.data[alias]["input_copy"] input_features_partition_selection = ( f"in_memory/{alias}_partition_base_select_{self.scale}" ) @@ -419,7 +447,7 @@ def process_input_features( def _process_inputs_in_partition(self, aliases, iteration_partition, object_id): inputs_present_in_partition = False for alias in aliases: - if "input" in self.data[alias]: + if "input_copy" in self.data[alias]: _, input_present = self.process_input_features( alias, iteration_partition, object_id ) @@ -432,8 +460,8 @@ def process_context_features(self, alias, iteration_partition): """ Process context features for a given partition if input features are present. """ - if "context" in self.data[alias]: - context_path = self.data[alias]["context"] + if "context_copy" in self.data[alias]: + context_path = self.data[alias]["context_copy"] context_selection_path = f"{self.root_file_partition_iterator}_{alias}_context_iteration_selection" self.iteration_file_paths.append(context_selection_path) @@ -446,13 +474,19 @@ def process_context_features(self, alias, iteration_partition): search_distance=self.search_distance, ) + self.update_alias_state( + alias=alias, + type_info="context", + path=context_selection_path, + ) + def _process_context_features_and_others( self, aliases, iteration_partition, object_id ): for alias in aliases: - if "context" not in self.data[alias]: + if "context_copy" not in self.data[alias]: print( - f"iteration partition {object_id} has no features for {alias} in the partition feature" + f"iteration partition {object_id} has no context features for {alias} in the partition feature" ) else: self.process_context_features(alias, iteration_partition) @@ -514,7 +548,7 @@ def partition_iteration(self): max_object_id = self.pre_iteration() # self.delete_existing_outputs() - self.create_dummy_features(types_to_include=["input", "context"]) + self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) self.delete_iteration_files(*self.iteration_file_paths) self.iteration_file_paths.clear() diff --git a/generalization/n100/building/not_in_use_models/testing_buildings.py b/generalization/n100/building/not_in_use_models/testing_buildings.py new file mode 100644 index 00000000..a95b35ee --- /dev/null +++ b/generalization/n100/building/not_in_use_models/testing_buildings.py @@ -0,0 +1,333 @@ +# Importing packages +import arcpy + +# Importing custom input files modules +from input_data import input_n50 +from input_data import input_n100 +from input_data import input_other + +# Importing custom modules +from file_manager.n100.file_manager_buildings import Building_N100 +from env_setup import environment_setup +from custom_tools.timing_decorator import timing_decorator +from custom_tools import custom_arcpy +from constants.n100_constants import N100_SQLResources + + +@timing_decorator("data_preparation.py") +def main(): + """ + Summary: + This is the main function of building data preparation, which aims to prepare the data for future building generalization processing. + """ + + environment_setup.main() + begrensningskurve_land_and_water_bodies() + begrensningskurve_river() + merge_begrensningskurve_all_water_features() + unsplit_roads() + matrikkel_and_n50_not_in_urban_areas() + adding_field_values_to_matrikkel() + merge_matrikkel_and_n50_points() + selecting_polygons_not_in_urban_areas() + reclassifying_polygon_values() + polygon_selections_based_on_size() + + +@timing_decorator +def begrensningskurve_land_and_water_bodies(): + """ + Summary: + This function creates a buffer for water features using the "begrensningskurve" feature. It prepares the data for future building placement on the water's edge. + + Details: + - Using the object types ('ElvBekk', 'Havflate', 'Innsjø', 'InnsjøRegulert') in the "begrensningskurve" feature in SQL expressions to select water features and in inverse to select land features. + - Create a temporary feature of water features from the "begrensningskurve" using the defined SQL expression. + - Select land features using an inverted selection using the defined SQL expression. + - Identify land features near water features by selecting those that boundary-touch with water features to reduce the amount of processing. + - Apply a 15-meter buffer to the identified land features to create buffered land features, this is the distance objects is allowed to be overlapping water features in the final output. + - Apply a 45-meter buffer to the selected water features to create buffered water features, this is to make sure features are not going past the water barrier and is instead pushed towrds land instead of further inside waterfeatures. + - Erase buffered water features from the buffered land features to create a final set of waterfeature buffer which is used throughout this generalization of buildings. + + Note: + - Additional logic may be required for rivers separately in future development as narrow polygons gets completly removed due to the land buffer being to large for the rivers. In processes actually needing barriers this will allow objects to cross narrow rivers. + """ + + # Defining the SQL selection expression for water features for begrensningskurve (not river) + sql_expr_begrensningskurve_waterfeatures_not_river = "OBJTYPE = 'Innsjøkant' Or OBJTYPE = 'InnsjøkantRegulert' Or OBJTYPE = 'Kystkontur'" + + # Creating a temporary feature of water features from begrensningskurve + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=input_n100.BegrensningsKurve, + expression=sql_expr_begrensningskurve_waterfeatures_not_river, + output_name=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_not_rivers___n100_building.value, + ) + + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=input_n100.ArealdekkeFlate, + expression="""OBJTYPE NOT IN ('ElvBekk', 'Havflate', 'Innsjø', 'InnsjøRegulert')""", + output_name=Building_N100.data_preparation___selected_land_features_area___n100_building.value, + ) + + custom_arcpy.select_location_and_make_permanent_feature( + input_layer=Building_N100.data_preparation___selected_land_features_area___n100_building.value, + overlap_type=custom_arcpy.OverlapType.BOUNDARY_TOUCHES.value, + select_features=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_not_rivers___n100_building.value, + output_name=Building_N100.data_preparation___land_features_near_water___n100_building.value, + ) + + arcpy.analysis.PairwiseBuffer( + in_features=Building_N100.data_preparation___land_features_near_water___n100_building.value, + out_feature_class=Building_N100.data_preparation___land_features_buffer___n100_building.value, + buffer_distance_or_field="15 Meters", + ) + + arcpy.analysis.PairwiseBuffer( + in_features=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_not_rivers___n100_building.value, + out_feature_class=Building_N100.data_preparation___begrensningskurve_waterfeatures_buffer___n100_building.value, + buffer_distance_or_field="45 Meters", + ) + + arcpy.analysis.PairwiseErase( + in_features=Building_N100.data_preparation___begrensningskurve_waterfeatures_buffer___n100_building.value, + erase_features=Building_N100.data_preparation___selected_land_features_area___n100_building.value, + out_feature_class=Building_N100.data_preparation___begrensningskurve_buffer_erase_1___n100_building.value, + ) + + arcpy.analysis.PairwiseErase( + in_features=Building_N100.data_preparation___begrensningskurve_buffer_erase_1___n100_building.value, + erase_features=Building_N100.data_preparation___land_features_buffer___n100_building.value, + out_feature_class=Building_N100.data_preparation___begrensningskurve_buffer_erase_2___n100_building.value, + ) + + +@timing_decorator +def begrensningskurve_river(): + sql_expr_begrensningskurve_river_outline = "OBJTYPE = 'ElvBekkKant'" + + # Creating a temporary feature of rivers from begrensningskurve + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=input_n100.BegrensningsKurve, + expression=sql_expr_begrensningskurve_river_outline, + output_name=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers___n100_building.value, + ) + + # Creating small buffer around begrensningskurve rivers + + arcpy.analysis.PairwiseBuffer( + in_features=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers___n100_building.value, + out_feature_class=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers_buffer___n100_building.value, + buffer_distance_or_field="0.1 Meters", + ) + + +@timing_decorator +def merge_begrensningskurve_all_water_features(): + # Merge begrensningskurve buffers (water bodies and rivers) + arcpy.management.Merge( + inputs=[ + Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers_buffer___n100_building.value, + Building_N100.data_preparation___begrensningskurve_buffer_erase_2___n100_building.value, + ], + output=Building_N100.data_preparation___merged_begrensningskurve_all_waterbodies___n100_building.value, + ) + + +@timing_decorator +def unsplit_roads(): + """ + Summary: + This function unsplit a line feature of roads to reduce the number of objects in future processing. + + Details: + - It takes the input line feature `input_n100.VegSti` and removes any geometric splits. + - The feature is dissolved based on the fields "subtypekode," "motorvegtype," and "UTTEGNING" to merge segments with the same values in these fields. + Note: + - In the future when the inputs make spatial selections of the features used for context for processing like roads this step is redundant and will instead increase processing time. + """ + + arcpy.UnsplitLine_management( + in_features=input_n100.VegSti, + out_feature_class=Building_N100.data_preparation___unsplit_roads___n100_building.value, + dissolve_field=["subtypekode", "motorvegtype", "UTTEGNING"], + ) + + +@timing_decorator +def matrikkel_and_n50_not_in_urban_areas(): + """ + Summary: + This function adds building points from the matrikkel dataset for areas that are no longer considered urban after the generalization of 'ArealdekkeFlate'. It also adds the required fields and values for future analysis. + + Details: + - Define an SQL expression to select urban areas ('Tettbebyggelse', 'Industriområde', 'BymessigBebyggelse') in the 'ArealdekkeFlate' dataset. + - Select urban areas from 'ArealdekkeFlate' in both n100 and n50 datasets using the defined SQL expression. + - Create a buffer of the selected urban areas from n100 to take into consideration that points should not be too close to urban areas. + - Remove areas from n50 urban areas that intersect with the buffer of n100 urban areas, resulting in areas in n100 that are no longer considered urban. + - Select matrikkel bygningspunkter based on the new urban selection layer. + - Transfer the NBR (building type) value to the matrikkel_bygningspunkt dataset by adding a new field "BYGGTYP_NBR" of type LONG and calculating its values from the "bygningstype" field. + Note: + - Should consider removing the logic of adding a buffer to the urban areas to prevent points to close to urban areas and instead using urban areas as a barrier feature in future processing. + """ + + # Defining sql expression to select urban areas + urban_areas_sql_expr = "OBJTYPE = 'Tettbebyggelse' Or OBJTYPE = 'Industriområde' Or OBJTYPE = 'BymessigBebyggelse'" + + # Selecting urban areas from n100 using sql expression + custom_arcpy.select_attribute_and_make_feature_layer( + input_layer=input_n100.ArealdekkeFlate, + expression=urban_areas_sql_expr, + output_name=Building_N100.data_preparation___urban_area_selection_n100___n100_building.value, + ) + + # Selecting urban areas from n50 using sql expression + custom_arcpy.select_attribute_and_make_feature_layer( + input_layer=input_n50.ArealdekkeFlate, + expression=urban_areas_sql_expr, + output_name=Building_N100.data_preparation___urban_area_selection_n50___n100_building.value, + ) + + # Creating a buffer of the urban selection of n100 to take into account symbology + arcpy.PairwiseBuffer_analysis( + in_features=Building_N100.data_preparation___urban_area_selection_n100___n100_building.value, + out_feature_class=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, + buffer_distance_or_field="50 Meters", + method="PLANAR", + ) + + # Removing areas from n50 urban areas from the buffer of n100 urban areas resulting in areas in n100 which no longer are urban + arcpy.PairwiseErase_analysis( + in_features=Building_N100.data_preparation___urban_area_selection_n50___n100_building.value, + erase_features=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, + out_feature_class=Building_N100.data_preparation___no_longer_urban_areas___n100_building.value, + ) + + # Selecting matrikkel building points based on this new urban selection layer + custom_arcpy.select_location_and_make_permanent_feature( + input_layer=input_other.matrikkel_bygningspunkt, + overlap_type=custom_arcpy.OverlapType.INTERSECT, + select_features=Building_N100.data_preparation___no_longer_urban_areas___n100_building.value, + output_name=Building_N100.data_preparation___matrikkel_points___n100_building.value, + ) + + # Selecting n50 so they are not in urban areas + custom_arcpy.select_location_and_make_permanent_feature( + input_layer=input_n50.BygningsPunkt, + overlap_type=custom_arcpy.OverlapType.INTERSECT, + select_features=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, + output_name=Building_N100.data_preparation___n50_points___n100_building.value, + inverted=True, + ) + + # Making sure we are not loosing churches or hospitals + custom_arcpy.select_location_and_make_permanent_feature( + input_layer=input_n50.BygningsPunkt, + overlap_type=custom_arcpy.OverlapType.INTERSECT, + select_features=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, + output_name="n50_points_in_urban_areas", + ) + + sql_church_hospitals = "BYGGTYP_NBR IN (970, 719, 671)" + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer="n50_points_in_urban_areas", + expression=sql_church_hospitals, + output_name="church_hospitals_in_urban_areas", + ) + + +@timing_decorator +def adding_field_values_to_matrikkel(): + # Adding transferring the NBR value to the matrikkel building points + arcpy.AddField_management( + in_table=Building_N100.data_preparation___matrikkel_points___n100_building.value, + field_name="BYGGTYP_NBR", + field_type="LONG", + ) + arcpy.CalculateField_management( + in_table=Building_N100.data_preparation___matrikkel_points___n100_building.value, + field="BYGGTYP_NBR", + expression="!bygningstype!", + ) + + +@timing_decorator +def merge_matrikkel_and_n50_points(): + # Merge the n50 building point and matrikkel + arcpy.management.Merge( + inputs=[ + Building_N100.data_preparation___n50_points___n100_building.value, + "church_hospitals_in_urban_areas", + Building_N100.data_preparation___matrikkel_points___n100_building.value, + ], + output=Building_N100.data_preperation___matrikkel_n50_points_merged___n100_building.value, + ) + + +@timing_decorator +def selecting_polygons_not_in_urban_areas(): + print( + "might want to remove this copyl ater when we have another script for just copying the database" + ) + # Copy the input data to not modify the original fields. + arcpy.management.Copy( + in_data=input_n50.Grunnriss, + out_data=Building_N100.data_preparation___grunnriss_copy___n100_building.value, + ) + + # Selecting n50 building points based on this new urban selection layer + custom_arcpy.select_location_and_make_permanent_feature( + input_layer=input_n50.Grunnriss, + overlap_type=custom_arcpy.OverlapType.INTERSECT, + select_features=Building_N100.data_preparation___no_longer_urban_areas___n100_building.value, + output_name=Building_N100.data_preparation___n50_polygons___n100_building.value, + ) + + +@timing_decorator +def reclassifying_polygon_values(): + # Reclassify the hospitals and churches to NBR value 729 ("other buildings" / "andre bygg") + reclassify_hospital_church_polygons = ( + "def reclassify(nbr):\n" + " mapping = {970: 729, 719: 729, 671: 729}\n" + " return mapping.get(nbr, nbr)" + ) + + arcpy.CalculateField_management( + in_table=Building_N100.data_preparation___n50_polygons___n100_building.value, + field="BYGGTYP_NBR", + expression="reclassify(!BYGGTYP_NBR!)", + expression_type="PYTHON3", + code_block=reclassify_hospital_church_polygons, + ) + + +def polygon_selections_based_on_size(): + # Selecting only building polygons over 2500 (the rest will be transformed to points due to size) + grunnriss_minimum_size = 2500 + sql_expression_too_small_polygons = f"Shape_Area < {grunnriss_minimum_size}" + sql_expression_correct_size_polygons = f"Shape_Area >= {grunnriss_minimum_size}" + + # Polygons over or equal to 2500 Square Meters are selected + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=Building_N100.data_preparation___grunnriss_copy___n100_building.value, + expression=sql_expression_correct_size_polygons, + output_name=Building_N100.data_preparation___polygons_that_are_large_enough___n100_building.value, + ) + + # Polygons under 2500 Square Meters are selected + custom_arcpy.select_attribute_and_make_feature_layer( + input_layer=Building_N100.data_preparation___grunnriss_copy___n100_building.value, + expression=sql_expression_too_small_polygons, + output_name=Building_N100.data_preparation___polygons_that_are_too_small___n100_building.value, + selection_type=custom_arcpy.SelectionType.NEW_SELECTION, + ) + + # Transforming small building polygons into points + arcpy.management.FeatureToPoint( + in_features=Building_N100.data_preparation___polygons_that_are_too_small___n100_building.value, + out_feature_class=Building_N100.data_preparation___points_created_from_small_polygons___n100_building.value, # Sent to polygon to point - to get merged as an additional input + ) + + +if __name__ == "__main__": + main() From 7f031db16e23042c04b7dd59b4287873c15da048 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Fri, 22 Mar 2024 16:30:43 +0100 Subject: [PATCH 05/19] Partition Iterator is now working (without custom logic) Fixed so that partition iterator logic is working to select the inputs using nested dictionaries and writing the outputs to specified locations. Not implemented insertion of custom logic --- .../partition_iterator_state_based.py | 70 +++++++++++++++---- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 2cabd7cd..f43138a9 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -167,21 +167,6 @@ def delete_final_outputs(self): arcpy.management.Delete(output_file_path) print(f"Deleted file: {output_file_path}") - def delete_existing_outputs(self): - for alias, output_info in self.output_feature_class.items(): - output_type, output_path = output_info - current_path = self.data.get(alias, {}).get(output_type) - if current_path == output_path and arcpy.Exists(current_path): - PartitionIterator.delete_feature_class( - current_path, - alias=alias, - output_type=output_type, - ) - else: - print( - f"Output feature class for '{alias}' of type '{output_type}' does not exist or path does not match: {current_path}" - ) - def delete_iteration_files(self, *file_paths): """Deletes multiple feature classes or files. Detailed alias and output_type logging is not available here.""" for file_path in file_paths: @@ -226,6 +211,37 @@ def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]) path=dummy_feature_path, ) + def initialize_dummy_used(self): + # Assuming `aliases` is a list of all your aliases + for alias in self.data: + self.data[alias]["dummy_used"] = False + + def reset_dummy_used(self): + # Assuming `aliases` is a list of all your aliases + for alias in self.data: + self.data[alias]["dummy_used"] = False + + def update_alias_with_dummy_if_needed(self, alias, type_info): + # Check if the dummy type exists in the alias data + if "dummy" in self.data[alias]: + # Check if the input type exists in the alias data + if ( + type_info in self.data[alias] + and self.data[alias][type_info] is not None + ): + # Get the dummy path from the alias data + dummy_path = self.data[alias]["dummy"] + # Set the value of the existing type_info to the dummy path + self.data[alias][type_info] = dummy_path + self.data[alias]["dummy_used"] = True + print( + f"The '{type_info}' for alias '{alias}' was updated with dummy path: {dummy_path}" + ) + else: + print(f"'{type_info}' does not exist for alias '{alias}' in data.") + else: + print(f"'dummy' type does not exist for alias '{alias}' in data.") + def write_data_to_json(self, file_name): with open(file_name, "w") as file: json.dump(self.data, file, indent=4) @@ -439,6 +455,11 @@ def process_input_features( ) return aliases_with_features, True else: + # Loads in dummy feature for this alias for this iteration and sets dummy_used = True + self.update_alias_with_dummy_if_needed( + alias, + type_info="input", + ) print( f"iteration partition {object_id} has no features for {alias} in the partition feature" ) @@ -485,6 +506,11 @@ def _process_context_features_and_others( ): for alias in aliases: if "context_copy" not in self.data[alias]: + # Loads in dummy feature for this alias for this iteration and sets dummy_used = True + self.update_alias_with_dummy_if_needed( + alias, + type_info="context", + ) print( f"iteration partition {object_id} has no context features for {alias} in the partition feature" ) @@ -498,8 +524,20 @@ def append_iteration_to_final(self, alias): # For each type under current alias, append the result of the current iteration for type_info, final_output_path in self.final_outputs[alias].items(): + if self.data[alias]["dummy_used"]: + continue + input_feature_class = self.data[alias][type_info] + if ( + not arcpy.Exists(input_feature_class) + or int(arcpy.GetCount_management(input_feature_class).getOutput(0)) <= 0 + ): + print( + f"No features found in partition target selection: {input_feature_class}" + ) + continue + partition_target_selection = ( f"in_memory/{alias}_{type_info}_partition_target_selection_{self.scale}" ) @@ -549,11 +587,13 @@ def partition_iteration(self): # self.delete_existing_outputs() self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) + self.initialize_dummy_used() self.delete_iteration_files(*self.iteration_file_paths) self.iteration_file_paths.clear() for object_id in range(1, max_object_id + 1): + self.reset_dummy_used() self.iteration_file_paths.clear() iteration_partition = f"{self.partition_feature}_{object_id}" self.select_partition_feature(iteration_partition, object_id) From 0246cc509dce7c89897e4a17640abb92e12cd544 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Fri, 22 Mar 2024 19:20:18 +0100 Subject: [PATCH 06/19] removed testing_buildings.py syntax --- .../not_in_use_models/testing_buildings.py | 333 ------------------ 1 file changed, 333 deletions(-) diff --git a/generalization/n100/building/not_in_use_models/testing_buildings.py b/generalization/n100/building/not_in_use_models/testing_buildings.py index a95b35ee..e69de29b 100644 --- a/generalization/n100/building/not_in_use_models/testing_buildings.py +++ b/generalization/n100/building/not_in_use_models/testing_buildings.py @@ -1,333 +0,0 @@ -# Importing packages -import arcpy - -# Importing custom input files modules -from input_data import input_n50 -from input_data import input_n100 -from input_data import input_other - -# Importing custom modules -from file_manager.n100.file_manager_buildings import Building_N100 -from env_setup import environment_setup -from custom_tools.timing_decorator import timing_decorator -from custom_tools import custom_arcpy -from constants.n100_constants import N100_SQLResources - - -@timing_decorator("data_preparation.py") -def main(): - """ - Summary: - This is the main function of building data preparation, which aims to prepare the data for future building generalization processing. - """ - - environment_setup.main() - begrensningskurve_land_and_water_bodies() - begrensningskurve_river() - merge_begrensningskurve_all_water_features() - unsplit_roads() - matrikkel_and_n50_not_in_urban_areas() - adding_field_values_to_matrikkel() - merge_matrikkel_and_n50_points() - selecting_polygons_not_in_urban_areas() - reclassifying_polygon_values() - polygon_selections_based_on_size() - - -@timing_decorator -def begrensningskurve_land_and_water_bodies(): - """ - Summary: - This function creates a buffer for water features using the "begrensningskurve" feature. It prepares the data for future building placement on the water's edge. - - Details: - - Using the object types ('ElvBekk', 'Havflate', 'Innsjø', 'InnsjøRegulert') in the "begrensningskurve" feature in SQL expressions to select water features and in inverse to select land features. - - Create a temporary feature of water features from the "begrensningskurve" using the defined SQL expression. - - Select land features using an inverted selection using the defined SQL expression. - - Identify land features near water features by selecting those that boundary-touch with water features to reduce the amount of processing. - - Apply a 15-meter buffer to the identified land features to create buffered land features, this is the distance objects is allowed to be overlapping water features in the final output. - - Apply a 45-meter buffer to the selected water features to create buffered water features, this is to make sure features are not going past the water barrier and is instead pushed towrds land instead of further inside waterfeatures. - - Erase buffered water features from the buffered land features to create a final set of waterfeature buffer which is used throughout this generalization of buildings. - - Note: - - Additional logic may be required for rivers separately in future development as narrow polygons gets completly removed due to the land buffer being to large for the rivers. In processes actually needing barriers this will allow objects to cross narrow rivers. - """ - - # Defining the SQL selection expression for water features for begrensningskurve (not river) - sql_expr_begrensningskurve_waterfeatures_not_river = "OBJTYPE = 'Innsjøkant' Or OBJTYPE = 'InnsjøkantRegulert' Or OBJTYPE = 'Kystkontur'" - - # Creating a temporary feature of water features from begrensningskurve - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=input_n100.BegrensningsKurve, - expression=sql_expr_begrensningskurve_waterfeatures_not_river, - output_name=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_not_rivers___n100_building.value, - ) - - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=input_n100.ArealdekkeFlate, - expression="""OBJTYPE NOT IN ('ElvBekk', 'Havflate', 'Innsjø', 'InnsjøRegulert')""", - output_name=Building_N100.data_preparation___selected_land_features_area___n100_building.value, - ) - - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=Building_N100.data_preparation___selected_land_features_area___n100_building.value, - overlap_type=custom_arcpy.OverlapType.BOUNDARY_TOUCHES.value, - select_features=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_not_rivers___n100_building.value, - output_name=Building_N100.data_preparation___land_features_near_water___n100_building.value, - ) - - arcpy.analysis.PairwiseBuffer( - in_features=Building_N100.data_preparation___land_features_near_water___n100_building.value, - out_feature_class=Building_N100.data_preparation___land_features_buffer___n100_building.value, - buffer_distance_or_field="15 Meters", - ) - - arcpy.analysis.PairwiseBuffer( - in_features=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_not_rivers___n100_building.value, - out_feature_class=Building_N100.data_preparation___begrensningskurve_waterfeatures_buffer___n100_building.value, - buffer_distance_or_field="45 Meters", - ) - - arcpy.analysis.PairwiseErase( - in_features=Building_N100.data_preparation___begrensningskurve_waterfeatures_buffer___n100_building.value, - erase_features=Building_N100.data_preparation___selected_land_features_area___n100_building.value, - out_feature_class=Building_N100.data_preparation___begrensningskurve_buffer_erase_1___n100_building.value, - ) - - arcpy.analysis.PairwiseErase( - in_features=Building_N100.data_preparation___begrensningskurve_buffer_erase_1___n100_building.value, - erase_features=Building_N100.data_preparation___land_features_buffer___n100_building.value, - out_feature_class=Building_N100.data_preparation___begrensningskurve_buffer_erase_2___n100_building.value, - ) - - -@timing_decorator -def begrensningskurve_river(): - sql_expr_begrensningskurve_river_outline = "OBJTYPE = 'ElvBekkKant'" - - # Creating a temporary feature of rivers from begrensningskurve - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=input_n100.BegrensningsKurve, - expression=sql_expr_begrensningskurve_river_outline, - output_name=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers___n100_building.value, - ) - - # Creating small buffer around begrensningskurve rivers - - arcpy.analysis.PairwiseBuffer( - in_features=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers___n100_building.value, - out_feature_class=Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers_buffer___n100_building.value, - buffer_distance_or_field="0.1 Meters", - ) - - -@timing_decorator -def merge_begrensningskurve_all_water_features(): - # Merge begrensningskurve buffers (water bodies and rivers) - arcpy.management.Merge( - inputs=[ - Building_N100.data_preperation___waterfeatures_from_begrensningskurve_rivers_buffer___n100_building.value, - Building_N100.data_preparation___begrensningskurve_buffer_erase_2___n100_building.value, - ], - output=Building_N100.data_preparation___merged_begrensningskurve_all_waterbodies___n100_building.value, - ) - - -@timing_decorator -def unsplit_roads(): - """ - Summary: - This function unsplit a line feature of roads to reduce the number of objects in future processing. - - Details: - - It takes the input line feature `input_n100.VegSti` and removes any geometric splits. - - The feature is dissolved based on the fields "subtypekode," "motorvegtype," and "UTTEGNING" to merge segments with the same values in these fields. - Note: - - In the future when the inputs make spatial selections of the features used for context for processing like roads this step is redundant and will instead increase processing time. - """ - - arcpy.UnsplitLine_management( - in_features=input_n100.VegSti, - out_feature_class=Building_N100.data_preparation___unsplit_roads___n100_building.value, - dissolve_field=["subtypekode", "motorvegtype", "UTTEGNING"], - ) - - -@timing_decorator -def matrikkel_and_n50_not_in_urban_areas(): - """ - Summary: - This function adds building points from the matrikkel dataset for areas that are no longer considered urban after the generalization of 'ArealdekkeFlate'. It also adds the required fields and values for future analysis. - - Details: - - Define an SQL expression to select urban areas ('Tettbebyggelse', 'Industriområde', 'BymessigBebyggelse') in the 'ArealdekkeFlate' dataset. - - Select urban areas from 'ArealdekkeFlate' in both n100 and n50 datasets using the defined SQL expression. - - Create a buffer of the selected urban areas from n100 to take into consideration that points should not be too close to urban areas. - - Remove areas from n50 urban areas that intersect with the buffer of n100 urban areas, resulting in areas in n100 that are no longer considered urban. - - Select matrikkel bygningspunkter based on the new urban selection layer. - - Transfer the NBR (building type) value to the matrikkel_bygningspunkt dataset by adding a new field "BYGGTYP_NBR" of type LONG and calculating its values from the "bygningstype" field. - Note: - - Should consider removing the logic of adding a buffer to the urban areas to prevent points to close to urban areas and instead using urban areas as a barrier feature in future processing. - """ - - # Defining sql expression to select urban areas - urban_areas_sql_expr = "OBJTYPE = 'Tettbebyggelse' Or OBJTYPE = 'Industriområde' Or OBJTYPE = 'BymessigBebyggelse'" - - # Selecting urban areas from n100 using sql expression - custom_arcpy.select_attribute_and_make_feature_layer( - input_layer=input_n100.ArealdekkeFlate, - expression=urban_areas_sql_expr, - output_name=Building_N100.data_preparation___urban_area_selection_n100___n100_building.value, - ) - - # Selecting urban areas from n50 using sql expression - custom_arcpy.select_attribute_and_make_feature_layer( - input_layer=input_n50.ArealdekkeFlate, - expression=urban_areas_sql_expr, - output_name=Building_N100.data_preparation___urban_area_selection_n50___n100_building.value, - ) - - # Creating a buffer of the urban selection of n100 to take into account symbology - arcpy.PairwiseBuffer_analysis( - in_features=Building_N100.data_preparation___urban_area_selection_n100___n100_building.value, - out_feature_class=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, - buffer_distance_or_field="50 Meters", - method="PLANAR", - ) - - # Removing areas from n50 urban areas from the buffer of n100 urban areas resulting in areas in n100 which no longer are urban - arcpy.PairwiseErase_analysis( - in_features=Building_N100.data_preparation___urban_area_selection_n50___n100_building.value, - erase_features=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, - out_feature_class=Building_N100.data_preparation___no_longer_urban_areas___n100_building.value, - ) - - # Selecting matrikkel building points based on this new urban selection layer - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=input_other.matrikkel_bygningspunkt, - overlap_type=custom_arcpy.OverlapType.INTERSECT, - select_features=Building_N100.data_preparation___no_longer_urban_areas___n100_building.value, - output_name=Building_N100.data_preparation___matrikkel_points___n100_building.value, - ) - - # Selecting n50 so they are not in urban areas - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=input_n50.BygningsPunkt, - overlap_type=custom_arcpy.OverlapType.INTERSECT, - select_features=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, - output_name=Building_N100.data_preparation___n50_points___n100_building.value, - inverted=True, - ) - - # Making sure we are not loosing churches or hospitals - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=input_n50.BygningsPunkt, - overlap_type=custom_arcpy.OverlapType.INTERSECT, - select_features=Building_N100.data_preparation___urban_area_selection_n100_buffer___n100_building.value, - output_name="n50_points_in_urban_areas", - ) - - sql_church_hospitals = "BYGGTYP_NBR IN (970, 719, 671)" - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer="n50_points_in_urban_areas", - expression=sql_church_hospitals, - output_name="church_hospitals_in_urban_areas", - ) - - -@timing_decorator -def adding_field_values_to_matrikkel(): - # Adding transferring the NBR value to the matrikkel building points - arcpy.AddField_management( - in_table=Building_N100.data_preparation___matrikkel_points___n100_building.value, - field_name="BYGGTYP_NBR", - field_type="LONG", - ) - arcpy.CalculateField_management( - in_table=Building_N100.data_preparation___matrikkel_points___n100_building.value, - field="BYGGTYP_NBR", - expression="!bygningstype!", - ) - - -@timing_decorator -def merge_matrikkel_and_n50_points(): - # Merge the n50 building point and matrikkel - arcpy.management.Merge( - inputs=[ - Building_N100.data_preparation___n50_points___n100_building.value, - "church_hospitals_in_urban_areas", - Building_N100.data_preparation___matrikkel_points___n100_building.value, - ], - output=Building_N100.data_preperation___matrikkel_n50_points_merged___n100_building.value, - ) - - -@timing_decorator -def selecting_polygons_not_in_urban_areas(): - print( - "might want to remove this copyl ater when we have another script for just copying the database" - ) - # Copy the input data to not modify the original fields. - arcpy.management.Copy( - in_data=input_n50.Grunnriss, - out_data=Building_N100.data_preparation___grunnriss_copy___n100_building.value, - ) - - # Selecting n50 building points based on this new urban selection layer - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=input_n50.Grunnriss, - overlap_type=custom_arcpy.OverlapType.INTERSECT, - select_features=Building_N100.data_preparation___no_longer_urban_areas___n100_building.value, - output_name=Building_N100.data_preparation___n50_polygons___n100_building.value, - ) - - -@timing_decorator -def reclassifying_polygon_values(): - # Reclassify the hospitals and churches to NBR value 729 ("other buildings" / "andre bygg") - reclassify_hospital_church_polygons = ( - "def reclassify(nbr):\n" - " mapping = {970: 729, 719: 729, 671: 729}\n" - " return mapping.get(nbr, nbr)" - ) - - arcpy.CalculateField_management( - in_table=Building_N100.data_preparation___n50_polygons___n100_building.value, - field="BYGGTYP_NBR", - expression="reclassify(!BYGGTYP_NBR!)", - expression_type="PYTHON3", - code_block=reclassify_hospital_church_polygons, - ) - - -def polygon_selections_based_on_size(): - # Selecting only building polygons over 2500 (the rest will be transformed to points due to size) - grunnriss_minimum_size = 2500 - sql_expression_too_small_polygons = f"Shape_Area < {grunnriss_minimum_size}" - sql_expression_correct_size_polygons = f"Shape_Area >= {grunnriss_minimum_size}" - - # Polygons over or equal to 2500 Square Meters are selected - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=Building_N100.data_preparation___grunnriss_copy___n100_building.value, - expression=sql_expression_correct_size_polygons, - output_name=Building_N100.data_preparation___polygons_that_are_large_enough___n100_building.value, - ) - - # Polygons under 2500 Square Meters are selected - custom_arcpy.select_attribute_and_make_feature_layer( - input_layer=Building_N100.data_preparation___grunnriss_copy___n100_building.value, - expression=sql_expression_too_small_polygons, - output_name=Building_N100.data_preparation___polygons_that_are_too_small___n100_building.value, - selection_type=custom_arcpy.SelectionType.NEW_SELECTION, - ) - - # Transforming small building polygons into points - arcpy.management.FeatureToPoint( - in_features=Building_N100.data_preparation___polygons_that_are_too_small___n100_building.value, - out_feature_class=Building_N100.data_preparation___points_created_from_small_polygons___n100_building.value, # Sent to polygon to point - to get merged as an additional input - ) - - -if __name__ == "__main__": - main() From 9d6d8a9a95200a00e29e88c60fcdc1d5bbfc6112 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Fri, 22 Mar 2024 19:35:03 +0100 Subject: [PATCH 07/19] Remove debug print statements and unused comments Commit removes unnecessary debug print statements and unused code comments scattered throughout partition_iterator_state_based.py. Simplifying the code increases readability and makes potential bugs or problems easier to identify. --- .../partition_iterator_state_based.py | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index f43138a9..b5d6dc74 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -47,7 +47,6 @@ def __init__( self.data = {} self.alias_path_data = alias_path_data self.alias_path_outputs = alias_path_outputs or {} - print("\nInitializing with alias_path_outputs = ", alias_path_outputs) self.root_file_partition_iterator = root_file_partition_iterator self.scale = scale self.output_feature_class = alias_path_outputs @@ -90,7 +89,6 @@ def unpack_alias_path_outputs(self, alias_path_outputs): if alias not in self.final_outputs: self.final_outputs[alias] = {} self.final_outputs[alias][type_info] = path_info - print("\nUnpacking alias_path_outputs = ", alias_path_outputs) def integrate_results(self): for alias, types in self.final_outputs.items(): @@ -128,7 +126,6 @@ def get_path_for_alias_and_type(self, alias, type_info): return self.data.get(alias, {}).get(type_info) def create_cartographic_partitions(self): - print("Debugging self.data before partition creation:", self.data) all_features = [ path for alias, types in self.data.items() @@ -324,7 +321,7 @@ def prepare_input_data(self): context_data_copy = ( f"{self.root_file_partition_iterator}_{alias}_context_copy" ) - # self.delete_feature_class(input_data_copy) + arcpy.management.Copy( in_data=context_data_path, out_data=context_data_copy, @@ -351,7 +348,7 @@ def select_partition_feature(self, iteration_partition, object_id): expression=f"{self.object_id_field} = {object_id}", output_name=iteration_partition, ) - print(f"Created partition selection for OBJECTID {object_id}") + print(f"\nCreated partition selection for OBJECTID {object_id}") def process_input_features( self, @@ -389,7 +386,6 @@ def process_input_features( if aliases_with_features[alias] > 0: print(f"{alias} has {count_points} features in {iteration_partition}") - # aliases_with_features += 1 arcpy.CalculateField_management( in_table=input_features_partition_selection, @@ -524,6 +520,7 @@ def append_iteration_to_final(self, alias): # For each type under current alias, append the result of the current iteration for type_info, final_output_path in self.final_outputs[alias].items(): + # Skipping append if the alias is a dummy feature if self.data[alias]["dummy_used"]: continue @@ -551,14 +548,6 @@ def append_iteration_to_final(self, alias): output_name=partition_target_selection, ) - # Number of features before append/copy - orig_num_features = ( - int(arcpy.GetCount_management(final_output_path).getOutput(0)) - if arcpy.Exists(final_output_path) - else 0 - ) - print(f"\nNumber of features originally in the file: {orig_num_features}") - if not arcpy.Exists(final_output_path): arcpy.management.CopyFeatures( in_features=partition_target_selection, @@ -572,20 +561,10 @@ def append_iteration_to_final(self, alias): schema_type="NO_TEST", ) - # Number of features after append/copy - new_num_features = int( - arcpy.GetCount_management(final_output_path).getOutput(0) - ) - print(f"\nNumber of features after append/copy: {new_num_features}") - - def _append_iteration_to_final_and_others(self, alias): - self.append_iteration_to_final(alias) - def partition_iteration(self): aliases = self.data.keys() max_object_id = self.pre_iteration() - # self.delete_existing_outputs() self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) self.initialize_dummy_used() @@ -617,7 +596,7 @@ def run(self): self.integrate_initial_data(self.alias_path_data) if self.alias_path_outputs is not None: self.unpack_alias_path_outputs(self.alias_path_outputs) - print("\nAfter unpacking, final_outputs = ", self.final_outputs) + self.delete_final_outputs() self.prepare_input_data() self.create_cartographic_partitions() From 4e0752cbc4dc99a67149e27cb759a2a59f6ec53a Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Fri, 22 Mar 2024 20:37:41 +0100 Subject: [PATCH 08/19] Add method for generating unique field names Introduced a new function, 'generate_unique_field_name', to automatically generate unique field names in the partition_iterator_state_based.py file. This function gets invoked whenever a new field is added, thereby eliminating duplicated field name concerns, and contributing to cleaner, more maintainable code. --- .../partition_iterator_state_based.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index b5d6dc74..768f7253 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -243,6 +243,13 @@ def write_data_to_json(self, file_name): with open(file_name, "w") as file: json.dump(self.data, file, indent=4) + def generate_unique_field_name(self, input_feature, field_name): + existing_field_names = [field.name for field in arcpy.ListFields(input_feature)] + unique_field_name = field_name + while unique_field_name in existing_field_names: + unique_field_name = f"{unique_field_name}_{random.randint(0, 9)}" + return unique_field_name + def pre_iteration(self): """ Determine the maximum OBJECTID for partitioning. @@ -283,6 +290,17 @@ def prepare_input_data(self): new_type_path=input_data_copy, ) + # Making sure the field is unique if it exists a field with the same name + self.PARTITION_FIELD = self.generate_unique_field_name( + input_feature=input_data_copy, + field_name=self.PARTITION_FIELD, + ) + + self.ORIGINAL_ID_FIELD = self.generate_unique_field_name( + input_feature=input_data_copy, + field_name=self.ORIGINAL_ID_FIELD, + ) + arcpy.AddField_management( in_table=input_data_copy, field_name=self.PARTITION_FIELD, @@ -290,31 +308,19 @@ def prepare_input_data(self): ) print(f"Added field {self.PARTITION_FIELD}") - # Making sure the field is unique if it exists a field with the same name - existing_field_names = [ - field.name for field in arcpy.ListFields(input_data_copy) - ] - unique_orig_id_field = self.ORIGINAL_ID_FIELD - while unique_orig_id_field in existing_field_names: - unique_orig_id_field = ( - f"{self.ORIGINAL_ID_FIELD}_{random.randint(0, 9)}" - ) arcpy.AddField_management( in_table=input_data_copy, - field_name=unique_orig_id_field, + field_name=self.ORIGINAL_ID_FIELD, field_type="LONG", ) - print(f"Added field {unique_orig_id_field}") + print(f"Added field {self.ORIGINAL_ID_FIELD}") arcpy.CalculateField_management( in_table=input_data_copy, - field=unique_orig_id_field, + field=self.ORIGINAL_ID_FIELD, expression=f"!{self.object_id_field}!", ) - print(f"Calculated field {unique_orig_id_field}") - - # Update the instance variable if a new unique field name was created - self.ORIGINAL_ID_FIELD = unique_orig_id_field + print(f"Calculated field {self.ORIGINAL_ID_FIELD}") if "context" in types: context_data_path = types["context"] From 1a321b9a9111a1af70cb41d73aefd9eca433c87e Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 23 Mar 2024 00:30:09 +0100 Subject: [PATCH 09/19] Refactor alias handling in partition_iterator_state_based.py Simplified methods for manipulating types and aliases in the partition_iterator_state_based.py file. Replaced 'update_alias_state' and 'add_type_to_alias' with a more general 'configure_alias_and_type' function, and enhanced 'create_new_alias' with an option to initialize a type and path, leading to a more efficient and readable codebase. --- .../partition_iterator_state_based.py | 81 ++++++++++--------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 768f7253..3c893135 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -19,7 +19,7 @@ class PartitionIterator: # Class-level constants PARTITION_FIELD = "partition_select" - ORIGINAL_ID_FIELD = "orig_id_field" + ORIGINAL_ID_FIELD = "original_id_field" def __init__( self, @@ -95,35 +95,42 @@ def integrate_results(self): for type_info, final_output_path in types.items(): iteration_output_path = self.data[alias][type_info] - def update_alias_state(self, alias, type_info, path=None): - if alias not in self.data: - self.data[alias] = {} - self.data[alias][type_info] = path - - def add_type_to_alias(self, alias, new_type, new_type_path=None): - """Adds a new type with optional file path to an existing alias.""" + def configure_alias_and_type( + self, + alias, + type_name, + type_path, + ): + # Check if alias exists if alias not in self.data: print(f"Alias '{alias}' not found in data.") return - if new_type in self.data[alias]: - print( - f"Type '{new_type}' already exists for alias '{alias}'. Current path: {self.data[alias][new_type]}" - ) - return - - self.data[alias][new_type] = new_type_path - print( - f"Added type '{new_type}' to alias '{alias}' in data with path: {new_type_path}" - ) + # Update path of an existing type or add a new type with the provided path + self.data[alias][type_name] = type_path + print(f"Set path for type '{type_name}' in alias '{alias}' to: {type_path}") - def create_new_alias(self, alias, initial_states): + def create_new_alias( + self, + alias, + initial_type_name=None, + initial_type_path=None, + ): + # Check if alias already exists if alias in self.data: raise ValueError(f"Alias {alias} already exists.") - self.data[alias] = initial_states - def get_path_for_alias_and_type(self, alias, type_info): - return self.data.get(alias, {}).get(type_info) + # Initialize data for alias + if initial_type_name: + # Create alias with initial type and path + self.data[alias] = {initial_type_name: initial_type_path} + else: + # Initialize alias as an empty dictionary + self.data[alias] = {} + + print( + f"Created new alias '{alias}' in data with type '{initial_type_name}' and path: {initial_type_path}" + ) def create_cartographic_partitions(self): all_features = [ @@ -202,10 +209,10 @@ def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]) f"Created dummy feature class for {alias} of type {type_info}: {dummy_feature_path}" ) # Update alias state to include this new dummy type and its path - self.update_alias_state( + self.configure_alias_and_type( alias=alias, - type_info="dummy", - path=dummy_feature_path, + type_name="dummy", + type_path=dummy_feature_path, ) def initialize_dummy_used(self): @@ -284,10 +291,10 @@ def prepare_input_data(self): print(f"Copied input data for: {alias}") # Add a new type for the alias the copied input data - self.add_type_to_alias( + self.configure_alias_and_type( alias=alias, - new_type="input_copy", - new_type_path=input_data_copy, + type_name="input_copy", + type_path=input_data_copy, ) # Making sure the field is unique if it exists a field with the same name @@ -334,10 +341,10 @@ def prepare_input_data(self): ) print(f"Copied context data for: {alias}") - self.add_type_to_alias( + self.configure_alias_and_type( alias=alias, - new_type="context_copy", - new_type_path=context_data_copy, + type_name="context_copy", + type_path=context_data_copy, ) def custom_function(inputs): @@ -446,10 +453,10 @@ def process_input_features( schema_type="NO_TEST", ) - self.update_alias_state( + self.configure_alias_and_type( alias=alias, - type_info="input", - path=iteration_append_feature, + type_name="input", + type_path=iteration_append_feature, ) print( @@ -497,10 +504,10 @@ def process_context_features(self, alias, iteration_partition): search_distance=self.search_distance, ) - self.update_alias_state( + self.configure_alias_and_type( alias=alias, - type_info="context", - path=context_selection_path, + type_name="context", + type_path=context_selection_path, ) def _process_context_features_and_others( From d5bfab0e2cbe7e519296b49ee7d0470d94945249 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 23 Mar 2024 08:28:45 +0100 Subject: [PATCH 10/19] removed unused code and renamed functions in partition iterator Unnecessary code blocks have been removed and naming of some functions have been renamed for better clarity. --- .../partition_iterator_state_based.py | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 3c893135..7957bdc7 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -73,15 +73,6 @@ def integrate_initial_data(self, alias_path_data): self.data[alias] = {} self.data[alias][type_info] = path_info - # for func_name, specs in custom_function_specs.items(): - # for alias, types in specs.items(): - # for type_info in types: - # self.update_alias_state( - # alias=alias, - # type_info=type_info, - # path=None, - # ) - def unpack_alias_path_outputs(self, alias_path_outputs): self.final_outputs = {} for alias, info in alias_path_outputs.items(): @@ -90,11 +81,6 @@ def unpack_alias_path_outputs(self, alias_path_outputs): self.final_outputs[alias] = {} self.final_outputs[alias][type_info] = path_info - def integrate_results(self): - for alias, types in self.final_outputs.items(): - for type_info, final_output_path in types.items(): - iteration_output_path = self.data[alias][type_info] - def configure_alias_and_type( self, alias, @@ -225,7 +211,7 @@ def reset_dummy_used(self): for alias in self.data: self.data[alias]["dummy_used"] = False - def update_alias_with_dummy_if_needed(self, alias, type_info): + def update_empty_alias_type_with_dummy_file(self, alias, type_info): # Check if the dummy type exists in the alias data if "dummy" in self.data[alias]: # Check if the input type exists in the alias data @@ -257,7 +243,7 @@ def generate_unique_field_name(self, input_feature, field_name): unique_field_name = f"{unique_field_name}_{random.randint(0, 9)}" return unique_field_name - def pre_iteration(self): + def find_maximum_object_id(self): """ Determine the maximum OBJECTID for partitioning. """ @@ -465,7 +451,7 @@ def process_input_features( return aliases_with_features, True else: # Loads in dummy feature for this alias for this iteration and sets dummy_used = True - self.update_alias_with_dummy_if_needed( + self.update_empty_alias_type_with_dummy_file( alias, type_info="input", ) @@ -516,7 +502,7 @@ def _process_context_features_and_others( for alias in aliases: if "context_copy" not in self.data[alias]: # Loads in dummy feature for this alias for this iteration and sets dummy_used = True - self.update_alias_with_dummy_if_needed( + self.update_empty_alias_type_with_dummy_file( alias, type_info="context", ) @@ -576,7 +562,7 @@ def append_iteration_to_final(self, alias): def partition_iteration(self): aliases = self.data.keys() - max_object_id = self.pre_iteration() + max_object_id = self.find_maximum_object_id() self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) self.initialize_dummy_used() From 509ed6bf143ef459867d4f34210199cd9c8eeb59 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 23 Mar 2024 09:04:17 +0100 Subject: [PATCH 11/19] refactored nested dictionaries names for more clarity refactored the nested dictionary names in partition iterator so it is more clear what they hold and what they are intended to be used for --- .../partition_iterator_state_based.py | 114 +++++++++--------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 7957bdc7..8e17e323 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -44,7 +44,7 @@ def __init__( :param partition_method: Method used for creating cartographic partitions. """ - self.data = {} + self.nested_alias_type_data = {} self.alias_path_data = alias_path_data self.alias_path_outputs = alias_path_outputs or {} self.root_file_partition_iterator = root_file_partition_iterator @@ -57,7 +57,7 @@ def __init__( ) self.custom_functions = custom_functions or [] self.iteration_file_paths = [] - self.final_outputs = {} + self.nested_final_outputs = {} self.search_distance = search_distance self.object_id_field = object_id_field @@ -69,17 +69,17 @@ def integrate_initial_data(self, alias_path_data): # Process initial alias_path_data for inputs and outputs for alias, info in alias_path_data.items(): type_info, path_info = info - if alias not in self.data: - self.data[alias] = {} - self.data[alias][type_info] = path_info + if alias not in self.nested_alias_type_data: + self.nested_alias_type_data[alias] = {} + self.nested_alias_type_data[alias][type_info] = path_info def unpack_alias_path_outputs(self, alias_path_outputs): - self.final_outputs = {} + self.nested_final_outputs = {} for alias, info in alias_path_outputs.items(): type_info, path_info = info - if alias not in self.final_outputs: - self.final_outputs[alias] = {} - self.final_outputs[alias][type_info] = path_info + if alias not in self.nested_final_outputs: + self.nested_final_outputs[alias] = {} + self.nested_final_outputs[alias][type_info] = path_info def configure_alias_and_type( self, @@ -88,12 +88,12 @@ def configure_alias_and_type( type_path, ): # Check if alias exists - if alias not in self.data: - print(f"Alias '{alias}' not found in data.") + if alias not in self.nested_alias_type_data: + print(f"Alias '{alias}' not found in nested_alias_type_data.") return # Update path of an existing type or add a new type with the provided path - self.data[alias][type_name] = type_path + self.nested_alias_type_data[alias][type_name] = type_path print(f"Set path for type '{type_name}' in alias '{alias}' to: {type_path}") def create_new_alias( @@ -103,25 +103,25 @@ def create_new_alias( initial_type_path=None, ): # Check if alias already exists - if alias in self.data: + if alias in self.nested_alias_type_data: raise ValueError(f"Alias {alias} already exists.") - # Initialize data for alias + # Initialize nested_alias_type_data for alias if initial_type_name: # Create alias with initial type and path - self.data[alias] = {initial_type_name: initial_type_path} + self.nested_alias_type_data[alias] = {initial_type_name: initial_type_path} else: # Initialize alias as an empty dictionary - self.data[alias] = {} + self.nested_alias_type_data[alias] = {} print( - f"Created new alias '{alias}' in data with type '{initial_type_name}' and path: {initial_type_path}" + f"Created new alias '{alias}' in nested_alias_type_data with type '{initial_type_name}' and path: {initial_type_path}" ) def create_cartographic_partitions(self): all_features = [ path - for alias, types in self.data.items() + for alias, types in self.nested_alias_type_data.items() for type_key, path in types.items() if type_key in ["input_copy", "context_copy"] and path is not None ] @@ -151,8 +151,8 @@ def delete_feature_class(self, feature_class_path, alias=None, output_type=None) def delete_final_outputs(self): """Deletes all final output files if they exist.""" - for alias in self.final_outputs: - for _, output_file_path in self.final_outputs[alias].items(): + for alias in self.nested_final_outputs: + for _, output_file_path in self.nested_final_outputs[alias].items(): if arcpy.Exists(output_file_path): arcpy.management.Delete(output_file_path) print(f"Deleted file: {output_file_path}") @@ -183,7 +183,7 @@ def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]) Args: types_to_include (list): Types for which dummy features should be created. """ - for alias, alias_data in self.data.items(): + for alias, alias_data in self.nested_alias_type_data.items(): for type_info, path in list(alias_data.items()): if type_info in types_to_include and path: dummy_feature_path = f"{self.root_file_partition_iterator}_{alias}_dummy_{self.scale}" @@ -203,38 +203,42 @@ def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]) def initialize_dummy_used(self): # Assuming `aliases` is a list of all your aliases - for alias in self.data: - self.data[alias]["dummy_used"] = False + for alias in self.nested_alias_type_data: + self.nested_alias_type_data[alias]["dummy_used"] = False def reset_dummy_used(self): # Assuming `aliases` is a list of all your aliases - for alias in self.data: - self.data[alias]["dummy_used"] = False + for alias in self.nested_alias_type_data: + self.nested_alias_type_data[alias]["dummy_used"] = False def update_empty_alias_type_with_dummy_file(self, alias, type_info): - # Check if the dummy type exists in the alias data - if "dummy" in self.data[alias]: - # Check if the input type exists in the alias data + # Check if the dummy type exists in the alias nested_alias_type_data + if "dummy" in self.nested_alias_type_data[alias]: + # Check if the input type exists in the alias nested_alias_type_data if ( - type_info in self.data[alias] - and self.data[alias][type_info] is not None + type_info in self.nested_alias_type_data[alias] + and self.nested_alias_type_data[alias][type_info] is not None ): - # Get the dummy path from the alias data - dummy_path = self.data[alias]["dummy"] + # Get the dummy path from the alias nested_alias_type_data + dummy_path = self.nested_alias_type_data[alias]["dummy"] # Set the value of the existing type_info to the dummy path - self.data[alias][type_info] = dummy_path - self.data[alias]["dummy_used"] = True + self.nested_alias_type_data[alias][type_info] = dummy_path + self.nested_alias_type_data[alias]["dummy_used"] = True print( f"The '{type_info}' for alias '{alias}' was updated with dummy path: {dummy_path}" ) else: - print(f"'{type_info}' does not exist for alias '{alias}' in data.") + print( + f"'{type_info}' does not exist for alias '{alias}' in nested_alias_type_data." + ) else: - print(f"'dummy' type does not exist for alias '{alias}' in data.") + print( + f"'dummy' type does not exist for alias '{alias}' in nested_alias_type_data." + ) def write_data_to_json(self, file_name): with open(file_name, "w") as file: - json.dump(self.data, file, indent=4) + json.dump(self.nested_alias_type_data, file, indent=4) def generate_unique_field_name(self, input_feature, field_name): existing_field_names = [field.name for field in arcpy.ListFields(input_feature)] @@ -263,7 +267,7 @@ def find_maximum_object_id(self): print(f"Error in finding max {self.object_id_field}: {e}") def prepare_input_data(self): - for alias, types in self.data.items(): + for alias, types in self.nested_alias_type_data.items(): if "input" in types: input_data_path = types["input"] input_data_copy = ( @@ -274,9 +278,9 @@ def prepare_input_data(self): in_data=input_data_path, out_data=input_data_copy, ) - print(f"Copied input data for: {alias}") + print(f"Copied input nested_alias_type_data for: {alias}") - # Add a new type for the alias the copied input data + # Add a new type for the alias the copied input nested_alias_type_data self.configure_alias_and_type( alias=alias, type_name="input_copy", @@ -325,7 +329,7 @@ def prepare_input_data(self): in_data=context_data_path, out_data=context_data_copy, ) - print(f"Copied context data for: {alias}") + print(f"Copied context nested_alias_type_data for: {alias}") self.configure_alias_and_type( alias=alias, @@ -358,11 +362,11 @@ def process_input_features( """ Process input features for a given partition. """ - if "input_copy" not in self.data[alias]: + if "input_copy" not in self.nested_alias_type_data[alias]: return None, False - if "input_copy" in self.data[alias]: - input_path = self.data[alias]["input_copy"] + if "input_copy" in self.nested_alias_type_data[alias]: + input_path = self.nested_alias_type_data[alias]["input_copy"] input_features_partition_selection = ( f"in_memory/{alias}_partition_base_select_{self.scale}" ) @@ -463,7 +467,7 @@ def process_input_features( def _process_inputs_in_partition(self, aliases, iteration_partition, object_id): inputs_present_in_partition = False for alias in aliases: - if "input_copy" in self.data[alias]: + if "input_copy" in self.nested_alias_type_data[alias]: _, input_present = self.process_input_features( alias, iteration_partition, object_id ) @@ -476,8 +480,8 @@ def process_context_features(self, alias, iteration_partition): """ Process context features for a given partition if input features are present. """ - if "context_copy" in self.data[alias]: - context_path = self.data[alias]["context_copy"] + if "context_copy" in self.nested_alias_type_data[alias]: + context_path = self.nested_alias_type_data[alias]["context_copy"] context_selection_path = f"{self.root_file_partition_iterator}_{alias}_context_iteration_selection" self.iteration_file_paths.append(context_selection_path) @@ -500,7 +504,7 @@ def _process_context_features_and_others( self, aliases, iteration_partition, object_id ): for alias in aliases: - if "context_copy" not in self.data[alias]: + if "context_copy" not in self.nested_alias_type_data[alias]: # Loads in dummy feature for this alias for this iteration and sets dummy_used = True self.update_empty_alias_type_with_dummy_file( alias, @@ -513,17 +517,17 @@ def _process_context_features_and_others( self.process_context_features(alias, iteration_partition) def append_iteration_to_final(self, alias): - # Guard clause if alias doesn't exist in final_outputs - if alias not in self.final_outputs: + # Guard clause if alias doesn't exist in nested_final_outputs + if alias not in self.nested_final_outputs: return # For each type under current alias, append the result of the current iteration - for type_info, final_output_path in self.final_outputs[alias].items(): + for type_info, final_output_path in self.nested_final_outputs[alias].items(): # Skipping append if the alias is a dummy feature - if self.data[alias]["dummy_used"]: + if self.nested_alias_type_data[alias]["dummy_used"]: continue - input_feature_class = self.data[alias][type_info] + input_feature_class = self.nested_alias_type_data[alias][type_info] if ( not arcpy.Exists(input_feature_class) @@ -561,7 +565,7 @@ def append_iteration_to_final(self, alias): ) def partition_iteration(self): - aliases = self.data.keys() + aliases = self.nested_alias_type_data.keys() max_object_id = self.find_maximum_object_id() self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) @@ -656,7 +660,7 @@ def run(self): -self.data = { +self.nested_alias_type_data = { 'alias_1': { 'input': 'file_path_1', 'function_1': 'file_path_3', From 968c24743822f79fbad88d51e9a7745c2980c218 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 23 Mar 2024 15:41:11 +0100 Subject: [PATCH 12/19] Removed unused instance variables Removed unused instance variables and general slight restructuring of partition_iterator_state_based.py --- .../partition_iterator_state_based.py | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 8e17e323..7d137126 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -45,11 +45,12 @@ def __init__( """ self.nested_alias_type_data = {} + self.nested_final_outputs = {} self.alias_path_data = alias_path_data self.alias_path_outputs = alias_path_outputs or {} self.root_file_partition_iterator = root_file_partition_iterator self.scale = scale - self.output_feature_class = alias_path_outputs + self.feature_count = feature_count self.partition_method = partition_method self.partition_feature = ( @@ -57,13 +58,10 @@ def __init__( ) self.custom_functions = custom_functions or [] self.iteration_file_paths = [] - self.nested_final_outputs = {} + self.search_distance = search_distance self.object_id_field = object_id_field - - self.input_data_copy = None self.max_object_id = None - self.final_append_feature = None def integrate_initial_data(self, alias_path_data): # Process initial alias_path_data for inputs and outputs @@ -258,11 +256,10 @@ def find_maximum_object_id(self): self.object_id_field, sql_clause=(None, f"ORDER BY {self.object_id_field} DESC"), ) as cursor: - max_object_id = next(cursor)[0] + self.max_object_id = next(cursor)[0] - print(f"Maximum {self.object_id_field} found: {max_object_id}") + print(f"Maximum {self.object_id_field} found: {self.max_object_id}") - return max_object_id except Exception as e: print(f"Error in finding max {self.object_id_field}: {e}") @@ -566,7 +563,7 @@ def append_iteration_to_final(self, alias): def partition_iteration(self): aliases = self.nested_alias_type_data.keys() - max_object_id = self.find_maximum_object_id() + self.find_maximum_object_id() self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) self.initialize_dummy_used() @@ -574,7 +571,7 @@ def partition_iteration(self): self.delete_iteration_files(*self.iteration_file_paths) self.iteration_file_paths.clear() - for object_id in range(1, max_object_id + 1): + for object_id in range(1, self.max_object_id + 1): self.reset_dummy_used() self.iteration_file_paths.clear() iteration_partition = f"{self.partition_feature}_{object_id}" @@ -620,6 +617,8 @@ def run(self): # Define your input feature classes and their aliases building_points = "building_points" building_polygons = "building_polygons" + church_hospital = "church_hospital" + restriction_lines = "restriction_lines" inputs = { building_points: [ @@ -654,6 +653,38 @@ def run(self): # Run the partition iterator partition_iterator.run() + # inputs_2 = { + # church_hospital: [ + # "input", + # Building_N100.polygon_propogate_displacement___hospital_church_points___n100_building.value, + # ], + # restriction_lines: [ + # "input", + # Building_N100.polygon_propogate_displacement___begrensningskurve_500m_from_displaced_polygon___n100_building.value, + # ], + # } + # + # outputs_2 = { + # church_hospital: [ + # "input", + # f"{Building_N100.iteration__partition_iterator_final_output_points__n100.value}_church_hospital", + # ], + # restriction_lines: [ + # "input", + # f"{Building_N100.iteration__partition_iterator_final_output_polygons__n100.value}_restriction_lines", + # ], + # } + # + # partition_iterator_2 = PartitionIterator( + # alias_path_data=inputs_2, + # alias_path_outputs=outputs_2, + # root_file_partition_iterator=f"{Building_N100.iteration__partition_iterator__n100.value}_2", + # scale=env_setup.global_config.scale_n100, + # ) + # + # # Run the partition iterator + # partition_iterator_2.run() + """" Can I use pattern matching (match) to find the alias for each param? From 40d6c7b5f92a330c5bc1b85a61fc8cdaf89ad7ac Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 23 Mar 2024 16:08:02 +0100 Subject: [PATCH 13/19] Refactor partition_iterator_state_based.py for clarity and efficiency Removed unused instance variables and rearranged the setup procedure for easier understanding. Improved variable naming for consistency. Updated method names to better reflect their actions. --- .../partition_iterator_state_based.py | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 7d137126..d6e63244 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -44,26 +44,31 @@ def __init__( :param partition_method: Method used for creating cartographic partitions. """ - self.nested_alias_type_data = {} - self.nested_final_outputs = {} - self.alias_path_data = alias_path_data - self.alias_path_outputs = alias_path_outputs or {} + # Raw inputs and initial setup + self.raw_input_data = alias_path_data + self.raw_output_data = alias_path_outputs or {} self.root_file_partition_iterator = root_file_partition_iterator self.scale = scale - + self.search_distance = search_distance self.feature_count = feature_count self.partition_method = partition_method + self.object_id_field = object_id_field + + # Initial processing results + self.nested_alias_type_data = {} + self.nested_final_outputs = {} + + # Variables related to features and iterations self.partition_feature = ( f"{root_file_partition_iterator}_partition_feature_{scale}" ) - self.custom_functions = custom_functions or [] - self.iteration_file_paths = [] - - self.search_distance = search_distance - self.object_id_field = object_id_field self.max_object_id = None + self.iteration_file_paths_list = [] + + # Variables related to custom operations + self.custom_functions = custom_functions or [] - def integrate_initial_data(self, alias_path_data): + def unpack_alias_path_data(self, alias_path_data): # Process initial alias_path_data for inputs and outputs for alias, info in alias_path_data.items(): type_info, path_info = info @@ -342,7 +347,7 @@ def select_partition_feature(self, iteration_partition, object_id): """ Selects partition feature based on OBJECTID. """ - self.iteration_file_paths.append(iteration_partition) + self.iteration_file_paths_list.append(iteration_partition) custom_arcpy.select_attribute_and_make_permanent_feature( input_layer=self.partition_feature, expression=f"{self.object_id_field} = {object_id}", @@ -367,7 +372,7 @@ def process_input_features( input_features_partition_selection = ( f"in_memory/{alias}_partition_base_select_{self.scale}" ) - self.iteration_file_paths.append(input_features_partition_selection) + self.iteration_file_paths_list.append(input_features_partition_selection) input_feature_count = custom_arcpy.select_location_and_make_feature_layer( input_layer=input_path, @@ -394,7 +399,7 @@ def process_input_features( ) iteration_append_feature = f"{self.root_file_partition_iterator}_{alias}_iteration_append_feature_{self.scale}" - self.iteration_file_paths.append(iteration_append_feature) + self.iteration_file_paths_list.append(iteration_append_feature) PartitionIterator.create_feature_class( full_feature_path=iteration_append_feature, @@ -408,7 +413,7 @@ def process_input_features( ) input_features_partition_context_selection = f"in_memory/{alias}_input_features_partition_context_selection_{self.scale}" - self.iteration_file_paths.append( + self.iteration_file_paths_list.append( input_features_partition_context_selection ) @@ -480,7 +485,7 @@ def process_context_features(self, alias, iteration_partition): if "context_copy" in self.nested_alias_type_data[alias]: context_path = self.nested_alias_type_data[alias]["context_copy"] context_selection_path = f"{self.root_file_partition_iterator}_{alias}_context_iteration_selection" - self.iteration_file_paths.append(context_selection_path) + self.iteration_file_paths_list.append(context_selection_path) custom_arcpy.select_location_and_make_permanent_feature( input_layer=context_path, @@ -538,8 +543,8 @@ def append_iteration_to_final(self, alias): partition_target_selection = ( f"in_memory/{alias}_{type_info}_partition_target_selection_{self.scale}" ) - self.iteration_file_paths.append(partition_target_selection) - self.iteration_file_paths.append(input_feature_class) + self.iteration_file_paths_list.append(partition_target_selection) + self.iteration_file_paths_list.append(input_feature_class) # Apply feature selection custom_arcpy.select_attribute_and_make_permanent_feature( @@ -568,12 +573,12 @@ def partition_iteration(self): self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) self.initialize_dummy_used() - self.delete_iteration_files(*self.iteration_file_paths) - self.iteration_file_paths.clear() + self.delete_iteration_files(*self.iteration_file_paths_list) + self.iteration_file_paths_list.clear() for object_id in range(1, self.max_object_id + 1): self.reset_dummy_used() - self.iteration_file_paths.clear() + self.iteration_file_paths_list.clear() iteration_partition = f"{self.partition_feature}_{object_id}" self.select_partition_feature(iteration_partition, object_id) @@ -587,15 +592,15 @@ def partition_iteration(self): if inputs_present_in_partition: for alias in aliases: self.append_iteration_to_final(alias) - self.delete_iteration_files(*self.iteration_file_paths) + self.delete_iteration_files(*self.iteration_file_paths_list) else: - self.delete_iteration_files(*self.iteration_file_paths) + self.delete_iteration_files(*self.iteration_file_paths_list) @timing_decorator def run(self): - self.integrate_initial_data(self.alias_path_data) - if self.alias_path_outputs is not None: - self.unpack_alias_path_outputs(self.alias_path_outputs) + self.unpack_alias_path_data(self.raw_input_data) + if self.raw_output_data is not None: + self.unpack_alias_path_outputs(self.raw_output_data) self.delete_final_outputs() self.prepare_input_data() From 781e8a1789edf45814f4a772fe125392fa0c12f6 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 23 Mar 2024 17:04:16 +0100 Subject: [PATCH 14/19] Added type hinting to params for partition iterator Added type hinting to params for partition iterator, making using it easier to use --- custom_tools/partition_iterator_state_based.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index d6e63244..ec3d312b 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -2,6 +2,7 @@ import os import random import json +from typing import Dict, Tuple, Literal import env_setup.global_config from env_setup import environment_setup @@ -23,15 +24,15 @@ class PartitionIterator: def __init__( self, - alias_path_data, - root_file_partition_iterator, - scale, - alias_path_outputs, + alias_path_data: Dict[str, Tuple[Literal["input", "context"], str]], + alias_path_outputs: Dict[str, Tuple[str, str]], + root_file_partition_iterator: str, + scale: str, custom_functions=None, - feature_count="15000", - partition_method="FEATURES", - search_distance="500 Meters", - object_id_field="OBJECTID", + feature_count: str = "15000", + partition_method: Literal["FEATURES", "VERTICES"] = "FEATURES", + search_distance: str = "500 Meters", + object_id_field: str = "OBJECTID", ): """ Initialize the PartitionIterator with input datasets for partitioning and processing. From 582bb9139e62b909559813ba4c41652270e0b69e Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sat, 23 Mar 2024 17:53:33 +0100 Subject: [PATCH 15/19] Added additional comments Added comments to better explain certain logics --- custom_tools/partition_iterator_state_based.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index ec3d312b..c7cb488b 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -366,6 +366,7 @@ def process_input_features( Process input features for a given partition. """ if "input_copy" not in self.nested_alias_type_data[alias]: + # If there are no inputs to process, return None for the aliases and a flag indicating no input was present. return None, False if "input_copy" in self.nested_alias_type_data[alias]: @@ -375,7 +376,7 @@ def process_input_features( ) self.iteration_file_paths_list.append(input_features_partition_selection) - input_feature_count = custom_arcpy.select_location_and_make_feature_layer( + custom_arcpy.select_location_and_make_feature_layer( input_layer=input_path, overlap_type=custom_arcpy.OverlapType.HAVE_THEIR_CENTER_IN.value, select_features=iteration_partition, @@ -455,6 +456,7 @@ def process_input_features( print( f"iteration partition {input_features_partition_context_selection} appended to {iteration_append_feature}" ) + # Return the processed input features and a flag indicating successful operation return aliases_with_features, True else: # Loads in dummy feature for this alias for this iteration and sets dummy_used = True @@ -465,15 +467,18 @@ def process_input_features( print( f"iteration partition {object_id} has no features for {alias} in the partition feature" ) + # If there are no inputs to process, return None for the aliases and a flag indicating no input was present. return None, False def _process_inputs_in_partition(self, aliases, iteration_partition, object_id): inputs_present_in_partition = False for alias in aliases: if "input_copy" in self.nested_alias_type_data[alias]: + # Using process_input_features to check whether inputs are present _, input_present = self.process_input_features( alias, iteration_partition, object_id ) + # Sets inputs_present_in_partition as True if any alias in partition has input present. Otherwise it remains False. inputs_present_in_partition = ( inputs_present_in_partition or input_present ) From 4d8743bbecdae86054d933ed31af2d645693b3b7 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sun, 24 Mar 2024 13:09:22 +0100 Subject: [PATCH 16/19] Refactor partition iterator and add file documentation feature Added functionality for recording and exporting state data into JSON files for later examination in the partition iterator module. Further refactored code to improve efficiency and readability, including removing repetitive code, clarifying comments, and optimizing certain operations. The main function 'partition_iteration' and the script's main clause now handle data exporting at various points of execution. --- .../partition_iterator_state_based.py | 153 ++++++++++++++++-- 1 file changed, 141 insertions(+), 12 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index c7cb488b..4ea9628a 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -1,5 +1,7 @@ import arcpy import os +import re +import shutil import random import json from typing import Dict, Tuple, Literal @@ -29,6 +31,7 @@ def __init__( root_file_partition_iterator: str, scale: str, custom_functions=None, + dictionary_documentation_path: str = None, feature_count: str = "15000", partition_method: Literal["FEATURES", "VERTICES"] = "FEATURES", search_distance: str = "500 Meters", @@ -49,6 +52,13 @@ def __init__( self.raw_input_data = alias_path_data self.raw_output_data = alias_path_outputs or {} self.root_file_partition_iterator = root_file_partition_iterator + if "." in dictionary_documentation_path: + self.dictionary_documentation_path = re.sub( + r"\.[^.]*$", "", dictionary_documentation_path + ) + else: + self.dictionary_documentation_path = dictionary_documentation_path + self.scale = scale self.search_distance = search_distance self.feature_count = feature_count @@ -65,10 +75,20 @@ def __init__( ) self.max_object_id = None self.iteration_file_paths_list = [] + self.first_call_directory_documentation = True # Variables related to custom operations self.custom_functions = custom_functions or [] + # self.handle_data_export( + # file_path=self.dictionary_documentation_path, + # alias_type_data=self.nested_alias_type_data, + # final_outputs=self.nested_final_outputs, + # file_name="initialization", + # iteration=False, + # object_id=None, + # ) + def unpack_alias_path_data(self, alias_path_data): # Process initial alias_path_data for inputs and outputs for alias, info in alias_path_data.items(): @@ -240,9 +260,110 @@ def update_empty_alias_type_with_dummy_file(self, alias, type_info): f"'dummy' type does not exist for alias '{alias}' in nested_alias_type_data." ) - def write_data_to_json(self, file_name): - with open(file_name, "w") as file: - json.dump(self.nested_alias_type_data, file, indent=4) + def create_directory( + self, + root_path: str, + target_dir: str, + iteration: bool, + ) -> str: + """ + Creates a directory at the given root_path for the target_dir. + Args: + root_path: The root directory where initial structure is created + target_dir: The target where the created directory should be placed + iteration: Boolean flag indicating if the iteration_documentation should be added + Returns: + A string containing the absolute path of the created directory. + """ + + # Determine base directory + directory_path = os.path.join(root_path, f"{target_dir}") + + # Ensure that the directory exists + os.makedirs(directory_path, exist_ok=True) + + if iteration: + iteration_documentation_dir = os.path.join( + directory_path, "iteration_documentation" + ) + os.makedirs(iteration_documentation_dir, exist_ok=True) + + return iteration_documentation_dir + + return directory_path + + def write_data_to_json( + self, + data: dict, + file_path: str, + file_name: str, + object_id=None, + ) -> None: + """ + Writes dictionary into a json file. + + Args: + data: The data to write. + file_path: The complete path (directory+file_name) where the file should be created + file_name: The name of the file to create + object_id: If provided, object_id will also be part of the file name. + """ + + if object_id: + complete_file_path = os.path.join( + file_path, f"{file_name}_{object_id}.json" + ) + else: + complete_file_path = os.path.join(file_path, f"{file_name}.json") + + with open(complete_file_path, "w") as f: + json.dump(data, f, indent=4) + + def handle_data_export( + self, + file_path: str = None, + alias_type_data: dict = None, + final_outputs: dict = None, + file_name: str = None, + iteration: bool = False, + object_id=None, + ) -> None: + """ + Handles the export of alias type data and final outputs into separate json files. + + Args: + file_path: The complete file path where to create the output directories. + alias_type_data: The alias type data to export. + final_outputs: The final outputs data to export. + file_name: The name of the file to create + iteration: Boolean flag indicating if the iteration_documentation should be added + object_id: Object ID to be included in the file name if it's an iteration (`iteration==True`). If `None`, will not be used. + """ + + if file_path is None: + file_path = self.dictionary_documentation_path + if alias_type_data is None: + alias_type_data = self.nested_alias_type_data + if final_outputs is None: + final_outputs = self.nested_final_outputs + + if self.first_call_directory_documentation and os.path.exists(file_path): + shutil.rmtree(file_path) + self.first_call_directory_documentation = False + + alias_type_data_directory = self.create_directory( + file_path, "nested_alias_type_data", iteration + ) + final_outputs_directory = self.create_directory( + file_path, "nested_final_outputs", iteration + ) + + self.write_data_to_json( + alias_type_data, alias_type_data_directory, file_name, object_id + ) + self.write_data_to_json( + final_outputs, final_outputs_directory, file_name, object_id + ) def generate_unique_field_name(self, input_feature, field_name): existing_field_names = [field.name for field in arcpy.ListFields(input_feature)] @@ -584,6 +705,11 @@ def partition_iteration(self): for object_id in range(1, self.max_object_id + 1): self.reset_dummy_used() + self.handle_data_export( + file_name="iteration_start", + iteration=True, + object_id=object_id, + ) self.iteration_file_paths_list.clear() iteration_partition = f"{self.partition_feature}_{object_id}" self.select_partition_feature(iteration_partition, object_id) @@ -602,25 +728,27 @@ def partition_iteration(self): else: self.delete_iteration_files(*self.iteration_file_paths_list) + self.handle_data_export( + file_name="iteration_end", + iteration=True, + object_id=object_id, + ) + @timing_decorator def run(self): self.unpack_alias_path_data(self.raw_input_data) if self.raw_output_data is not None: self.unpack_alias_path_outputs(self.raw_output_data) + self.handle_data_export(file_name="post_alias_unpack") + self.delete_final_outputs() self.prepare_input_data() - self.create_cartographic_partitions() - self.write_data_to_json( - Building_N100.iteration___json_documentation_before___building_n100.value - ) + self.create_cartographic_partitions() self.partition_iteration() - - self.write_data_to_json( - Building_N100.iteration___json_documentation_after___building_n100.value - ) + self.handle_data_export(file_name="post_everything") if __name__ == "__main__": @@ -659,6 +787,7 @@ def run(self): alias_path_outputs=outputs, root_file_partition_iterator=Building_N100.iteration__partition_iterator__n100.value, scale=env_setup.global_config.scale_n100, + dictionary_documentation_path=Building_N100.iteration___partition_iterator_json_documentation___building_n100.value, ) # Run the partition iterator @@ -774,6 +903,6 @@ def run(self): # Thoughts on PartitionIterator: """ -Working on append_iteration_to_final need it to select the correct file from nested dictionary based on type for alias +Building_N100.iteration___json_documentation_after___building_n100.value """ From 9f105fabc802fdaa52315a5452ea6f95154c4a79 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Sun, 24 Mar 2024 13:14:19 +0100 Subject: [PATCH 17/19] Modify function names to improve code clarity Renamed the function 'create_directory' to 'create_directory_json_documentation' and 'handle_data_export' to 'export_dictionaries_to_json' to make the function names more informative and reflective of their action. This ensures better code readability and maintainability. --- custom_tools/partition_iterator_state_based.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/custom_tools/partition_iterator_state_based.py b/custom_tools/partition_iterator_state_based.py index 4ea9628a..a1f58a0e 100644 --- a/custom_tools/partition_iterator_state_based.py +++ b/custom_tools/partition_iterator_state_based.py @@ -260,7 +260,7 @@ def update_empty_alias_type_with_dummy_file(self, alias, type_info): f"'dummy' type does not exist for alias '{alias}' in nested_alias_type_data." ) - def create_directory( + def create_directory_json_documentation( self, root_path: str, target_dir: str, @@ -319,7 +319,7 @@ def write_data_to_json( with open(complete_file_path, "w") as f: json.dump(data, f, indent=4) - def handle_data_export( + def export_dictionaries_to_json( self, file_path: str = None, alias_type_data: dict = None, @@ -351,10 +351,10 @@ def handle_data_export( shutil.rmtree(file_path) self.first_call_directory_documentation = False - alias_type_data_directory = self.create_directory( + alias_type_data_directory = self.create_directory_json_documentation( file_path, "nested_alias_type_data", iteration ) - final_outputs_directory = self.create_directory( + final_outputs_directory = self.create_directory_json_documentation( file_path, "nested_final_outputs", iteration ) @@ -705,7 +705,7 @@ def partition_iteration(self): for object_id in range(1, self.max_object_id + 1): self.reset_dummy_used() - self.handle_data_export( + self.export_dictionaries_to_json( file_name="iteration_start", iteration=True, object_id=object_id, @@ -728,7 +728,7 @@ def partition_iteration(self): else: self.delete_iteration_files(*self.iteration_file_paths_list) - self.handle_data_export( + self.export_dictionaries_to_json( file_name="iteration_end", iteration=True, object_id=object_id, @@ -740,7 +740,7 @@ def run(self): if self.raw_output_data is not None: self.unpack_alias_path_outputs(self.raw_output_data) - self.handle_data_export(file_name="post_alias_unpack") + self.export_dictionaries_to_json(file_name="post_alias_unpack") self.delete_final_outputs() self.prepare_input_data() @@ -748,7 +748,7 @@ def run(self): self.create_cartographic_partitions() self.partition_iteration() - self.handle_data_export(file_name="post_everything") + self.export_dictionaries_to_json(file_name="post_everything") if __name__ == "__main__": From 7e8abd8a8aa7a17b5fd7d8b1a3f16d0dba1def65 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Tue, 2 Apr 2024 12:52:36 +0200 Subject: [PATCH 18/19] Introduced logic from partition_iterator_state_based into partition_iterator.py Introduced logic from partition_iterator_state_based into partition_iterator.py after it being restructured in a separate file --- custom_tools/partition_iterator.py | 1011 +++++++++++-------- file_manager/n100/file_manager_buildings.py | 12 +- 2 files changed, 617 insertions(+), 406 deletions(-) diff --git a/custom_tools/partition_iterator.py b/custom_tools/partition_iterator.py index 1f03f9f2..2c57793e 100644 --- a/custom_tools/partition_iterator.py +++ b/custom_tools/partition_iterator.py @@ -1,10 +1,16 @@ import arcpy import os +import re +import shutil import random +import json +from typing import Dict, Tuple, Literal import env_setup.global_config from env_setup import environment_setup from custom_tools import custom_arcpy +from custom_tools.timing_decorator import timing_decorator + from input_data import input_n50 from file_manager.n100.file_manager_buildings import Building_N100 @@ -14,119 +20,359 @@ class PartitionIterator: """THIS IS WORK IN PROGRESS NOT READY FOR USE YET""" + # Class-level constants + PARTITION_FIELD = "partition_select" + ORIGINAL_ID_FIELD = "original_id_field" + def __init__( self, - alias_path_data, - root_file_partition_iterator, - scale, - output_feature_class, + alias_path_data: Dict[str, Tuple[Literal["input", "context"], str]], + alias_path_outputs: Dict[str, Tuple[str, str]], + root_file_partition_iterator: str, + scale: str, custom_functions=None, - feature_count="15000", - partition_method="FEATURES", - search_distance="500 Meters", - object_id_field="OBJECTID", + dictionary_documentation_path: str = None, + feature_count: str = "15000", + partition_method: Literal["FEATURES", "VERTICES"] = "FEATURES", + search_distance: str = "500 Meters", + object_id_field: str = "OBJECTID", ): """ Initialize the PartitionIterator with input datasets for partitioning and processing. - :param alias_path_inputs: A dictionary of input feature class paths with their aliases. + :param alias_path_data: A nested dictionary of input feature class paths with their aliases. :param root_file_partition_iterator: Base path for in progress outputs. :param scale: Scale for the partitions. - :param output_feature_class: The output feature class for final results. + :param alias_path_outputs: A nested dictionary of output feature class for final results. :param feature_count: Feature count for cartographic partitioning. :param partition_method: Method used for creating cartographic partitions. """ - self.data = { - alias: {"type": type_info, "path": path_info} - for alias, (type_info, path_info) in alias_path_data.items() - } + # Raw inputs and initial setup + self.raw_input_data = alias_path_data + self.raw_output_data = alias_path_outputs or {} self.root_file_partition_iterator = root_file_partition_iterator + if "." in dictionary_documentation_path: + self.dictionary_documentation_path = re.sub( + r"\.[^.]*$", "", dictionary_documentation_path + ) + else: + self.dictionary_documentation_path = dictionary_documentation_path + self.scale = scale - self.output_feature_class = output_feature_class + self.search_distance = search_distance self.feature_count = feature_count self.partition_method = partition_method + self.object_id_field = object_id_field + + # Initial processing results + self.nested_alias_type_data = {} + self.nested_final_outputs = {} + + # Variables related to features and iterations self.partition_feature = ( f"{root_file_partition_iterator}_partition_feature_{scale}" ) + self.max_object_id = None + self.iteration_file_paths_list = [] + self.first_call_directory_documentation = True + + # Variables related to custom operations self.custom_functions = custom_functions or [] - self.iteration_file_paths = [] - self.final_append_features = {} - self.search_distance = search_distance - self.object_id_field = object_id_field + + # self.handle_data_export( + # file_path=self.dictionary_documentation_path, + # alias_type_data=self.nested_alias_type_data, + # final_outputs=self.nested_final_outputs, + # file_name="initialization", + # iteration=False, + # object_id=None, + # ) + + def unpack_alias_path_data(self, alias_path_data): + # Process initial alias_path_data for inputs and outputs + for alias, info in alias_path_data.items(): + type_info, path_info = info + if alias not in self.nested_alias_type_data: + self.nested_alias_type_data[alias] = {} + self.nested_alias_type_data[alias][type_info] = path_info + + def unpack_alias_path_outputs(self, alias_path_outputs): + self.nested_final_outputs = {} + for alias, info in alias_path_outputs.items(): + type_info, path_info = info + if alias not in self.nested_final_outputs: + self.nested_final_outputs[alias] = {} + self.nested_final_outputs[alias][type_info] = path_info + + def configure_alias_and_type( + self, + alias, + type_name, + type_path, + ): + # Check if alias exists + if alias not in self.nested_alias_type_data: + print(f"Alias '{alias}' not found in nested_alias_type_data.") + return + + # Update path of an existing type or add a new type with the provided path + self.nested_alias_type_data[alias][type_name] = type_path + print(f"Set path for type '{type_name}' in alias '{alias}' to: {type_path}") + + def create_new_alias( + self, + alias, + initial_type_name=None, + initial_type_path=None, + ): + # Check if alias already exists + if alias in self.nested_alias_type_data: + raise ValueError(f"Alias {alias} already exists.") + + # Initialize nested_alias_type_data for alias + if initial_type_name: + # Create alias with initial type and path + self.nested_alias_type_data[alias] = {initial_type_name: initial_type_path} + else: + # Initialize alias as an empty dictionary + self.nested_alias_type_data[alias] = {} + + print( + f"Created new alias '{alias}' in nested_alias_type_data with type '{initial_type_name}' and path: {initial_type_path}" + ) def create_cartographic_partitions(self): + all_features = [ + path + for alias, types in self.nested_alias_type_data.items() + for type_key, path in types.items() + if type_key in ["input_copy", "context_copy"] and path is not None + ] + + print(f"all_features: {all_features}") + if all_features: + arcpy.cartography.CreateCartographicPartitions( + in_features=all_features, + out_features=self.partition_feature, + feature_count=self.feature_count, + partition_method=self.partition_method, + ) + print(f"Created partitions in {self.partition_feature}") + else: + print("No input or context features available for creating partitions.") + + def delete_feature_class(self, feature_class_path, alias=None, output_type=None): + """Deletes a feature class if it exists.""" + if arcpy.Exists(feature_class_path): + arcpy.management.Delete(feature_class_path) + if alias and output_type: + print( + f"Deleted existing output feature class for '{alias}' of type '{output_type}': {feature_class_path}" + ) + else: + print(f"Deleted feature class: {feature_class_path}") + + def delete_final_outputs(self): + """Deletes all final output files if they exist.""" + for alias in self.nested_final_outputs: + for _, output_file_path in self.nested_final_outputs[alias].items(): + if arcpy.Exists(output_file_path): + arcpy.management.Delete(output_file_path) + print(f"Deleted file: {output_file_path}") + + def delete_iteration_files(self, *file_paths): + """Deletes multiple feature classes or files. Detailed alias and output_type logging is not available here.""" + for file_path in file_paths: + self.delete_feature_class(file_path) + print(f"Deleted file: {file_path}") + + @staticmethod + def create_feature_class(full_feature_path, template_feature): + """Creates a new feature class from a template feature class, given a full path.""" + out_path, out_name = os.path.split(full_feature_path) + if arcpy.Exists(full_feature_path): + arcpy.management.Delete(full_feature_path) + print(f"Deleted existing feature class: {full_feature_path}") + + arcpy.management.CreateFeatureclass( + out_path=out_path, out_name=out_name, template=template_feature + ) + print(f"Created feature class: {full_feature_path}") + + def create_dummy_features(self, types_to_include=["input_copy", "context_copy"]): """ - Creates cartographic partitions based on all available feature classes, - including inputs and context features. + Creates dummy features for aliases with specified types. + + Args: + types_to_include (list): Types for which dummy features should be created. """ - all_features = [details["path"] for alias, details in self.data.items()] + for alias, alias_data in self.nested_alias_type_data.items(): + for type_info, path in list(alias_data.items()): + if type_info in types_to_include and path: + dummy_feature_path = f"{self.root_file_partition_iterator}_{alias}_dummy_{self.scale}" + PartitionIterator.create_feature_class( + full_feature_path=dummy_feature_path, + template_feature=path, + ) + print( + f"Created dummy feature class for {alias} of type {type_info}: {dummy_feature_path}" + ) + # Update alias state to include this new dummy type and its path + self.configure_alias_and_type( + alias=alias, + type_name="dummy", + type_path=dummy_feature_path, + ) - arcpy.cartography.CreateCartographicPartitions( - in_features=all_features, - out_features=self.partition_feature, - feature_count=self.feature_count, - partition_method=self.partition_method, - ) - print(f"Created partitions in {self.partition_feature}") + def initialize_dummy_used(self): + # Assuming `aliases` is a list of all your aliases + for alias in self.nested_alias_type_data: + self.nested_alias_type_data[alias]["dummy_used"] = False + + def reset_dummy_used(self): + # Assuming `aliases` is a list of all your aliases + for alias in self.nested_alias_type_data: + self.nested_alias_type_data[alias]["dummy_used"] = False + + def update_empty_alias_type_with_dummy_file(self, alias, type_info): + # Check if the dummy type exists in the alias nested_alias_type_data + if "dummy" in self.nested_alias_type_data[alias]: + # Check if the input type exists in the alias nested_alias_type_data + if ( + type_info in self.nested_alias_type_data[alias] + and self.nested_alias_type_data[alias][type_info] is not None + ): + # Get the dummy path from the alias nested_alias_type_data + dummy_path = self.nested_alias_type_data[alias]["dummy"] + # Set the value of the existing type_info to the dummy path + self.nested_alias_type_data[alias][type_info] = dummy_path + self.nested_alias_type_data[alias]["dummy_used"] = True + print( + f"The '{type_info}' for alias '{alias}' was updated with dummy path: {dummy_path}" + ) + else: + print( + f"'{type_info}' does not exist for alias '{alias}' in nested_alias_type_data." + ) + else: + print( + f"'dummy' type does not exist for alias '{alias}' in nested_alias_type_data." + ) - def delete_feature_class( + def create_directory_json_documentation( self, - feature_class_path, - ): - """Deletes a feature class if it exists. - + root_path: str, + target_dir: str, + iteration: bool, + ) -> str: + """ + Creates a directory at the given root_path for the target_dir. Args: - feature_class_path (str): The path to the feature class to be deleted. + root_path: The root directory where initial structure is created + target_dir: The target where the created directory should be placed + iteration: Boolean flag indicating if the iteration_documentation should be added + Returns: + A string containing the absolute path of the created directory. """ - if arcpy.Exists(feature_class_path): - arcpy.management.Delete(feature_class_path) - print(f"Deleted feature class: {feature_class_path}") - def create_feature_class( + # Determine base directory + directory_path = os.path.join(root_path, f"{target_dir}") + + # Ensure that the directory exists + os.makedirs(directory_path, exist_ok=True) + + if iteration: + iteration_documentation_dir = os.path.join( + directory_path, "iteration_documentation" + ) + os.makedirs(iteration_documentation_dir, exist_ok=True) + + return iteration_documentation_dir + + return directory_path + + def write_data_to_json( self, - out_path, - out_name, - template_feature, - ): - """Creates a new feature class from a template feature class. + data: dict, + file_path: str, + file_name: str, + object_id=None, + ) -> None: + """ + Writes dictionary into a json file. - Args: - out_path (str): The output path where the feature class will be created. - out_name (str): The name of the new feature class. - template_feature (str): The path to the template feature class. + Args: + data: The data to write. + file_path: The complete path (directory+file_name) where the file should be created + file_name: The name of the file to create + object_id: If provided, object_id will also be part of the file name. """ - full_out_path = os.path.join( - out_path, - out_name, - ) - if arcpy.Exists(full_out_path): - arcpy.management.Delete(full_out_path) - print(f"Deleted existing feature class: {full_out_path}") - arcpy.management.CreateFeatureclass( - out_path=out_path, - out_name=out_name, - template=template_feature, - ) - print(f"Created feature class: {full_out_path}") + if object_id: + complete_file_path = os.path.join( + file_path, f"{file_name}_{object_id}.json" + ) + else: + complete_file_path = os.path.join(file_path, f"{file_name}.json") - def delete_iteration_files(self, *file_paths): - """Deletes multiple feature classes or files. + with open(complete_file_path, "w") as f: + json.dump(data, f, indent=4) + + def export_dictionaries_to_json( + self, + file_path: str = None, + alias_type_data: dict = None, + final_outputs: dict = None, + file_name: str = None, + iteration: bool = False, + object_id=None, + ) -> None: + """ + Handles the export of alias type data and final outputs into separate json files. Args: - *file_paths: A variable number of file paths to delete. + file_path: The complete file path where to create the output directories. + alias_type_data: The alias type data to export. + final_outputs: The final outputs data to export. + file_name: The name of the file to create + iteration: Boolean flag indicating if the iteration_documentation should be added + object_id: Object ID to be included in the file name if it's an iteration (`iteration==True`). If `None`, will not be used. """ - for file_path in file_paths: - try: - if arcpy.Exists(file_path): - arcpy.Delete_management(file_path) - print(f"Deleted iteration file: {file_path}") - except Exception as e: - print(f"Error deleting {file_path}: {e}") + if file_path is None: + file_path = self.dictionary_documentation_path + if alias_type_data is None: + alias_type_data = self.nested_alias_type_data + if final_outputs is None: + final_outputs = self.nested_final_outputs + + if self.first_call_directory_documentation and os.path.exists(file_path): + shutil.rmtree(file_path) + self.first_call_directory_documentation = False + + alias_type_data_directory = self.create_directory_json_documentation( + file_path, "nested_alias_type_data", iteration + ) + final_outputs_directory = self.create_directory_json_documentation( + file_path, "nested_final_outputs", iteration + ) + + self.write_data_to_json( + alias_type_data, alias_type_data_directory, file_name, object_id + ) + self.write_data_to_json( + final_outputs, final_outputs_directory, file_name, object_id + ) + + def generate_unique_field_name(self, input_feature, field_name): + existing_field_names = [field.name for field in arcpy.ListFields(input_feature)] + unique_field_name = field_name + while unique_field_name in existing_field_names: + unique_field_name = f"{unique_field_name}_{random.randint(0, 9)}" + return unique_field_name - def pre_iteration(self): + def find_maximum_object_id(self): """ Determine the maximum OBJECTID for partitioning. """ @@ -137,415 +383,381 @@ def pre_iteration(self): self.object_id_field, sql_clause=(None, f"ORDER BY {self.object_id_field} DESC"), ) as cursor: - max_object_id = next(cursor)[0] + self.max_object_id = next(cursor)[0] - print(f"Maximum {self.object_id_field} found: {max_object_id}") + print(f"Maximum {self.object_id_field} found: {self.max_object_id}") - return max_object_id except Exception as e: print(f"Error in finding max {self.object_id_field}: {e}") def prepare_input_data(self): - # Iterate only over 'input' type data in self.data - for alias, details in self.data.items(): - if details["type"] == "input": + for alias, types in self.nested_alias_type_data.items(): + if "input" in types: + input_data_path = types["input"] input_data_copy = ( f"{self.root_file_partition_iterator}_{alias}_input_copy" ) + # self.delete_feature_class(input_data_copy) arcpy.management.Copy( - in_data=details["path"], + in_data=input_data_path, out_data=input_data_copy, ) - print(f"Copied input data for: {alias}") + print(f"Copied input nested_alias_type_data for: {alias}") + + # Add a new type for the alias the copied input nested_alias_type_data + self.configure_alias_and_type( + alias=alias, + type_name="input_copy", + type_path=input_data_copy, + ) + + # Making sure the field is unique if it exists a field with the same name + self.PARTITION_FIELD = self.generate_unique_field_name( + input_feature=input_data_copy, + field_name=self.PARTITION_FIELD, + ) - self.data[alias]["path"] = input_data_copy + self.ORIGINAL_ID_FIELD = self.generate_unique_field_name( + input_feature=input_data_copy, + field_name=self.ORIGINAL_ID_FIELD, + ) - partition_field = "partition_select" arcpy.AddField_management( in_table=input_data_copy, - field_name=partition_field, + field_name=self.PARTITION_FIELD, field_type="LONG", ) - print(f"Added field {partition_field}") + print(f"Added field {self.PARTITION_FIELD}") - # Add a unique ID field to the copied feature class, ensuring it's a new field - existing_field_names = [ - field.name for field in arcpy.ListFields(input_data_copy) - ] - orig_id_field = "orig_id_field" - while orig_id_field in existing_field_names: - orig_id_field = f"{orig_id_field}_{random.randint(0, 9)}" arcpy.AddField_management( in_table=input_data_copy, - field_name=orig_id_field, + field_name=self.ORIGINAL_ID_FIELD, field_type="LONG", ) - print(f"Added field {orig_id_field}") + print(f"Added field {self.ORIGINAL_ID_FIELD}") arcpy.CalculateField_management( in_table=input_data_copy, - field=orig_id_field, + field=self.ORIGINAL_ID_FIELD, expression=f"!{self.object_id_field}!", ) - print(f"Calculated field {orig_id_field}") + print(f"Calculated field {self.ORIGINAL_ID_FIELD}") + + if "context" in types: + context_data_path = types["context"] + context_data_copy = ( + f"{self.root_file_partition_iterator}_{alias}_context_copy" + ) + + arcpy.management.Copy( + in_data=context_data_path, + out_data=context_data_copy, + ) + print(f"Copied context nested_alias_type_data for: {alias}") + + self.configure_alias_and_type( + alias=alias, + type_name="context_copy", + type_path=context_data_copy, + ) def custom_function(inputs): outputs = [] return outputs - def delete_existing_outputs(self): - for alias, details in self.data.items(): - if details["type"] == "output": - output_path = details["path"] - self.delete_feature_class(output_path) - print(f"Deleted existing feature class: {output_path}") - - def create_dummy_features(self, alias, details): - dummy_feature_path = ( - f"{self.root_file_partition_iterator}_{alias}_dummy_{self.scale}" - ) - self.create_feature_class( - out_path=os.path.dirname(dummy_feature_path), - out_name=os.path.basename(dummy_feature_path), - template_feature=details["path"], - ) - def select_partition_feature(self, iteration_partition, object_id): """ Selects partition feature based on OBJECTID. """ - self.iteration_file_paths.append(iteration_partition) + self.iteration_file_paths_list.append(iteration_partition) custom_arcpy.select_attribute_and_make_permanent_feature( input_layer=self.partition_feature, expression=f"{self.object_id_field} = {object_id}", output_name=iteration_partition, ) - print(f"Created partition selection for OBJECTID {object_id}") + print(f"\nCreated partition selection for OBJECTID {object_id}") - def process_input_features(self, alias, details, iteration_partition): + def process_input_features( + self, + alias, + iteration_partition, + object_id, + ): """ Process input features for a given partition. """ - input_features_partition_selection = ( - f"in_memory/{alias}_partition_base_select_{self.scale}" - ) - self.iteration_file_paths.append(input_features_partition_selection) - - input_feature_count = custom_arcpy.select_location_and_make_feature_layer( - input_layer=details["path"], - overlap_type=custom_arcpy.OverlapType.HAVE_THEIR_CENTER_IN.value, - select_features=iteration_partition, - output_name=input_features_partition_selection, - ) - return input_feature_count > 0 - - def process_context_features(self, alias, details, iteration_partition): - """ - Process context features for a given partition if input features are present. - """ - context_selection_path = ( - f"{self.root_file_partition_iterator}_{alias}_context_iteration_selection" - ) - self.iteration_file_paths.append(context_selection_path) - - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=details["path"], - overlap_type=custom_arcpy.OverlapType.WITHIN_A_DISTANCE, - select_features=iteration_partition, - output_name=context_selection_path, - selection_type=custom_arcpy.SelectionType.NEW_SELECTION.value, - search_distance=self.search_distance, - ) - - def partition_iteration( - self, - input_data_copy, - partition_feature, - max_object_id, - root_file_partition_iterator, - scale, - partition_field, - orig_id_field, - final_append_feature, - ): - self.delete_existing_outputs() - - for object_id in range(1, max_object_id + 1): - self.iteration_file_paths.clear() + if "input_copy" not in self.nested_alias_type_data[alias]: + # If there are no inputs to process, return None for the aliases and a flag indicating no input was present. + return None, False + + if "input_copy" in self.nested_alias_type_data[alias]: + input_path = self.nested_alias_type_data[alias]["input_copy"] + input_features_partition_selection = ( + f"in_memory/{alias}_partition_base_select_{self.scale}" + ) + self.iteration_file_paths_list.append(input_features_partition_selection) - iteration_partition = f"{self.partition_feature}_{object_id}" - self.select_partition_feature(iteration_partition, object_id) + custom_arcpy.select_location_and_make_feature_layer( + input_layer=input_path, + overlap_type=custom_arcpy.OverlapType.HAVE_THEIR_CENTER_IN.value, + select_features=iteration_partition, + output_name=input_features_partition_selection, + ) - inputs_present_in_partition = False + aliases_with_features = {} + count_points = int( + arcpy.management.GetCount(input_features_partition_selection).getOutput( + 0 + ) + ) + aliases_with_features[alias] = count_points - # Processing 'input' type features. - for alias, details in self.data.items(): - if details["type"] == "input": - inputs_present = self.process_input_features( - alias, details, iteration_partition - ) - inputs_present_in_partition |= inputs_present + if aliases_with_features[alias] > 0: + print(f"{alias} has {count_points} features in {iteration_partition}") - # Processing 'context' type features only if 'input' features are present. - if inputs_present_in_partition: - for alias, details in self.data.items(): - if details["type"] == "context": - self.process_context_features( - alias, details, iteration_partition - ) - - # Creating dummy features and selecting partition features for all types. - for alias, details in self.data.items(): - dummy_feature_path = ( - f"{self.root_file_partition_iterator}_{alias}_dummy_{self.scale}" - ) - self.create_feature_class( - out_path=os.path.dirname(dummy_feature_path), - out_name=os.path.basename(dummy_feature_path), - template_feature=details["path"], - ) + arcpy.CalculateField_management( + in_table=input_features_partition_selection, + field=self.PARTITION_FIELD, + expression="1", + ) - for object_id in range(1, max_object_id + 1): - self.iteration_file_paths.clear() - iteration_partition = f"{self.partition_feature}_{object_id}" - # Flag to check if any input features exist in this partition. - inputs_present_in_partition = False - - # Processing 'input' type features - for alias, details in self.data.items(): - if details["type"] == "input": - input_features_partition_selection = ( - f"in_memory/{alias}_partition_base_select_{scale}" - ) - self.iteration_file_paths.append(input_features_partition_selection) - input_feature_count = custom_arcpy.select_location_and_make_feature_layer( - input_layer=details["path"], - overlap_type=custom_arcpy.OverlapType.HAVE_THEIR_CENTER_IN.value, - select_features=iteration_partition, - output_name=input_features_partition_selection, - ) + iteration_append_feature = f"{self.root_file_partition_iterator}_{alias}_iteration_append_feature_{self.scale}" + self.iteration_file_paths_list.append(iteration_append_feature) - if input_feature_count > 0: - inputs_present_in_partition = True - # Processing 'context' type features only if 'input' features are present in this partition. - if inputs_present_in_partition: - for alias, details in self.data.items(): - if details["type"] == "context": - context_selection_path = f"{self.root_file_partition_iterator}_{alias}_context_iteration_selection_{object_id}" - self.iteration_file_paths.append(context_selection_path) - - custom_arcpy.select_location_and_make_permanent_feature( - input_layer=details["path"], - overlap_type=custom_arcpy.OverlapType.WITHIN_A_DISTANCE, - select_features=iteration_partition, - output_name=context_selection_path, - selection_type=custom_arcpy.SelectionType.NEW_SELECTION.value, - search_distance=self.search_distance, - ) - - aliases_feature_counts = {alias: 0 for alias in self.alias} - - for object_id in range(1, max_object_id + 1): - self.iteration_file_paths.clear() - for alias in self.alias: - # Retrieve the output path for the current alias - output_path = self.outputs.get(alias) - - if object_id == 1: - self.delete_feature_class(output_path) - - print(f"\nProcessing {self.object_id_field} {object_id}") - iteration_partition = f"{partition_feature}_{object_id}" - self.iteration_file_paths.append(iteration_partition) + PartitionIterator.create_feature_class( + full_feature_path=iteration_append_feature, + template_feature=input_features_partition_selection, + ) - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=partition_feature, - expression=f"{self.object_id_field} = {object_id}", - output_name=iteration_partition, - ) + arcpy.management.Append( + inputs=input_features_partition_selection, + target=iteration_append_feature, + schema_type="NO_TEST", + ) - # Check for features for each alias and set features_present accordingly - for alias in self.alias: - input_data_copy = self.file_mapping[alias]["current_output"] - base_partition_selection = ( - f"in_memory/{alias}_partition_base_select_{scale}" + input_features_partition_context_selection = f"in_memory/{alias}_input_features_partition_context_selection_{self.scale}" + self.iteration_file_paths_list.append( + input_features_partition_context_selection ) - self.iteration_file_paths.append(base_partition_selection) custom_arcpy.select_location_and_make_feature_layer( - input_layer=input_data_copy, - overlap_type=custom_arcpy.OverlapType.HAVE_THEIR_CENTER_IN.value, + input_layer=input_path, + overlap_type=custom_arcpy.OverlapType.WITHIN_A_DISTANCE, select_features=iteration_partition, - output_name=base_partition_selection, + output_name=input_features_partition_context_selection, + selection_type=custom_arcpy.SelectionType.NEW_SELECTION.value, + search_distance=self.search_distance, ) - aliases_with_features = 0 + arcpy.management.SelectLayerByLocation( + in_layer=input_features_partition_context_selection, + overlap_type="HAVE_THEIR_CENTER_IN", + select_features=iteration_partition, + selection_type="REMOVE_FROM_SELECTION", + ) - count_points = int( - arcpy.management.GetCount(base_partition_selection).getOutput(0) + arcpy.CalculateField_management( + in_table=input_features_partition_context_selection, + field=self.PARTITION_FIELD, + expression="0", ) - aliases_feature_counts[alias] = count_points - # Check if there are features for this alias - if count_points > 0: - print( - f"{alias} has {count_points} features in {iteration_partition}" - ) - aliases_with_features += 1 + arcpy.management.Append( + inputs=input_features_partition_context_selection, + target=iteration_append_feature, + schema_type="NO_TEST", + ) - iteration_append_feature = f"{root_file_partition_iterator}_{alias}_iteration_append_feature_{scale}" - self.iteration_file_paths.append(iteration_append_feature) + self.configure_alias_and_type( + alias=alias, + type_name="input", + type_path=iteration_append_feature, + ) - self.create_feature_class( - out_path=os.path.dirname(iteration_append_feature), - out_name=os.path.basename(iteration_append_feature), - template_feature=input_data_copy, - ) + print( + f"iteration partition {input_features_partition_context_selection} appended to {iteration_append_feature}" + ) + # Return the processed input features and a flag indicating successful operation + return aliases_with_features, True + else: + # Loads in dummy feature for this alias for this iteration and sets dummy_used = True + self.update_empty_alias_type_with_dummy_file( + alias, + type_info="input", + ) + print( + f"iteration partition {object_id} has no features for {alias} in the partition feature" + ) + # If there are no inputs to process, return None for the aliases and a flag indicating no input was present. + return None, False + + def _process_inputs_in_partition(self, aliases, iteration_partition, object_id): + inputs_present_in_partition = False + for alias in aliases: + if "input_copy" in self.nested_alias_type_data[alias]: + # Using process_input_features to check whether inputs are present + _, input_present = self.process_input_features( + alias, iteration_partition, object_id + ) + # Sets inputs_present_in_partition as True if any alias in partition has input present. Otherwise it remains False. + inputs_present_in_partition = ( + inputs_present_in_partition or input_present + ) + return inputs_present_in_partition - arcpy.CalculateField_management( - in_table=base_partition_selection, - field=partition_field, - expression="1", - ) + def process_context_features(self, alias, iteration_partition): + """ + Process context features for a given partition if input features are present. + """ + if "context_copy" in self.nested_alias_type_data[alias]: + context_path = self.nested_alias_type_data[alias]["context_copy"] + context_selection_path = f"{self.root_file_partition_iterator}_{alias}_context_iteration_selection" + self.iteration_file_paths_list.append(context_selection_path) + + custom_arcpy.select_location_and_make_permanent_feature( + input_layer=context_path, + overlap_type=custom_arcpy.OverlapType.WITHIN_A_DISTANCE, + select_features=iteration_partition, + output_name=context_selection_path, + selection_type=custom_arcpy.SelectionType.NEW_SELECTION.value, + search_distance=self.search_distance, + ) - arcpy.management.Append( - inputs=base_partition_selection, - target=iteration_append_feature, - schema_type="NO_TEST", - ) + self.configure_alias_and_type( + alias=alias, + type_name="context", + type_path=context_selection_path, + ) - base_partition_selection_2 = ( - f"in_memory/{alias}_partition_base_select_2_{scale}" - ) - self.iteration_file_paths.append(base_partition_selection_2) - - custom_arcpy.select_location_and_make_feature_layer( - input_layer=input_data_copy, - overlap_type=custom_arcpy.OverlapType.WITHIN_A_DISTANCE, - select_features=iteration_partition, - output_name=base_partition_selection_2, - selection_type=custom_arcpy.SelectionType.NEW_SELECTION.value, - search_distance=self.search_distance, - ) + def _process_context_features_and_others( + self, aliases, iteration_partition, object_id + ): + for alias in aliases: + if "context_copy" not in self.nested_alias_type_data[alias]: + # Loads in dummy feature for this alias for this iteration and sets dummy_used = True + self.update_empty_alias_type_with_dummy_file( + alias, + type_info="context", + ) + print( + f"iteration partition {object_id} has no context features for {alias} in the partition feature" + ) + else: + self.process_context_features(alias, iteration_partition) + + def append_iteration_to_final(self, alias): + # Guard clause if alias doesn't exist in nested_final_outputs + if alias not in self.nested_final_outputs: + return + + # For each type under current alias, append the result of the current iteration + for type_info, final_output_path in self.nested_final_outputs[alias].items(): + # Skipping append if the alias is a dummy feature + if self.nested_alias_type_data[alias]["dummy_used"]: + continue - arcpy.management.SelectLayerByLocation( - in_layer=base_partition_selection_2, - overlap_type="HAVE_THEIR_CENTER_IN", - select_features=iteration_partition, - selection_type="REMOVE_FROM_SELECTION", - ) + input_feature_class = self.nested_alias_type_data[alias][type_info] - arcpy.CalculateField_management( - in_table=base_partition_selection_2, - field=partition_field, - expression="0", - ) + if ( + not arcpy.Exists(input_feature_class) + or int(arcpy.GetCount_management(input_feature_class).getOutput(0)) <= 0 + ): + print( + f"No features found in partition target selection: {input_feature_class}" + ) + continue - arcpy.management.Append( - inputs=base_partition_selection_2, - target=iteration_append_feature, - schema_type="NO_TEST", - ) + partition_target_selection = ( + f"in_memory/{alias}_{type_info}_partition_target_selection_{self.scale}" + ) + self.iteration_file_paths_list.append(partition_target_selection) + self.iteration_file_paths_list.append(input_feature_class) - print( - f"iteration partition {base_partition_selection_2} appended to {iteration_append_feature}" - ) - else: - print( - f"iteration partition {object_id} has no features for {alias} in the partition feature" - ) + # Apply feature selection + custom_arcpy.select_attribute_and_make_permanent_feature( + input_layer=input_feature_class, + expression=f"{self.PARTITION_FIELD} = 1", + output_name=partition_target_selection, + ) - # If no aliases had features, skip the rest of the processing for this object_id - if aliases_with_features == 0: - for alias in self.alias: - self.delete_iteration_files(*self.iteration_file_paths) - continue + if not arcpy.Exists(final_output_path): + arcpy.management.CopyFeatures( + in_features=partition_target_selection, + out_feature_class=final_output_path, + ) - for func in self.custom_functions: - try: - pass - # Determine inputs for the current function - # inputs = [ - # self.file_mapping[fc]["func_output"] or fc - # for fc in self.input_feature_classes - # ] - - # Call the function and get outputs - outputs = func(inputs) - - # Update file mapping with the outputs - for fc, output in zip(self.input_feature_classes, outputs): - self.file_mapping[fc]["current_output"] = output - except: - print(f"Error in custom function: {func}") - - # Process each alias after custom functions - for alias in self.alias: - if aliases_feature_counts[alias] > 0: - # Retrieve the output path for the current alias - output_path = self.outputs.get(alias) - iteration_append_feature = f"{root_file_partition_iterator}_{alias}_iteration_append_feature_{scale}" - - if not arcpy.Exists(output_path): - self.create_feature_class( - out_path=os.path.dirname(output_path), - out_name=os.path.basename(output_path), - template_feature=iteration_append_feature, - ) - - partition_target_selection = ( - f"in_memory/{alias}_partition_target_selection_{scale}" - ) - self.iteration_file_paths.append(partition_target_selection) + else: + arcpy.management.Append( + inputs=partition_target_selection, + target=final_output_path, + schema_type="NO_TEST", + ) - custom_arcpy.select_attribute_and_make_permanent_feature( - input_layer=iteration_append_feature, - expression=f"{partition_field} = 1", - output_name=partition_target_selection, - ) + def partition_iteration(self): + aliases = self.nested_alias_type_data.keys() + self.find_maximum_object_id() - print( - f"for {alias} in {iteration_append_feature} \nThe input is: {partition_target_selection}\nAppending to {output_path}" - ) + self.create_dummy_features(types_to_include=["input_copy", "context_copy"]) + self.initialize_dummy_used() - arcpy.management.Append( - inputs=partition_target_selection, - target=output_path, - schema_type="NO_TEST", - ) - else: - print( - f"No features found in {alias} for {self.object_id_field} {object_id} to append to {output_path}" - ) + self.delete_iteration_files(*self.iteration_file_paths_list) + self.iteration_file_paths_list.clear() + + for object_id in range(1, self.max_object_id + 1): + self.reset_dummy_used() + self.export_dictionaries_to_json( + file_name="iteration_start", + iteration=True, + object_id=object_id, + ) + self.iteration_file_paths_list.clear() + iteration_partition = f"{self.partition_feature}_{object_id}" + self.select_partition_feature(iteration_partition, object_id) - for alias in self.alias: - self.delete_iteration_files(*self.iteration_file_paths) - print(f"Finished iteration {object_id}") + inputs_present_in_partition = self._process_inputs_in_partition( + aliases, iteration_partition, object_id + ) + if inputs_present_in_partition: + self._process_context_features_and_others( + aliases, iteration_partition, object_id + ) + if inputs_present_in_partition: + for alias in aliases: + self.append_iteration_to_final(alias) + self.delete_iteration_files(*self.iteration_file_paths_list) + else: + self.delete_iteration_files(*self.iteration_file_paths_list) + + self.export_dictionaries_to_json( + file_name="iteration_end", + iteration=True, + object_id=object_id, + ) + @timing_decorator def run(self): - environment_setup.main() - self.create_cartographic_partitions() + self.unpack_alias_path_data(self.raw_input_data) + if self.raw_output_data is not None: + self.unpack_alias_path_outputs(self.raw_output_data) - max_object_id = self.pre_iteration() + self.export_dictionaries_to_json(file_name="post_alias_unpack") + self.delete_final_outputs() self.prepare_input_data() - self.partition_iteration( - [self.file_mapping[alias]["current_output"] for alias in self.alias], - self.partition_feature, - max_object_id, - self.root_file_partition_iterator, - self.scale, - "partition_select", - "id_field", - [self.final_append_features.get(alias) for alias in self.alias], - ) + self.create_cartographic_partitions() + + self.partition_iteration() + self.export_dictionaries_to_json(file_name="post_everything") if __name__ == "__main__": + environment_setup.main() # Define your input feature classes and their aliases building_points = "building_points" building_polygons = "building_polygons" + church_hospital = "church_hospital" + restriction_lines = "restriction_lines" inputs = { building_points: [ @@ -553,24 +765,29 @@ def run(self): Building_N100.data_preparation___matrikkel_bygningspunkt___n100_building.value, ], building_polygons: [ - "context", + "input", input_n50.Grunnriss, ], } outputs = { - building_points: Building_N100.iteration__partition_iterator_final_output_points__n100.value, - building_polygons: Building_N100.iteration__partition_iterator_final_output_polygons__n100.value, + building_points: [ + "input", + Building_N100.iteration__partition_iterator_final_output_points__n100.value, + ], + building_polygons: [ + "input", + Building_N100.iteration__partition_iterator_final_output_polygons__n100.value, + ], } # Instantiate PartitionIterator with necessary parameters partition_iterator = PartitionIterator( alias_path_data=inputs, - # alias_path_outputs=outputs, + alias_path_outputs=outputs, root_file_partition_iterator=Building_N100.iteration__partition_iterator__n100.value, scale=env_setup.global_config.scale_n100, - output_feature_class=Building_N100.iteration__partition_iterator_final_output__n100.value, - # Add other parameters like custom_functions if you have any + dictionary_documentation_path=Building_N100.iteration___partition_iterator_json_documentation___building_n100.value, ) # Run the partition iterator diff --git a/file_manager/n100/file_manager_buildings.py b/file_manager/n100/file_manager_buildings.py index 95c9a258..13ba2468 100644 --- a/file_manager/n100/file_manager_buildings.py +++ b/file_manager/n100/file_manager_buildings.py @@ -893,17 +893,11 @@ class Building_N100(Enum): description="building_polygon_base_partition_selection", ) ) - iteration___json_documentation_before___building_n100 = ( - file_manager.generate_file_name_general_files( - script_source_name=iteration, - description="json_documentation_before", - file_type="json", - ) - ) - iteration___json_documentation_after___building_n100 = ( + + iteration___partition_iterator_json_documentation___building_n100 = ( file_manager.generate_file_name_general_files( script_source_name=iteration, - description="json_documentation_after", + description="partition_iterator_json_documentation", file_type="json", ) ) From 876793917853e3b99863d20afb4a81e382575f5f Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Tue, 2 Apr 2024 12:53:29 +0200 Subject: [PATCH 19/19] Refactored timing_decorator for clarity and separation of concerns Functionality of the `timing_decorator` has been broken down into smaller functions for improved clarity and separation of concerns. Unnecessary code elements have been removed, and the function now logs execution time to both the console and a designated log file. --- custom_tools/timing_decorator.py | 114 +++++++++++++------------------ 1 file changed, 46 insertions(+), 68 deletions(-) diff --git a/custom_tools/timing_decorator.py b/custom_tools/timing_decorator.py index 31cf5e1f..1bac292f 100644 --- a/custom_tools/timing_decorator.py +++ b/custom_tools/timing_decorator.py @@ -1,75 +1,53 @@ import time +import os from functools import wraps -# Importing temporary files from file_manager.n100.file_manager_buildings import Building_N100 -import time -from functools import wraps -# Importing temporary files -from file_manager.n100.file_manager_buildings import Building_N100 +def timing_decorator(func): + """Logs the execution time of a function to both the console and a log file""" + + @wraps(func) + def wrapper(*args, **kwargs): + start_time = time.time() + + result = func(*args, **kwargs) + + elapsed_time = compute_elapsed_time(start_time) + + log_to_console_and_file(func.__name__, elapsed_time) + + return result + + return wrapper + + +def compute_elapsed_time(start_time): + """Computes the elapsed time given a starting time""" + elapsed_time_seconds = time.time() - start_time + + elapsed_minutes, elapsed_seconds = divmod(elapsed_time_seconds, 60) + + return elapsed_minutes, elapsed_seconds + + +def log_to_console_and_file(function_name, elapsed_time): + """Logs a messages to both the console and a file""" + elapsed_minutes, elapsed_seconds = elapsed_time + output = f"{function_name} execution time: {int(elapsed_minutes)} minutes {elapsed_seconds:.0f} seconds" + + log_to_console(output) + log_to_file(output) + + +def log_to_console(message): + """Prints a given message to the console""" + print(message) + -# List to store print statements -print_output = [] - -# Decorator to measure execution time of functions -def timing_decorator(arg=None): - if isinstance(arg, str): # If arg is a string, use it as a custom name - custom_name = arg - - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - start_time = time.time() - result = func(*args, **kwargs) - end_time = time.time() - elapsed_time = end_time - start_time - minutes = int(elapsed_time // 60) - seconds = elapsed_time % 60 - output = f"{custom_name} execution time: {minutes} minutes {seconds:.2f} seconds" - print_output.append(output) # Append to the list - return result - - return wrapper - - return decorator - else: # If arg is not a string (or None), use the function name as the default name - func = arg - - @wraps(func) - def wrapper(*args, **kwargs): - start_time = time.time() - result = func(*args, **kwargs) - end_time = time.time() - elapsed_time = end_time - start_time - minutes = int(elapsed_time // 60) - seconds = elapsed_time % 60 - output = f"{func.__name__} execution time: {minutes} minutes {seconds:.2f} seconds" - print_output.append(output) # Append to the list - return result - - return wrapper - - -# Total elapsed time accumulator -total_elapsed_time = 0 - -# Calculate total elapsed time -for line in print_output: - minutes = int(line.split(":")[1].split()[0]) - seconds = float(line.split(":")[1].split()[2]) - total_elapsed_time += minutes * 60 + seconds - -# Write all print statements to a file -output_file = Building_N100.overview__runtime_all_building_functions__n100.value - -# Write total elapsed time to the file -with open(output_file, "w") as f: - f.write( - f"Total run time: {int(total_elapsed_time // 3600)} hours {int((total_elapsed_time % 3600) // 60)} minutes {total_elapsed_time % 60:.2f} seconds\n\n" - ) - - # Write all print statements to the file with additional newline characters - for line in print_output: - f.write(line + "\n") +def log_to_file(message): + """Writes a given message to a log file""" + log_file_path = Building_N100.overview__runtime_all_building_functions__n100.value + with open(log_file_path, "a") as f: + f.write(message + "\n")