From 3d38546c4beabea4312184f3fc87fd4c3b191741 Mon Sep 17 00:00:00 2001 From: EllingOftedalKV Date: Thu, 14 Mar 2024 23:43:14 +0100 Subject: [PATCH] Implement dynamic batch parameters in polygon_processor.py A function, calculate_batch_params, has been added to dynamically calculate BATCH_PERCENTAGE and NUMBER_OF_SUBSETS based on the number of total data points. This improves the scaling and performance of the polygon processing operation. The other changes involve restructuring the code for better organization. --- custom_tools/polygon_processor.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/custom_tools/polygon_processor.py b/custom_tools/polygon_processor.py index 3c73e98b..12c3d4f1 100644 --- a/custom_tools/polygon_processor.py +++ b/custom_tools/polygon_processor.py @@ -90,14 +90,30 @@ def __init__( # Delay initialization of attributes that depend on external resources self.spatial_reference_system = None self.origin_id_field = None + # Constants and configurations self.IN_MEMORY_WORKSPACE = config.default_project_workspace self.TEMPORARY_FEATURE_CLASS_NAME = "temporary_polygon_feature_class" - self.BATCH_PERCENTAGE = 0.02 - self.NUMBER_OF_SUBSETS = 5 - self.PERCENTAGE_OF_CPU_CORES = 1.0 + self.BATCH_PERCENTAGE = None + self.NUMBER_OF_SUBSETS = None + self.PERCENTAGE_OF_CPU_CORES = 1 + self.calculate_batch_params(input_building_points) # Utility Functions + def calculate_batch_params(self, input_building_points): + total_data_points = int( + arcpy.GetCount_management(input_building_points).getOutput(0) + ) + if total_data_points < 1000: + self.BATCH_PERCENTAGE = 1 + self.NUMBER_OF_SUBSETS = 1 + elif 1000 <= total_data_points <= 10000: + self.BATCH_PERCENTAGE = 0.1 + self.NUMBER_OF_SUBSETS = 3 + else: + self.BATCH_PERCENTAGE = 0.02 + self.NUMBER_OF_SUBSETS = 5 + def generate_unique_field_name(self, dataset, base_name): existing_field_names = [field.name for field in arcpy.ListFields(dataset)] unique_name = base_name @@ -292,8 +308,9 @@ def run(self): if __name__ == "__main__": - # Example parameters - replace these with actual values suitable for your test + environment_setup.main() + # Example parameters - replace these with actual values suitable for your test polygon_processor = PolygonProcessor( input_building_points=Building_N100.calculate_field_values___points_pre_resolve_building_conflicts___n100_building.value, output_polygon_feature_class=Building_N100.building_point_buffer_displacement__iteration_points_to_square_polygons__n100.value,