diff --git a/custom_tools/decorators/partition_io_decorator.py b/custom_tools/decorators/partition_io_decorator.py new file mode 100644 index 00000000..8e233726 --- /dev/null +++ b/custom_tools/decorators/partition_io_decorator.py @@ -0,0 +1,10 @@ +def partition_io_decorator(input_param_names=None, output_param_names=None): + def decorator(func): + setattr( + func, + "_partition_io_metadata", + {"inputs": input_param_names, "outputs": output_param_names}, + ) + return func + + return decorator diff --git a/custom_tools/polygon_processor.py b/custom_tools/polygon_processor.py index 54bab67d..c1a07c93 100644 --- a/custom_tools/polygon_processor.py +++ b/custom_tools/polygon_processor.py @@ -7,10 +7,15 @@ import config from env_setup import environment_setup +from decorators.partition_io_decorator import partition_io_decorator from file_manager.n100.file_manager_buildings import Building_N100 from constants.n100_constants import N100_Symbology +@partition_io_decorator( + input_param_names=["input_building_points"], + output_param_names=["output_polygon_feature_class"], +) class PolygonProcessor: """ Example diff --git a/custom_tools/timing_decorator.py b/custom_tools/timing_decorator.py index dcb3c4f6..3f1e034a 100644 --- a/custom_tools/timing_decorator.py +++ b/custom_tools/timing_decorator.py @@ -1,37 +1,47 @@ import time +import inspect +import os from functools import wraps + from file_manager.n100.file_manager_buildings import Building_N100 -def timing_decorator(func=None, name=None): +TIMING_DECORATOR_LOG_FILE_USED = False + + +def timing_decorator(func): """Logs the execution time of a function to both the console and a log file""" - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - start_time = time.time() + @wraps(func) + def wrapper(*args, **kwargs): + start_time = time.time() - result = func(*args, **kwargs) + result = func(*args, **kwargs) - elapsed_time = compute_elapsed_time(start_time) + elapsed_time = compute_elapsed_time(start_time) - log_to_console_and_file(name or func.__name__, elapsed_time) + function_name = func.__name__ + file_path = inspect.getfile(func) + file_name = os.path.basename(file_path) - return result + formatted_file_name = file_name.ljust(40) + formatted_function_name = function_name.ljust(55) - return wrapper + log_to_console_and_file( + f"File name: {formatted_file_name} Function name: {formatted_function_name}", + elapsed_time, + ) - if func is None: - return decorator - else: - return decorator(func) + return result + + return wrapper def compute_elapsed_time(start_time): """Computes the elapsed time given a starting time""" elapsed_time_seconds = time.time() - start_time - elapsed_minutes, elapsed_seconds = divmod(elapsed_time_seconds, 60) + elapsed_minutes, elapsed_seconds = divmod(elapsed_time_seconds, 30) return elapsed_minutes, elapsed_seconds @@ -39,7 +49,9 @@ def compute_elapsed_time(start_time): def log_to_console_and_file(function_name, elapsed_time): """Logs a messages to both the console and a file""" elapsed_minutes, elapsed_seconds = elapsed_time - output = f"{function_name} execution time: {int(elapsed_minutes)} minutes {elapsed_seconds:.0f} seconds" + output = f"{function_name} Execution time: {int(elapsed_minutes)} minutes {elapsed_seconds:.0f} seconds".ljust( + 60 + ) log_to_console(output) log_to_file(output) @@ -52,6 +64,16 @@ def log_to_console(message): def log_to_file(message): """Writes a given message to a log file""" + + global TIMING_DECORATOR_LOG_FILE_USED + log_file_path = Building_N100.overview__runtime_all_building_functions__n100.value + + if os.path.exists(log_file_path) and not TIMING_DECORATOR_LOG_FILE_USED: + # if it's the first time this runtime, delete the log file + os.remove(log_file_path) + + TIMING_DECORATOR_LOG_FILE_USED = True + with open(log_file_path, "a") as f: f.write(message + "\n") diff --git a/generalization/n100/building/building_main.py b/generalization/n100/building/building_main.py index 5e101155..b0a6dd12 100644 --- a/generalization/n100/building/building_main.py +++ b/generalization/n100/building/building_main.py @@ -19,8 +19,9 @@ from generalization.n100.building import polygon_resolve_building_conflicts from generalization.n100.building import removing_overlapping_points + # Main function that runs all the building scripts -@timing_decorator("building_main.py") +@timing_decorator def main(): """ This is the main function that runs all the building scripts. diff --git a/generalization/n100/building/calculate_point_values.py b/generalization/n100/building/calculate_point_values.py index 7045eb4f..066339fc 100644 --- a/generalization/n100/building/calculate_point_values.py +++ b/generalization/n100/building/calculate_point_values.py @@ -13,7 +13,7 @@ from custom_tools.timing_decorator import timing_decorator -@timing_decorator("calculate_point_values.py") +@timing_decorator def main(): """ Summary: diff --git a/generalization/n100/building/calculate_polygon_values.py b/generalization/n100/building/calculate_polygon_values.py index a64ae958..1b22f3d9 100644 --- a/generalization/n100/building/calculate_polygon_values.py +++ b/generalization/n100/building/calculate_polygon_values.py @@ -10,7 +10,7 @@ from custom_tools.timing_decorator import timing_decorator -@timing_decorator("calculate_polygon_values.py") +@timing_decorator def main(): """ Summary: @@ -22,7 +22,6 @@ def main(): def adding_angle_hierarchy_invisibility_fields(): - # Adding multiple fields print("Adding fields...") arcpy.management.AddFields( @@ -48,7 +47,6 @@ def adding_angle_hierarchy_invisibility_fields(): def adding_symbol_val(): - arcpy.AddField_management( in_table=Building_N100.simplify_polygons___spatial_join_polygons___n100_building.value, field_name="symbol_val", diff --git a/generalization/n100/building/data_clean_up.py b/generalization/n100/building/data_clean_up.py index aac2b057..f7e272ce 100644 --- a/generalization/n100/building/data_clean_up.py +++ b/generalization/n100/building/data_clean_up.py @@ -9,7 +9,7 @@ # Main function -@timing_decorator("data_clean_up.py") +@timing_decorator def main(): environment_setup.main() delete_uneccessary_fields() diff --git a/generalization/n100/building/data_preparation.py b/generalization/n100/building/data_preparation.py index 1bd85209..906cac44 100644 --- a/generalization/n100/building/data_preparation.py +++ b/generalization/n100/building/data_preparation.py @@ -17,7 +17,7 @@ from constants.n100_constants import N100_Symbology, N100_SQLResources -@timing_decorator("data_preparation.py") +@timing_decorator def main(): """ Summary: @@ -248,7 +248,6 @@ def matrikkel_and_n50_not_in_urban_areas(): @timing_decorator def railway_station_points_to_polygons(): - # Railway stations from input data railway_stations = input_n100.JernbaneStasjon diff --git a/generalization/n100/building/hospital_church_clusters.py b/generalization/n100/building/hospital_church_clusters.py index 073ea2cf..14edce0b 100644 --- a/generalization/n100/building/hospital_church_clusters.py +++ b/generalization/n100/building/hospital_church_clusters.py @@ -19,7 +19,7 @@ # Main function -@timing_decorator("hospital_church_clusters.py") +@timing_decorator def main(): """ Summary: @@ -365,7 +365,6 @@ def reducing_clusters(): @timing_decorator def hospitals_and_churches_too_close(): - # SQL-expressions to select hospitals and churches sql_select_all_hospital = "BYGGTYP_NBR IN (970, 719)" sql_select_all_church = "BYGGTYP_NBR = 671" diff --git a/generalization/n100/building/point_displacement_with_buffer.py b/generalization/n100/building/point_displacement_with_buffer.py index d665d652..706b072b 100644 --- a/generalization/n100/building/point_displacement_with_buffer.py +++ b/generalization/n100/building/point_displacement_with_buffer.py @@ -12,7 +12,7 @@ from custom_tools import custom_arcpy -@timing_decorator("point_displacement_with_buffer.py") +@timing_decorator def main(): """ Summary: diff --git a/generalization/n100/building/point_propogate_displacement.py b/generalization/n100/building/point_propogate_displacement.py index 386dc77a..87225601 100644 --- a/generalization/n100/building/point_propogate_displacement.py +++ b/generalization/n100/building/point_propogate_displacement.py @@ -15,7 +15,7 @@ from custom_tools.timing_decorator import timing_decorator -@timing_decorator("points_propogate_displacement.py") +@timing_decorator def main(): """ Summary: diff --git a/generalization/n100/building/point_resolve_building_conflicts.py b/generalization/n100/building/point_resolve_building_conflicts.py index 8a298316..e7ce3137 100644 --- a/generalization/n100/building/point_resolve_building_conflicts.py +++ b/generalization/n100/building/point_resolve_building_conflicts.py @@ -19,7 +19,7 @@ iteration_fc = config.resolve_building_conflicts_iteration_feature -@timing_decorator("point_resolve_building_conflicts.py") +@timing_decorator def main(): """ This script resolves building conflicts, both building polygons and points @@ -41,7 +41,6 @@ def main(): @timing_decorator def building_points_to_squares(): - # Transforms all the building points to squares polygon_processor = PolygonProcessor( input_building_points=Building_N100.building_point_buffer_displacement__displaced_building_points__n100.value, @@ -137,7 +136,6 @@ def apply_symbology_to_the_layers(): def barriers_for_rbc(): - input_barriers_for_rbc = [ [ Building_N100.point_resolve_building_conflicts___veg_sti_selection___n100_building_lyrx.value, @@ -198,7 +196,6 @@ def resolve_building_conflicts_1(): def building_polygons_to_keep_after_rbc_1(): - # Sql expression to keep only building polygons that are visible (0) after the tool has run sql_expression_resolve_building_conflicts_polygon = "invisibility = 0" @@ -211,7 +208,6 @@ def building_polygons_to_keep_after_rbc_1(): def transforming_invisible_polygons_to_points_and_then_to_squares(): - # Sql expression to keep only building polygons that have invisbility value 1 after the tool has run sql_expression_resolve_building_conflicts_polygon = "invisibility = 1" @@ -241,7 +237,6 @@ def transforming_invisible_polygons_to_points_and_then_to_squares(): def adding_symbology_to_layers_being_used_for_rbc_2(): - # Building squares (from points, transformed to squares in the first function) that are kept after rbc 1 custom_arcpy.apply_symbology( input_layer=Building_N100.point_resolve_building_conflicts___squares_to_keep_after_rbc1___n100_building.value, @@ -263,7 +258,6 @@ def adding_symbology_to_layers_being_used_for_rbc_2(): def resolve_building_conflicts_2(): - print("Starting resolve building conflicts 2") input_buildings_rbc_2 = [ @@ -283,7 +277,6 @@ def resolve_building_conflicts_2(): def selecting_features_to_be_kept_after_rbc_2(): - sql_expression_resolve_building_conflicts = ( "(invisibility = 0) OR (symbol_val IN (1, 2, 3))" ) @@ -310,7 +303,6 @@ def selecting_features_to_be_kept_after_rbc_2(): def transforming_squares_back_to_points(): - # Squares from points are transformed back to points arcpy.management.FeatureToPoint( in_features=Building_N100.point_resolve_building_conflicts___squares_from_points_rbc2___n100_building.value, @@ -327,7 +319,6 @@ def transforming_squares_back_to_points(): def merging_building_points(): - arcpy.management.Merge( inputs=[ Building_N100.point_resolve_building_conflicts___squares_from_points_transformed_back_to_points___n100_building.value, @@ -338,7 +329,6 @@ def merging_building_points(): def assigning_final_names(): - arcpy.management.CopyFeatures( Building_N100.point_resolve_building_conflicts___building_polygons_rbc2___n100_building.value, Building_N100.point_resolve_building_conflicts___building_polygons_final___n100_building.value, diff --git a/generalization/n100/building/polygon_propogate_displacement.py b/generalization/n100/building/polygon_propogate_displacement.py index 4edd9eb4..23c55b26 100644 --- a/generalization/n100/building/polygon_propogate_displacement.py +++ b/generalization/n100/building/polygon_propogate_displacement.py @@ -20,7 +20,7 @@ from custom_tools.timing_decorator import timing_decorator -@timing_decorator("polygon_propogate_displacement.py") +@timing_decorator def main(): """ Summary: diff --git a/generalization/n100/building/polygon_resolve_building_conflicts.py b/generalization/n100/building/polygon_resolve_building_conflicts.py index ef6165cb..8c536fdd 100644 --- a/generalization/n100/building/polygon_resolve_building_conflicts.py +++ b/generalization/n100/building/polygon_resolve_building_conflicts.py @@ -22,7 +22,7 @@ from custom_tools.timing_decorator import timing_decorator -@timing_decorator("polygon_propogate_displacement.py") +@timing_decorator def main(): """ Summary: @@ -124,7 +124,6 @@ def roads_and_water_barriers_500_m_from_building_polygons(): @timing_decorator def railway_500_m_from_building_polygons(): - # Selecting railway 500 meters from railways (togbane) custom_arcpy.select_location_and_make_permanent_feature( input_layer=input_data.input_n100.Bane, @@ -137,7 +136,6 @@ def railway_500_m_from_building_polygons(): @timing_decorator def create_railway_buffer(): - # Buffering the railways arcpy.analysis.PairwiseBuffer( in_features=Building_N100.polygon_resolve_building_conflicts___railway_500m_from_displaced_polygon___n100_building.value, @@ -148,7 +146,6 @@ def create_railway_buffer(): @timing_decorator def hospital_church_points_to_squares(): - # Selecting hospital and churches from n50 custom_arcpy.select_attribute_and_make_permanent_feature( input_layer=input_data.input_n50.BygningsPunkt, diff --git a/generalization/n100/building/polygon_to_point.py b/generalization/n100/building/polygon_to_point.py index c9fddb64..e18caf5f 100644 --- a/generalization/n100/building/polygon_to_point.py +++ b/generalization/n100/building/polygon_to_point.py @@ -14,7 +14,7 @@ from custom_tools.timing_decorator import timing_decorator -@timing_decorator("creating_points_from_polygon.py") +@timing_decorator def main(): """ This function creates points from small polygons lost during aggregation, and merges @@ -26,7 +26,6 @@ def main(): @timing_decorator def building_polygons_to_points(): - # List of building points which will be spatially joined with building polygons input_points = [ f"{Building_N100.simplify_polygons___simplify_polygon___n100_building.value}_Pnt", diff --git a/generalization/n100/building/removing_overlapping_points.py b/generalization/n100/building/removing_overlapping_points.py index ab44ceac..96b16217 100644 --- a/generalization/n100/building/removing_overlapping_points.py +++ b/generalization/n100/building/removing_overlapping_points.py @@ -11,6 +11,7 @@ from custom_tools.compare_feature_classes import compare_feature_classes +@timing_decorator def main(): environment_setup.main() copying_previous_file() @@ -24,7 +25,6 @@ def main(): @timing_decorator def copying_previous_file(): - # Copying and assigning new name to layer arcpy.management.Copy( in_data=Building_N100.removing_points_in_water_features___final_points___n100_building.value, @@ -34,7 +34,6 @@ def copying_previous_file(): @timing_decorator def detecting_graphic_conflicts(): - arcpy.env.referenceScale = "100000" # Detecting Graphic Conflicts @@ -48,7 +47,6 @@ def detecting_graphic_conflicts(): @timing_decorator def selecting_points_close_to_graphic_conflict_polygons(): - # Find points that are close to the graphic conflict polygons custom_arcpy.select_location_and_make_permanent_feature( input_layer=Building_N100.removing_overlapping_points___all_building_points___n100_building.value, @@ -71,7 +69,6 @@ def selecting_points_close_to_graphic_conflict_polygons(): @timing_decorator def finding_clusters_amongst_the_points(): - # Finding church clusters arcpy.gapro.FindPointClusters( input_points=Building_N100.removing_overlapping_points___points_close_to_graphic_conflict_polygons___n100_building.value, @@ -84,7 +81,6 @@ def finding_clusters_amongst_the_points(): @timing_decorator def selecting_points_in_a_cluster_and_not_in_a_cluster(): - expression_cluster = "CLUSTER_ID > 0" expression_not_cluster = "CLUSTER_ID < 0" @@ -157,7 +153,6 @@ def keep_point_with_highest_hierarchy_for_each_cluster(): @timing_decorator def merging_final_points_together(): - # Merge the final hospital and church layers arcpy.management.Merge( inputs=[ diff --git a/generalization/n100/building/removing_points_in_water_features.py b/generalization/n100/building/removing_points_in_water_features.py index ebad8474..83181bf1 100644 --- a/generalization/n100/building/removing_points_in_water_features.py +++ b/generalization/n100/building/removing_points_in_water_features.py @@ -12,7 +12,7 @@ from custom_tools.timing_decorator import timing_decorator -@timing_decorator("removing_points_in_water_features.py") +@timing_decorator def main(): environment_setup.main() removing_points_in_water_features() diff --git a/generalization/n100/building/simplify_polygons.py b/generalization/n100/building/simplify_polygons.py index 98cd22a7..b7f8a73f 100644 --- a/generalization/n100/building/simplify_polygons.py +++ b/generalization/n100/building/simplify_polygons.py @@ -12,7 +12,7 @@ # Main function -@timing_decorator("simplify_polygons.py") +@timing_decorator def main(): """ Summary: