From 677a1e04fa4b81e73a6cc4e1e7cc70efb776e512 Mon Sep 17 00:00:00 2001 From: Takara Baumbach Date: Mon, 29 Jan 2024 17:04:26 +0100 Subject: [PATCH 01/12] feat: config @ arg > env > default locations, config info messages, default disabled profiles --- ors-api/ors-config.yml | 173 +--------------- .../org/heigit/ors/api/EngineProperties.java | 3 + .../ors/api/ORSEnvironmentPostProcessor.java | 23 ++- .../listeners/ORSInitContextListener.java | 35 ++-- ors-api/src/main/resources/application.yml | 184 +++++++++++++++++- 5 files changed, 223 insertions(+), 195 deletions(-) diff --git a/ors-api/ors-config.yml b/ors-api/ors-config.yml index 9b37b5f17b..b26920429d 100644 --- a/ors-api/ors-config.yml +++ b/ors-api/ors-config.yml @@ -1,6 +1,6 @@ ##### openrouteservice settings file ##### # This file contains parameters for openrouteservice. -# For a full list of possible parameters see documentation, below are some examples. +# For a full list of possible parameters see https://giscience.github.io/openrouteservice/, below are some examples. ### uncomment the following line to turn on debug output #logging.level.org.heigit: DEBUG @@ -11,9 +11,8 @@ ors: # allowed_origins: "*" # allowed_headers: Content-Type, X-Requested-With, accept, Origin, Access-Control-Request-Method, Access-Control-Request-Headers, Authorization # preflight_max_age: 600 - engine: - graphs_root_path: ./ors-core/data/graphs + engine: ### use the following line to enable mmap mode # graphs_data_access: MMAP @@ -28,168 +27,6 @@ ors: profiles: car: - profile: driving-car - elevation: true - encoder_options: - turn_costs: true - block_fords: false - use_acceleration: true - preparation: - min_network_size: 200 - min_one_way_network_size: 200 - methods: - ch: - enabled: true - threads: 1 - weightings: fastest - lm: - enabled: false - threads: 1 - weightings: fastest,shortest - landmarks: 16 - core: - enabled: true - threads: 1 - weightings: fastest,shortest - landmarks: 64 - lmsets: highways;allow_all - execution: - methods: - ch: - disabling_allowed: true - lm: - disabling_allowed: true - active_landmarks: 6 - core: - disabling_allowed: true - active_landmarks: 6 - ext_storages: - WayCategory: - HeavyVehicle: - WaySurfaceType: - RoadAccessRestrictions: - use_for_warnings: true - -### the following lines are example profile setups. Uncomment profile blocks (and the parent elements) to activate. -# hgv: -# profile: driving-hgv -# encoder_flags_size: 8 -# encoder_options: -# turn_costs: true -# block_fords: false -# use_acceleration: true -# maximum_distance: 100000 -# elevation: true -# preparation: -# min_network_size: 200 -# min_one_way_network_size: 200 -# methods: -# ch: -# enabled: true -# threads: 1 -# weightings: recommended -# core: -# enabled: true -# threads: 1 -# weightings: recommended,shortest -# landmarks: 64 -# lmsets: highways;allow_all -# execution: -# methods: -# ch: -# disabling_allowed: true -# core: -# disabling_allowed: true -# active_landmarks: 6 -# ext_storages: -# WayCategory: -# HeavyVehicle: -# restrictions: true -# WaySurfaceType: -# bike-regular: -# profile: cycling-regular -# encoder_options: -# consider_elevation: true -# turn_costs: true -# block_fords: false -# elevation: true -# ext_storages: -# WayCategory: -# WaySurfaceType: -# HillIndex: -# TrailDifficulty: -# bike-mountain: -# profile: cycling-mountain -# encoder_options: -# consider_elevation: true -# turn_costs: true -# block_fords: false -# elevation: true -# ext_storages: -# WayCategory: -# WaySurfaceType: -# HillIndex: -# TrailDifficulty: -# bike-road: -# profile: cycling-road -# encoder_options: -# consider_elevation: true -# turn_costs: true -# block_fords: false -# elevation: true -# ext_storages: -# WayCategory: -# WaySurfaceType: -# HillIndex: -# TrailDifficulty: -# bike-electric: -# profile: cycling-electric -# encoder_options: -# consider_elevation: true -# turn_costs: true -# block_fords: false -# elevation: true -# ext_storages: -# WayCategory: -# WaySurfaceType: -# HillIndex: -# TrailDifficulty: -# walking: -# profile: foot-walking -# encoder_options: -# block_fords: false -# elevation: true -# ext_storages: -# WayCategory: -# WaySurfaceType: -# HillIndex: -# TrailDifficulty: -# hiking: -# profile: foot-hiking -# encoder_options: -# block_fords: false -# elevation: true -# ext_storages: -# WayCategory: -# WaySurfaceType: -# HillIndex: -# TrailDifficulty: -# wheelchair: -# profile: wheelchair -# encoder_options: -# block_fords: true -# elevation: true -# maximum_snapping_radius: 50 -# ext_storages: -# WayCategory: -# WaySurfaceType: -# Wheelchair: -# KerbsOnCrossings: true -# OsmId: -# public-transport: -# profile: public-transport -# encoder_options: -# block_fords: false -# elevation: true -# maximum_visited_nodes: 1000000 -# gtfs_file: ./src/test/files/vrn_gtfs_cut.zip + enabled: true + hgv: + enabled: true \ No newline at end of file diff --git a/ors-api/src/main/java/org/heigit/ors/api/EngineProperties.java b/ors-api/src/main/java/org/heigit/ors/api/EngineProperties.java index e5d5e3a256..67e74c2038 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/EngineProperties.java +++ b/ors-api/src/main/java/org/heigit/ors/api/EngineProperties.java @@ -96,6 +96,9 @@ public RouteProfileConfiguration[] getConvertedProfiles() { if (profiles != null) { for (Map.Entry profileEntry : profiles.entrySet()) { ProfileProperties profile = profileEntry.getValue(); + if (!profile.isEnabled()) { + continue; + } RouteProfileConfiguration convertedProfile = new RouteProfileConfiguration(); convertedProfile.setName(profileEntry.getKey()); convertedProfile.setEnabled(profile.enabled != null ? profile.enabled : profileDefault.isEnabled()); diff --git a/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java b/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java index 1563ea09d8..ca593abbb5 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java +++ b/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java @@ -22,23 +22,34 @@ public class ORSEnvironmentPostProcessor implements EnvironmentPostProcessor { @Override public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) { // Override values from application.yml with contents of custom config yml file. - // Later in array => higher precedence List configLocations = new ArrayList<>(); - if (!StringUtility.isNullOrEmpty(System.getenv(ORS_CONFIG_LOCATION_ENV))) { - configLocations.add(System.getenv(ORS_CONFIG_LOCATION_ENV)); - } + String output = ""; if (!StringUtility.isNullOrEmpty(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY))) { configLocations.add(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY)); + output = output.concat("Configuration file set by program argument."); + } + if (configLocations.isEmpty() && !StringUtility.isNullOrEmpty(System.getenv(ORS_CONFIG_LOCATION_ENV))) { + configLocations.add(System.getenv(ORS_CONFIG_LOCATION_ENV)); + output = output.concat("Configuration file set by environment variable."); + } + if (configLocations.isEmpty()) { + configLocations.add("./ors-config.yml"); + configLocations.add("./ors-api/ors-config.yml"); + configLocations.add("~/.config/openrouteservice/ors-config.yml"); + configLocations.add("/etc/openrouteservice/ors-config.yml"); + output = output.concat("Configuration file lookup by default locations."); } for (String path : configLocations) { try { List> sources = this.loader.load("yml config", new FileSystemResource(path)); if (!sources.isEmpty()) { environment.getPropertySources().addAfter(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, sources.get(0)); + output = output.concat(" Loaded file '%s'".formatted(path)); + break; } - } catch (IllegalStateException | IOException ex) { - System.out.printf("WARNING: Configuration file '%s' could not be loaded.%n", path); + } catch (IllegalStateException | IOException ignored) { } } + System.setProperty(ORS_CONFIG_LOCATION_PROPERTY, output); } } diff --git a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java index 81460576c0..30d2a0e3a9 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java +++ b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java @@ -20,6 +20,7 @@ */ package org.heigit.ors.api.servlet.listeners; +import com.google.common.base.Strings; import jakarta.servlet.ServletContextEvent; import jakarta.servlet.ServletContextListener; import org.apache.juli.logging.LogFactory; @@ -31,9 +32,9 @@ import org.heigit.ors.routing.RoutingProfileManager; import org.heigit.ors.routing.RoutingProfileManagerStatus; import org.heigit.ors.util.FormatUtility; -import org.heigit.ors.util.StringUtility; -import static org.heigit.ors.api.ORSEnvironmentPostProcessor.ORS_CONFIG_LOCATION_ENV; +import java.util.Map; + import static org.heigit.ors.api.ORSEnvironmentPostProcessor.ORS_CONFIG_LOCATION_PROPERTY; public class ORSInitContextListener implements ServletContextListener { @@ -46,23 +47,25 @@ public ORSInitContextListener(EngineProperties engineProperties) { @Override public void contextInitialized(ServletContextEvent contextEvent) { - if (LOGGER.isDebugEnabled()) { - if (!StringUtility.isNullOrEmpty(System.getenv(ORS_CONFIG_LOCATION_ENV))) { - LOGGER.debug("Configuration loaded by ENV, location: " + System.getenv(ORS_CONFIG_LOCATION_ENV)); - } - if (!StringUtility.isNullOrEmpty(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY))) { - LOGGER.debug("Configuration loaded by ARG, location: " + System.getProperty(ORS_CONFIG_LOCATION_PROPERTY)); + LOGGER.info(""); + if (!Strings.isNullOrEmpty(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY))) { + LOGGER.info(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY)); + } + for (Map.Entry env : System.getenv().entrySet()) { + if (env.getKey().startsWith("ORS_") || env.getKey().startsWith("LOGGING_") || env.getKey().startsWith("SPRINGDOC_") || env.getKey().startsWith("SPRING_") || env.getKey().startsWith("SERVER_")) { + LOGGER.info("ENV Parameter: %s=%s".formatted(env.getKey(), env.getValue())); } } + LOGGER.info(""); final EngineConfig config = EngineConfig.EngineConfigBuilder.init() - .setInitializationThreads(engineProperties.getInitThreads()) - .setPreparationMode(engineProperties.isPreparationMode()) - .setElevationPreprocessed(engineProperties.getElevation().isPreprocessed()) - .setSourceFile(engineProperties.getSourceFile()) - .setGraphsRootPath(engineProperties.getGraphsRootPath()) - .setGraphsDataAccess(engineProperties.getGraphsDataAccess()) - .setProfiles(engineProperties.getConvertedProfiles()) - .buildWithAppConfigOverride(); + .setInitializationThreads(engineProperties.getInitThreads()) + .setPreparationMode(engineProperties.isPreparationMode()) + .setElevationPreprocessed(engineProperties.getElevation().isPreprocessed()) + .setSourceFile(engineProperties.getSourceFile()) + .setGraphsRootPath(engineProperties.getGraphsRootPath()) + .setGraphsDataAccess(engineProperties.getGraphsDataAccess()) + .setProfiles(engineProperties.getConvertedProfiles()) + .buildWithAppConfigOverride(); Runnable runnable = () -> { try { LOGGER.info("Initializing ORS..."); diff --git a/ors-api/src/main/resources/application.yml b/ors-api/src/main/resources/application.yml index a74afbef76..4203c27339 100644 --- a/ors-api/src/main/resources/application.yml +++ b/ors-api/src/main/resources/application.yml @@ -1,7 +1,5 @@ -##### openrouteservice default settings ##### -# This file contains the default parameters for openrouteservice. -# To change values for your instance, you can add a file named "ors-config.yml" to the JVM's working directory. -# The parameters in said file will override any value set here. +##### openrouteservice settings file ##### +# This file contains parameters for openrouteservice. ##### General server settings ##### server: @@ -12,6 +10,7 @@ server: # Keep the context-path at / else the war file run with tomcat will have the context-path of /ors/v2 as well. servlet: context-path: /ors + spring: profiles: active: default @@ -113,7 +112,7 @@ ors: provider: multi cache_path: ./elevation_cache profile_default: - enabled: true + enabled: false elevation: false elevation_smoothing: false encoder_flags_size: 8 @@ -147,3 +146,178 @@ ors: lm: disabling_allowed: true active_landmarks: 8 + profiles: + car: + enabled: false + profile: driving-car + elevation: true + encoder_options: + turn_costs: true + block_fords: false + use_acceleration: true + preparation: + min_network_size: 200 + min_one_way_network_size: 200 + methods: + ch: + enabled: true + threads: 1 + weightings: fastest + lm: + enabled: false + threads: 1 + weightings: fastest,shortest + landmarks: 16 + core: + enabled: true + threads: 1 + weightings: fastest,shortest + landmarks: 64 + lmsets: highways;allow_all + execution: + methods: + ch: + disabling_allowed: true + lm: + disabling_allowed: true + active_landmarks: 6 + core: + disabling_allowed: true + active_landmarks: 6 + ext_storages: + WayCategory: + HeavyVehicle: + WaySurfaceType: + RoadAccessRestrictions: + use_for_warnings: true + hgv: + enabled: false + profile: driving-hgv + encoder_flags_size: 8 + encoder_options: + turn_costs: true + block_fords: false + use_acceleration: true + maximum_distance: 100000 + elevation: true + preparation: + min_network_size: 200 + min_one_way_network_size: 200 + methods: + ch: + enabled: true + threads: 1 + weightings: recommended + core: + enabled: true + threads: 1 + weightings: recommended,shortest + landmarks: 64 + lmsets: highways;allow_all + execution: + methods: + ch: + disabling_allowed: true + core: + disabling_allowed: true + active_landmarks: 6 + ext_storages: + WayCategory: + HeavyVehicle: + restrictions: true + WaySurfaceType: + bike-regular: + enabled: false + profile: cycling-regular + encoder_options: + consider_elevation: true + turn_costs: true + block_fords: false + elevation: true + ext_storages: + WayCategory: + WaySurfaceType: + HillIndex: + TrailDifficulty: + bike-mountain: + enabled: false + profile: cycling-mountain + encoder_options: + consider_elevation: true + turn_costs: true + block_fords: false + elevation: true + ext_storages: + WayCategory: + WaySurfaceType: + HillIndex: + TrailDifficulty: + bike-road: + enabled: false + profile: cycling-road + encoder_options: + consider_elevation: true + turn_costs: true + block_fords: false + elevation: true + ext_storages: + WayCategory: + WaySurfaceType: + HillIndex: + TrailDifficulty: + bike-electric: + enabled: false + profile: cycling-electric + encoder_options: + consider_elevation: true + turn_costs: true + block_fords: false + elevation: true + ext_storages: + WayCategory: + WaySurfaceType: + HillIndex: + TrailDifficulty: + walking: + enabled: false + profile: foot-walking + encoder_options: + block_fords: false + elevation: true + ext_storages: + WayCategory: + WaySurfaceType: + HillIndex: + TrailDifficulty: + hiking: + enabled: false + profile: foot-hiking + encoder_options: + block_fords: false + elevation: true + ext_storages: + WayCategory: + WaySurfaceType: + HillIndex: + TrailDifficulty: + wheelchair: + enabled: false + profile: wheelchair + encoder_options: + block_fords: true + elevation: true + maximum_snapping_radius: 50 + ext_storages: + WayCategory: + WaySurfaceType: + Wheelchair: + KerbsOnCrossings: true + OsmId: + public-transport: + enabled: false + profile: public-transport + encoder_options: + block_fords: false + elevation: true + maximum_visited_nodes: 1000000 + gtfs_file: ./src/test/files/vrn_gtfs_cut.zip From 965e15a3d915d0d7393ec0b05ff22916c51c6b73 Mon Sep 17 00:00:00 2001 From: Julian Psotta Date: Mon, 29 Jan 2024 17:33:20 +0100 Subject: [PATCH 02/12] feat(config): Add bash script to rewrite the application.yml to the ors-config.yml This feature intents to keep the application.yml and the ors-config.yml in sync. The ors-config.yml is intented as a minimal working configuration for users that want an up and running system without configuration. The in-depth configs are still in the ors-config.yml but commented out. The sync script will make sure in a workflow that if the original application.yml changes, the ors-config.yml mirrors the new changes but commented out. --- .github/utils/config_rewrite.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 .github/utils/config_rewrite.sh diff --git a/.github/utils/config_rewrite.sh b/.github/utils/config_rewrite.sh new file mode 100755 index 0000000000..78119ccaee --- /dev/null +++ b/.github/utils/config_rewrite.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 input.yaml output.yaml" + exit 1 +fi + +input_file=$1 +output_file=$2 + +# Add # to the beginning of each line that is not empty or a comment +awk '!/^[[:space:]]*#/ && !/^[[:space:]]*$/ {print "#" $0; next} {print}' "$input_file" > "$output_file" +# Replace '#ors:' with 'ors:' +sed -i 's/#ors:/ors:/g' "$output_file" +# Replace '# engine:' with ' engine:' +sed -i 's/# engine:/ engine:/g' "$output_file" +# Replace '# source_file:' with ' source_file: "YOUR_FILE"' +sed -i 's/# source_file:/ source_file: "YOUR_FILE"/g' "$output_file" +# Replace '# profiles:' with ' profiles:' +sed -i 's/# profiles:/ profiles:/g' "$output_file" + +# Replace the default_profile. Ignore the value of enabled and always set to false. +awk -i inplace '/# profile_default:/{getline; if ($0 ~ /# enabled:/) {print " profile_default:"; print " enabled: false"; next}} {print}' "$output_file" + +# Replace the individual profiles. Ignore the value of enabled and always set to false. +for profile in car hgv bike-regular bike-mountain bike-road bike-electric walking hiking public-transport; do + awk -i inplace "/# $profile:/{getline; if (\$0 ~ /# enabled:/) {print \" $profile:\"; print \" enabled: false\"; next}} {print}" "$output_file" +done + +echo "Parsing complete. Result saved to $output_file" From dd18645d9e43e931de2b69ddba67b2afbf365984 Mon Sep 17 00:00:00 2001 From: Takara Baumbach Date: Tue, 30 Jan 2024 13:47:57 +0100 Subject: [PATCH 03/12] fix: logging with DeferredLogger --- .../ors/api/ORSEnvironmentPostProcessor.java | 30 +++++++++++++++---- .../listeners/ORSInitContextListener.java | 15 ---------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java b/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java index ca593abbb5..7704c73318 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java +++ b/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java @@ -1,9 +1,11 @@ package org.heigit.ors.api; +import org.apache.commons.logging.Log; import org.heigit.ors.util.StringUtility; import org.springframework.boot.SpringApplication; import org.springframework.boot.env.EnvironmentPostProcessor; import org.springframework.boot.env.YamlPropertySourceLoader; +import org.springframework.boot.logging.DeferredLogFactory; import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.PropertySource; import org.springframework.core.env.StandardEnvironment; @@ -12,44 +14,60 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; public class ORSEnvironmentPostProcessor implements EnvironmentPostProcessor { public static final String ORS_CONFIG_LOCATION_ENV = "ORS_CONFIG_LOCATION"; public static final String ORS_CONFIG_LOCATION_PROPERTY = "ors.config.location"; private final YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); + private final Log log; + + public ORSEnvironmentPostProcessor(DeferredLogFactory logFactory) { + log = logFactory.getLog(ORSEnvironmentPostProcessor.class); + } @Override public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) { // Override values from application.yml with contents of custom config yml file. List configLocations = new ArrayList<>(); - String output = ""; + log.info(""); if (!StringUtility.isNullOrEmpty(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY))) { configLocations.add(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY)); - output = output.concat("Configuration file set by program argument."); + log.info("Configuration file set by program argument."); } if (configLocations.isEmpty() && !StringUtility.isNullOrEmpty(System.getenv(ORS_CONFIG_LOCATION_ENV))) { configLocations.add(System.getenv(ORS_CONFIG_LOCATION_ENV)); - output = output.concat("Configuration file set by environment variable."); + log.info("Configuration file set by environment variable."); } if (configLocations.isEmpty()) { configLocations.add("./ors-config.yml"); configLocations.add("./ors-api/ors-config.yml"); configLocations.add("~/.config/openrouteservice/ors-config.yml"); configLocations.add("/etc/openrouteservice/ors-config.yml"); - output = output.concat("Configuration file lookup by default locations."); + log.info("Configuration file lookup by default locations."); } for (String path : configLocations) { try { List> sources = this.loader.load("yml config", new FileSystemResource(path)); if (!sources.isEmpty()) { environment.getPropertySources().addAfter(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME, sources.get(0)); - output = output.concat(" Loaded file '%s'".formatted(path)); + log.info("Loaded file '%s'".formatted(path)); break; } } catch (IllegalStateException | IOException ignored) { } } - System.setProperty(ORS_CONFIG_LOCATION_PROPERTY, output); + List> relevantENVs = System.getenv().entrySet() + .stream().filter(env -> env.getKey().startsWith("ORS_") || env.getKey().startsWith("LOGGING_") || env.getKey().startsWith("SPRINGDOC_") || env.getKey().startsWith("SPRING_") || env.getKey().startsWith("SERVER_")) + .toList(); + if (!relevantENVs.isEmpty()) { + log.info(""); + log.info("Environment variables overriding openrouteservice configuration parameters detected: "); + for (Map.Entry env : relevantENVs) { + log.info("%s=%s".formatted(env.getKey(), env.getValue())); + } + } + log.info(""); } } diff --git a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java index 30d2a0e3a9..bfe16368a2 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java +++ b/ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java @@ -20,7 +20,6 @@ */ package org.heigit.ors.api.servlet.listeners; -import com.google.common.base.Strings; import jakarta.servlet.ServletContextEvent; import jakarta.servlet.ServletContextListener; import org.apache.juli.logging.LogFactory; @@ -33,10 +32,6 @@ import org.heigit.ors.routing.RoutingProfileManagerStatus; import org.heigit.ors.util.FormatUtility; -import java.util.Map; - -import static org.heigit.ors.api.ORSEnvironmentPostProcessor.ORS_CONFIG_LOCATION_PROPERTY; - public class ORSInitContextListener implements ServletContextListener { private static final Logger LOGGER = Logger.getLogger(ORSInitContextListener.class); private final EngineProperties engineProperties; @@ -47,16 +42,6 @@ public ORSInitContextListener(EngineProperties engineProperties) { @Override public void contextInitialized(ServletContextEvent contextEvent) { - LOGGER.info(""); - if (!Strings.isNullOrEmpty(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY))) { - LOGGER.info(System.getProperty(ORS_CONFIG_LOCATION_PROPERTY)); - } - for (Map.Entry env : System.getenv().entrySet()) { - if (env.getKey().startsWith("ORS_") || env.getKey().startsWith("LOGGING_") || env.getKey().startsWith("SPRINGDOC_") || env.getKey().startsWith("SPRING_") || env.getKey().startsWith("SERVER_")) { - LOGGER.info("ENV Parameter: %s=%s".formatted(env.getKey(), env.getValue())); - } - } - LOGGER.info(""); final EngineConfig config = EngineConfig.EngineConfigBuilder.init() .setInitializationThreads(engineProperties.getInitThreads()) .setPreparationMode(engineProperties.isPreparationMode()) From 2066b7a1ca1ecd39870f6473899f53e21e6be4c0 Mon Sep 17 00:00:00 2001 From: Julian Psotta Date: Tue, 30 Jan 2024 11:30:27 +0100 Subject: [PATCH 04/12] feat(config): Add workflow for the automatic config conversion The workflow is triggered on PRs for main and releases/** branches if the PR is set for review or synchronized. --- ...config_rewrite.sh => config_conversion.sh} | 0 .../config-conversion-automation.yml | 56 +++++++++++++++++++ 2 files changed, 56 insertions(+) rename .github/utils/{config_rewrite.sh => config_conversion.sh} (100%) create mode 100644 .github/workflows/config-conversion-automation.yml diff --git a/.github/utils/config_rewrite.sh b/.github/utils/config_conversion.sh similarity index 100% rename from .github/utils/config_rewrite.sh rename to .github/utils/config_conversion.sh diff --git a/.github/workflows/config-conversion-automation.yml b/.github/workflows/config-conversion-automation.yml new file mode 100644 index 0000000000..2b8abb765d --- /dev/null +++ b/.github/workflows/config-conversion-automation.yml @@ -0,0 +1,56 @@ +name: Automatic conversion of application.yml to ors-config.yml +on: + pull_request: + branches: + - main + - releases/** + types: [ edited, opened, ready_for_review, review_requested, reopened, synchronize ] + workflow_dispatch: + + +jobs: + detect_config_change: + name: Detect and commit config changes + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + - name: Check PR state is ready_for_review + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + pr_state=$(gh pr view --json state --jq .state) + pr_is_draft=$(gh pr view --json isDraft --jq .isDraft) + echo "pr_state=$pr_state" >> $GITHUB_ENV + echo "pr_is_draft=$pr_is_draft" >> $GITHUB_ENV + else + echo "This action is only intended to be run on pull requests." + exit 1 + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Convert application.yml to ors-config.yml + run: | + .github/utils/config_conversion.sh ors-api/src/main/resources/application.yml ors-api/ors-config.yml + - name: Check with git if ors-api/ors-config.yml has changed + id: git-check + if: env.pr_state == 'OPEN' && env.pr_is_draft == 'false' + run: | + # Don't fail on exit code 1 (diff found) + set +e + git diff --exit-code --name-only ors-api/ors-config.yml + exit_value=$? + echo Found exit code $exit_value + # Write out the exit code using environment file + if [ $exit_value == 1 ]; then + echo "config_has_changed=true" >> $GITHUB_ENV + else + echo "Config hasn't changed. Skipping commit." + fi + - uses: MichaelsJP/git-auto-commit-action@v5 + if: env.pr_state == 'OPEN' && env.pr_is_draft == 'false' && env.config_has_changed == 'true' + with: + commit_message: 'chore(config): automatic conversion of application.yml to ors-config.yml' From ed4358a3361184144123dcd8948caac8340f66d7 Mon Sep 17 00:00:00 2001 From: MichaelsJP Date: Tue, 30 Jan 2024 11:57:34 +0000 Subject: [PATCH 05/12] chore(config): automatic conversion of application.yml to ors-config.yml --- ors-api/ors-config.yml | 325 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 308 insertions(+), 17 deletions(-) diff --git a/ors-api/ors-config.yml b/ors-api/ors-config.yml index b26920429d..f71d2601ac 100644 --- a/ors-api/ors-config.yml +++ b/ors-api/ors-config.yml @@ -1,32 +1,323 @@ ##### openrouteservice settings file ##### # This file contains parameters for openrouteservice. -# For a full list of possible parameters see https://giscience.github.io/openrouteservice/, below are some examples. -### uncomment the following line to turn on debug output -#logging.level.org.heigit: DEBUG +##### General server settings ##### +#server: +# port: 8082 +# error: +# whitelabel: +# enabled: false + # Keep the context-path at / else the war file run with tomcat will have the context-path of /ors/v2 as well. +# servlet: +# context-path: /ors +#spring: +# profiles: +# active: default +# mvc: +# servlet: +# path: / + +##### Settings related to springdoc ##### +#springdoc: +# swagger-ui: +# enabled: true +# path: /swagger-ui +# tryItOutEnabled: true +# filter: false +# syntaxHighlight: +# activated: true +# showExtensions: true +# api-docs: +# path: /v2/api-docs +# version: OPENAPI_3_0 +# packages-to-scan: org.heigit.ors +# pathsToMatch: /v2/** + +##### Logging settings ##### +#logging: +# level: +# root: WARN +# org.heigit: INFO + +##### openrouteservice specific settings ##### ors: - ### uncomment the following lines to change CORS settings. # cors: # allowed_origins: "*" # allowed_headers: Content-Type, X-Requested-With, accept, Origin, Access-Control-Request-Method, Access-Control-Request-Headers, Authorization # preflight_max_age: 600 - engine: - ### use the following line to enable mmap mode -# graphs_data_access: MMAP +# messages: - ### use the following lines for development setup using test OSM file for Heidelberg -# source_file: ./ors-api/src/test/files/heidelberg.osm.gz -# elevation: -# cache_path: ./ors-api/src/test/files/elevation - ### instead of the following 3 lines: - source_file: ./ors-core/data/osm_file.pbf - elevation: - cache_path: ./ors-core/data/elevation_cache + ##### ORS endpoints settings ##### +# endpoints: +# routing: +# enabled: true +# attribution: openrouteservice.org, OpenStreetMap contributors, tmc - BASt +# gpx_name: ORSRouting +# gpx_description: This is a directions instructions file as GPX, generated from openrouteservice +# gpx_base_url: https://openrouteservice.org/ +# gpx_support_mail: support@openrouteservice.org +# gpx_author: openrouteservice +# gpx_content_licence: LGPL 3.0 +# maximum_avoid_polygon_area: 200000000 +# maximum_avoid_polygon_extent: 20000 +# maximum_alternative_routes: 3 +# matrix: +# enabled: true +# attribution: openrouteservice.org, OpenStreetMap contributors +# maximum_routes: 2500 +# maximum_routes_flexible: 25 +# maximum_visited_nodes: 100000 +# maximum_search_radius: 2000 +# u_turn_costs: -1 +# isochrone: +# enabled: true +# attribution: openrouteservice.org, OpenStreetMap contributors +# maximum_locations: 2 +# maximum_intervals: 1 +# allow_compute_area: true +# maximum_range_distance_default: 50000 +# maximum_range_distance: +# - profiles: driving-car, driving-hgv +# value: 100000 +# maximum_range_time_default: 18000 +# maximum_range_time: +# - profiles: driving-car, driving-hgv +# value: 3600 +# fastisochrones: +# maximum_range_distance_default: 50000 +# maximum_range_distance: +# - profiles: driving-car, driving-hgv +# value: 500000 +# maximum_range_time_default: 18000 +# maximum_range_time: +# - profiles: driving-car, driving-hgv +# value: 10800 +# Snap: +# enabled: true +# attribution: openrouteservice.org, OpenStreetMap contributors + ##### ORS engine settings ##### + engine: +# init_threads: 1 +# preparation_mode: false + source_file: "YOUR_FILE" +# graphs_root_path: ./graphs +# graphs_data_access: RAM_STORE +# elevation: +# preprocessed: false +# data_access: MMAP +# cache_clear: false +# provider: multi +# cache_path: ./elevation_cache + profile_default: + enabled: false +# elevation: false +# elevation_smoothing: false +# encoder_flags_size: 8 +# instructions: true +# optimize: false +# traffic: false +# maximum_distance: 100000 +# maximum_distance_dynamic_weights: 100000 +# maximum_distance_avoid_areas: 100000 +# maximum_waypoints: 50 +# maximum_snapping_radius: 400 +# maximum_distance_alternative_routes: 100000 +# maximum_distance_round_trip_routes: 100000 +# maximum_speed_lower_bound: 80 +# maximum_visited_nodes: 1000000 +# location_index_resolution: 500 +# location_index_search_iterations: 4 +# force_turn_costs: false +# interpolate_bridges_and_tunnels: true +# preparation: +# min_network_size: 200 +# min_one_way_network_size: 200 +# methods: +# lm: +# enabled: true +# threads: 1 +# weightings: recommended,shortest +# landmarks: 16 +# execution: +# methods: +# lm: +# disabling_allowed: true +# active_landmarks: 8 profiles: car: - enabled: true + enabled: false +# profile: driving-car +# elevation: true +# encoder_options: +# turn_costs: true +# block_fords: false +# use_acceleration: true +# preparation: +# min_network_size: 200 +# min_one_way_network_size: 200 +# methods: +# ch: +# enabled: true +# threads: 1 +# weightings: fastest +# lm: +# enabled: false +# threads: 1 +# weightings: fastest,shortest +# landmarks: 16 +# core: +# enabled: true +# threads: 1 +# weightings: fastest,shortest +# landmarks: 64 +# lmsets: highways;allow_all +# execution: +# methods: +# ch: +# disabling_allowed: true +# lm: +# disabling_allowed: true +# active_landmarks: 6 +# core: +# disabling_allowed: true +# active_landmarks: 6 +# ext_storages: +# WayCategory: +# HeavyVehicle: +# WaySurfaceType: +# RoadAccessRestrictions: +# use_for_warnings: true hgv: - enabled: true \ No newline at end of file + enabled: false +# profile: driving-hgv +# encoder_flags_size: 8 +# encoder_options: +# turn_costs: true +# block_fords: false +# use_acceleration: true +# maximum_distance: 100000 +# elevation: true +# preparation: +# min_network_size: 200 +# min_one_way_network_size: 200 +# methods: +# ch: +# enabled: true +# threads: 1 +# weightings: recommended +# core: +# enabled: true +# threads: 1 +# weightings: recommended,shortest +# landmarks: 64 +# lmsets: highways;allow_all +# execution: +# methods: +# ch: +# disabling_allowed: true +# core: +# disabling_allowed: true +# active_landmarks: 6 +# ext_storages: +# WayCategory: +# HeavyVehicle: +# restrictions: true +# WaySurfaceType: + bike-regular: + enabled: false +# profile: cycling-regular +# encoder_options: +# consider_elevation: true +# turn_costs: true +# block_fords: false +# elevation: true +# ext_storages: +# WayCategory: +# WaySurfaceType: +# HillIndex: +# TrailDifficulty: + bike-mountain: + enabled: false +# profile: cycling-mountain +# encoder_options: +# consider_elevation: true +# turn_costs: true +# block_fords: false +# elevation: true +# ext_storages: +# WayCategory: +# WaySurfaceType: +# HillIndex: +# TrailDifficulty: + bike-road: + enabled: false +# profile: cycling-road +# encoder_options: +# consider_elevation: true +# turn_costs: true +# block_fords: false +# elevation: true +# ext_storages: +# WayCategory: +# WaySurfaceType: +# HillIndex: +# TrailDifficulty: + bike-electric: + enabled: false +# profile: cycling-electric +# encoder_options: +# consider_elevation: true +# turn_costs: true +# block_fords: false +# elevation: true +# ext_storages: +# WayCategory: +# WaySurfaceType: +# HillIndex: +# TrailDifficulty: + walking: + enabled: false +# profile: foot-walking +# encoder_options: +# block_fords: false +# elevation: true +# ext_storages: +# WayCategory: +# WaySurfaceType: +# HillIndex: +# TrailDifficulty: + hiking: + enabled: false +# profile: foot-hiking +# encoder_options: +# block_fords: false +# elevation: true +# ext_storages: +# WayCategory: +# WaySurfaceType: +# HillIndex: +# TrailDifficulty: +# wheelchair: +# enabled: false +# profile: wheelchair +# encoder_options: +# block_fords: true +# elevation: true +# maximum_snapping_radius: 50 +# ext_storages: +# WayCategory: +# WaySurfaceType: +# Wheelchair: +# KerbsOnCrossings: true +# OsmId: + public-transport: + enabled: false +# profile: public-transport +# encoder_options: +# block_fords: false +# elevation: true +# maximum_visited_nodes: 1000000 +# gtfs_file: ./src/test/files/vrn_gtfs_cut.zip From 4cf16e54e76c01849576704c0e70f550e47bb6f6 Mon Sep 17 00:00:00 2001 From: Takara Baumbach Date: Tue, 30 Jan 2024 14:30:00 +0100 Subject: [PATCH 06/12] feat: sort ENV list --- .../java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java b/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java index 7704c73318..2ac251e70c 100644 --- a/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java +++ b/ors-api/src/main/java/org/heigit/ors/api/ORSEnvironmentPostProcessor.java @@ -60,7 +60,7 @@ public void postProcessEnvironment(ConfigurableEnvironment environment, SpringAp } List> relevantENVs = System.getenv().entrySet() .stream().filter(env -> env.getKey().startsWith("ORS_") || env.getKey().startsWith("LOGGING_") || env.getKey().startsWith("SPRINGDOC_") || env.getKey().startsWith("SPRING_") || env.getKey().startsWith("SERVER_")) - .toList(); + .sorted(Map.Entry.comparingByKey()).toList(); if (!relevantENVs.isEmpty()) { log.info(""); log.info("Environment variables overriding openrouteservice configuration parameters detected: "); From 0d32a102b6bf78599b19e12373a14ca28246032a Mon Sep 17 00:00:00 2001 From: Takara Baumbach Date: Tue, 30 Jan 2024 15:15:41 +0100 Subject: [PATCH 07/12] fix: api tests, default source file, ors-config.yml location --- .github/utils/config_conversion.sh | 11 ++--- .../config-conversion-automation.yml | 2 +- ors-api/src/main/resources/application.yml | 1 + .../src/test/resources/application-test.yml | 10 +++++ ors-api/ors-config.yml => ors-config.yml | 41 ++++++++++--------- 5 files changed, 37 insertions(+), 28 deletions(-) rename ors-api/ors-config.yml => ors-config.yml (93%) diff --git a/.github/utils/config_conversion.sh b/.github/utils/config_conversion.sh index 78119ccaee..5e7610959f 100755 --- a/.github/utils/config_conversion.sh +++ b/.github/utils/config_conversion.sh @@ -14,17 +14,14 @@ awk '!/^[[:space:]]*#/ && !/^[[:space:]]*$/ {print "#" $0; next} {print}' "$inpu sed -i 's/#ors:/ors:/g' "$output_file" # Replace '# engine:' with ' engine:' sed -i 's/# engine:/ engine:/g' "$output_file" -# Replace '# source_file:' with ' source_file: "YOUR_FILE"' -sed -i 's/# source_file:/ source_file: "YOUR_FILE"/g' "$output_file" +# Replace '# source_file:' with ' source_file: ors-api/src/test/files/heidelberg.osm.gz' +sed -i 's/# source_file:/ source_file: ors-api\/src\/test\/files\/heidelberg.osm.gz/g' "$output_file" # Replace '# profiles:' with ' profiles:' sed -i 's/# profiles:/ profiles:/g' "$output_file" -# Replace the default_profile. Ignore the value of enabled and always set to false. -awk -i inplace '/# profile_default:/{getline; if ($0 ~ /# enabled:/) {print " profile_default:"; print " enabled: false"; next}} {print}' "$output_file" - # Replace the individual profiles. Ignore the value of enabled and always set to false. -for profile in car hgv bike-regular bike-mountain bike-road bike-electric walking hiking public-transport; do - awk -i inplace "/# $profile:/{getline; if (\$0 ~ /# enabled:/) {print \" $profile:\"; print \" enabled: false\"; next}} {print}" "$output_file" +for profile in car ; do + awk -i inplace "/# $profile:/{getline; if (\$0 ~ /# enabled:/) {print \" $profile:\"; print \" enabled: true\"; next}} {print}" "$output_file" done echo "Parsing complete. Result saved to $output_file" diff --git a/.github/workflows/config-conversion-automation.yml b/.github/workflows/config-conversion-automation.yml index 2b8abb765d..f50b57e04d 100644 --- a/.github/workflows/config-conversion-automation.yml +++ b/.github/workflows/config-conversion-automation.yml @@ -34,7 +34,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Convert application.yml to ors-config.yml run: | - .github/utils/config_conversion.sh ors-api/src/main/resources/application.yml ors-api/ors-config.yml + .github/utils/config_conversion.sh ors-api/src/main/resources/application.yml ors-config.yml - name: Check with git if ors-api/ors-config.yml has changed id: git-check if: env.pr_state == 'OPEN' && env.pr_is_draft == 'false' diff --git a/ors-api/src/main/resources/application.yml b/ors-api/src/main/resources/application.yml index 4203c27339..09b923f418 100644 --- a/ors-api/src/main/resources/application.yml +++ b/ors-api/src/main/resources/application.yml @@ -1,5 +1,6 @@ ##### openrouteservice settings file ##### # This file contains parameters for openrouteservice. +# For a full list of possible parameters see https://giscience.github.io/openrouteservice/ ##### General server settings ##### server: diff --git a/ors-api/src/test/resources/application-test.yml b/ors-api/src/test/resources/application-test.yml index bc9bb3605f..bc7f08cc50 100644 --- a/ors-api/src/test/resources/application-test.yml +++ b/ors-api/src/test/resources/application-test.yml @@ -29,6 +29,7 @@ ors: maxcellnodes: 5000 profiles: car: + enabled: true profile: driving-car encoder_options: turn_costs: true @@ -83,6 +84,7 @@ ors: output_log: false log_location: ors/traffic_log hgv: + enabled: true profile: driving-hgv maximum_speed_lower_bound: 75 encoder_options: @@ -138,6 +140,7 @@ ors: output_log: false log_location: ors/traffic_log bike-regular: + enabled: true profile: cycling-regular encoder_options: consider_elevation: false @@ -163,6 +166,7 @@ ors: HillIndex: TrailDifficulty: bike-mountain: + enabled: true profile: cycling-mountain maximum_snapping_radius: 10 encoder_options: @@ -175,6 +179,7 @@ ors: HillIndex: TrailDifficulty: bike-road: + enabled: true profile: cycling-road encoder_options: consider_elevation: false @@ -186,6 +191,7 @@ ors: HillIndex: TrailDifficulty: bike-electric: + enabled: true profile: cycling-electric encoder_options: consider_elevation: false @@ -197,6 +203,7 @@ ors: HillIndex: TrailDifficulty: walking: + enabled: true profile: foot-walking interpolate_bridges_and_tunnels: false encoder_options: @@ -215,6 +222,7 @@ ors: HillIndex: TrailDifficulty: hiking: + enabled: true profile: foot-hiking encoder_options: block_fords: false @@ -230,6 +238,7 @@ ors: HillIndex: TrailDifficulty: wheelchair: + enabled: true profile: wheelchair maximum_snapping_radius: 50 encoder_options: @@ -241,6 +250,7 @@ ors: WayCategory: OsmId: public-transport: + enabled: true profile: public-transport encoder_options: block_fords: false diff --git a/ors-api/ors-config.yml b/ors-config.yml similarity index 93% rename from ors-api/ors-config.yml rename to ors-config.yml index f71d2601ac..996323fc12 100644 --- a/ors-api/ors-config.yml +++ b/ors-config.yml @@ -1,5 +1,6 @@ ##### openrouteservice settings file ##### # This file contains parameters for openrouteservice. +# For a full list of possible parameters see https://giscience.github.io/openrouteservice/ ##### General server settings ##### #server: @@ -102,7 +103,7 @@ ors: engine: # init_threads: 1 # preparation_mode: false - source_file: "YOUR_FILE" + source_file: ors-api/src/test/files/heidelberg.osm.gz # graphs_root_path: ./graphs # graphs_data_access: RAM_STORE # elevation: @@ -111,8 +112,8 @@ ors: # cache_clear: false # provider: multi # cache_path: ./elevation_cache - profile_default: - enabled: false +# profile_default: +# enabled: false # elevation: false # elevation_smoothing: false # encoder_flags_size: 8 @@ -148,7 +149,7 @@ ors: # active_landmarks: 8 profiles: car: - enabled: false + enabled: true # profile: driving-car # elevation: true # encoder_options: @@ -190,8 +191,8 @@ ors: # WaySurfaceType: # RoadAccessRestrictions: # use_for_warnings: true - hgv: - enabled: false +# hgv: +# enabled: false # profile: driving-hgv # encoder_flags_size: 8 # encoder_options: @@ -226,8 +227,8 @@ ors: # HeavyVehicle: # restrictions: true # WaySurfaceType: - bike-regular: - enabled: false +# bike-regular: +# enabled: false # profile: cycling-regular # encoder_options: # consider_elevation: true @@ -239,8 +240,8 @@ ors: # WaySurfaceType: # HillIndex: # TrailDifficulty: - bike-mountain: - enabled: false +# bike-mountain: +# enabled: false # profile: cycling-mountain # encoder_options: # consider_elevation: true @@ -252,8 +253,8 @@ ors: # WaySurfaceType: # HillIndex: # TrailDifficulty: - bike-road: - enabled: false +# bike-road: +# enabled: false # profile: cycling-road # encoder_options: # consider_elevation: true @@ -265,8 +266,8 @@ ors: # WaySurfaceType: # HillIndex: # TrailDifficulty: - bike-electric: - enabled: false +# bike-electric: +# enabled: false # profile: cycling-electric # encoder_options: # consider_elevation: true @@ -278,8 +279,8 @@ ors: # WaySurfaceType: # HillIndex: # TrailDifficulty: - walking: - enabled: false +# walking: +# enabled: false # profile: foot-walking # encoder_options: # block_fords: false @@ -289,8 +290,8 @@ ors: # WaySurfaceType: # HillIndex: # TrailDifficulty: - hiking: - enabled: false +# hiking: +# enabled: false # profile: foot-hiking # encoder_options: # block_fords: false @@ -313,8 +314,8 @@ ors: # Wheelchair: # KerbsOnCrossings: true # OsmId: - public-transport: - enabled: false +# public-transport: +# enabled: false # profile: public-transport # encoder_options: # block_fords: false From e2b698fde40ccf3a1f390756200990fd376ace5c Mon Sep 17 00:00:00 2001 From: Julian Psotta Date: Tue, 30 Jan 2024 15:43:07 +0100 Subject: [PATCH 08/12] fix(config): Adjust the conversion script Some prior changes got overwritten. This introduces the old state as well as an additional testing if the config is actually in the right format to be parsed. --- .github/utils/config_conversion.sh | 38 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/utils/config_conversion.sh b/.github/utils/config_conversion.sh index 5e7610959f..8eac250326 100755 --- a/.github/utils/config_conversion.sh +++ b/.github/utils/config_conversion.sh @@ -7,21 +7,37 @@ fi input_file=$1 output_file=$2 +ors_file="ors-api/src/test/files/heidelberg.osm.gz" +########################### +### Validate input file ### +########################### +# Fail if ors: can't be found +echo "Checking for ors section" +awk '/^ors:/{print "Found ors section"}' "$input_file" || exit 1 + +# For engine and source file section, make sure they are in place +echo "Checking for engine and source_file section" +awk '/ engine:/{getline; if ($0 ~ /source_file:/) {print "Found engine and source_file section"} else { exit 1 }}' "$input_file" || exit 1 + +# For profiles section for car wit enabled +echo "Checking for profiles section" +awk '/ profiles:/{getline; if ($0 ~ / car:/) {getline; if ($0 ~ / enabled:/) {print "Found profiles section"} else { exit 1 }}}' "$input_file" || exit 1 + +########################### +### Convert input file #### +########################### # Add # to the beginning of each line that is not empty or a comment awk '!/^[[:space:]]*#/ && !/^[[:space:]]*$/ {print "#" $0; next} {print}' "$input_file" > "$output_file" + # Replace '#ors:' with 'ors:' sed -i 's/#ors:/ors:/g' "$output_file" -# Replace '# engine:' with ' engine:' -sed -i 's/# engine:/ engine:/g' "$output_file" -# Replace '# source_file:' with ' source_file: ors-api/src/test/files/heidelberg.osm.gz' -sed -i 's/# source_file:/ source_file: ors-api\/src\/test\/files\/heidelberg.osm.gz/g' "$output_file" -# Replace '# profiles:' with ' profiles:' -sed -i 's/# profiles:/ profiles:/g' "$output_file" - -# Replace the individual profiles. Ignore the value of enabled and always set to false. -for profile in car ; do - awk -i inplace "/# $profile:/{getline; if (\$0 ~ /# enabled:/) {print \" $profile:\"; print \" enabled: true\"; next}} {print}" "$output_file" -done + + +# Remove the comments for the engine and source_file section +awk -i inplace "/engine:/{getline; if (\$0 ~ /source_file:/) {print \" engine:\"; print \" source_file: '$ors_file'\"; next} else { exit1 }} {print}" "$output_file" + +# Remove the comments for the profiles section for car +awk -i inplace "/# profiles:/{getline; if (\$0 ~ /# car:/) {getline; if (\$0 ~ /# enabled:/) {print \" profiles:\"; print \" car:\"; print \" enabled: true\"; next}}}{print}" "$output_file" echo "Parsing complete. Result saved to $output_file" From 88ffd346b91c3c74b35545f98f8d9bfd825223bb Mon Sep 17 00:00:00 2001 From: Julian Psotta Date: Tue, 30 Jan 2024 15:48:00 +0100 Subject: [PATCH 09/12] fix(config): Move the source_file parameter for conversion reasons To allow a smooth conversion of the application.yml the source_file parameter needs to be next after engine:. Else, awk operations will just become unnecessary tedious. --- ors-api/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ors-api/src/main/resources/application.yml b/ors-api/src/main/resources/application.yml index 09b923f418..0cd4261ccf 100644 --- a/ors-api/src/main/resources/application.yml +++ b/ors-api/src/main/resources/application.yml @@ -101,9 +101,9 @@ ors: ##### ORS engine settings ##### engine: + source_file: init_threads: 1 preparation_mode: false - source_file: graphs_root_path: ./graphs graphs_data_access: RAM_STORE elevation: From ee0921be1d05e723527b3af9ad5c5314dc391481 Mon Sep 17 00:00:00 2001 From: Takara Baumbach Date: Tue, 30 Jan 2024 15:59:35 +0100 Subject: [PATCH 10/12] fix: clean exit on graphs_data_access misconfiguration --- .../ors/routing/RoutingProfileManager.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java index eed5e6857e..0fb96c4cbf 100644 --- a/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java +++ b/ors-engine/src/main/java/org/heigit/ors/routing/RoutingProfileManager.java @@ -14,9 +14,13 @@ package org.heigit.ors.routing; import com.graphhopper.GHResponse; -import com.graphhopper.util.*; +import com.graphhopper.util.AngleCalc; +import com.graphhopper.util.DistanceCalc; +import com.graphhopper.util.DistanceCalcEarth; +import com.graphhopper.util.PointList; import com.graphhopper.util.exceptions.ConnectionNotFoundException; import com.graphhopper.util.exceptions.MaximumNodesExceededException; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.log4j.Logger; import org.heigit.ors.config.EngineConfig; import org.heigit.ors.exceptions.*; @@ -26,13 +30,19 @@ import org.heigit.ors.isochrones.IsochroneMap; import org.heigit.ors.isochrones.IsochroneSearchParameters; import org.heigit.ors.mapmatching.MapMatchingRequest; -import org.heigit.ors.matrix.*; +import org.heigit.ors.matrix.MatrixErrorCodes; +import org.heigit.ors.matrix.MatrixRequest; +import org.heigit.ors.matrix.MatrixResult; import org.heigit.ors.routing.configuration.RouteProfileConfiguration; import org.heigit.ors.routing.configuration.RoutingManagerConfiguration; import org.heigit.ors.routing.pathprocessors.ExtraInfoProcessor; -import org.heigit.ors.util.*; +import org.heigit.ors.util.FormatUtility; +import org.heigit.ors.util.RuntimeUtility; +import org.heigit.ors.util.StringUtility; +import org.heigit.ors.util.TimeUtility; import org.locationtech.jts.geom.Coordinate; +import java.io.FileNotFoundException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -110,7 +120,10 @@ public void initialize(EngineConfig config) { if (!routingProfiles.add(rp)) LOGGER.warn("Routing profile has already been added."); } catch (ExecutionException e) { - LOGGER.error(e); + LOGGER.debug(e); + if (ExceptionUtils.indexOfThrowable(e, FileNotFoundException.class) != -1) { + throw new IllegalStateException("Output files can not be written. Make sure ors.engine.graphs_data_access is set to a writable type! "); + } throw e; } catch (InterruptedException e) { LOGGER.error(e); @@ -129,9 +142,9 @@ public void initialize(EngineConfig config) { Thread.currentThread().interrupt(); return; } catch (Exception ex) { - fail("Failed to initialize RoutingProfileManager instance. " + ex.getMessage()); + fail("Failed to initialize RoutingProfileManager instance! " + ex.getMessage()); Thread.currentThread().interrupt(); - return; + System.exit(1); } RuntimeUtility.clearMemory(LOGGER); From 3f9e53c0d10921098f501d0235f1493528e58985 Mon Sep 17 00:00:00 2001 From: Julian Psotta Date: Wed, 31 Jan 2024 11:54:20 +0100 Subject: [PATCH 11/12] fix(docker): Fix the dockerfile to use the example .yml config properly The old dockerfile was coined to use the json config. This commit just makes sure everything works again. The whole setup will probably get a revamp before release anyways --- Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bbfd79aeb2..ffecd654e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,11 +77,20 @@ COPY --chown=ors:ors --from=tomcat /tmp/tomcat ${BASE_FOLDER}/tomcat COPY --chown=ors:ors --from=build /ors-core/ors-api/target/ors.war ${BASE_FOLDER}/tomcat/webapps/ors.war COPY --chown=ors:ors --from=build /ors-core/ors-api/src/main/resources/log4j-tomcat.properties ${BASE_FOLDER}/tomcat/conf/logging.properties COPY --chown=ors:ors ./docker-entrypoint.sh ${BASE_FOLDER}/docker-entrypoint.sh -COPY --chown=ors:ors ./ors-api/ors-config.yml ${BASE_FOLDER}/tmp/ors-config.yml +COPY --chown=ors:ors ./ors-config.yml ${BASE_FOLDER}/tmp/ors-config.yml COPY --chown=ors:ors ./$OSM_FILE ${BASE_FOLDER}/tmp/osm_file.pbf USER ${UID}:${GID} +# Rewrite the ' source_file: ors-api/src/test/files/heidelberg.osm.gz' line in the config file to ' source_file: ${BASE_FOLDER}/ors-core/data/osm_file.pbf' +RUN sed -i "s| source_file: ors-api/src/test/files/heidelberg.osm.gz| source_file: ${BASE_FOLDER}/ors-core/data/osm_file.pbf|g" ${BASE_FOLDER}/tmp/ors-config.yml +# Rewrite the '# graphs_root_path: ./graphs' line in the config file to ' graphs_root_path: ${BASE_FOLDER}/ors-core/data/graphs' +RUN sed -i "s|# graphs_root_path: ./graphs| graphs_root_path: ${BASE_FOLDER}/ors-core/data/graphs|g" ${BASE_FOLDER}/tmp/ors-config.yml +# Rewrite the '# elevation:' line in the config file to ' elevation:' +RUN sed -i "s|# elevation:| elevation:|g" ${BASE_FOLDER}/tmp/ors-config.yml +# Rewrite the '# cache_path: ./elevation_cache' line in the config file to ' cache_path: ${BASE_FOLDER}/ors-core/data/elevation_cache' +RUN sed -i "s|# cache_path: ./elevation_cache| cache_path: ${BASE_FOLDER}/ors-core/data/elevation_cache|g" ${BASE_FOLDER}/tmp/ors-config.yml + ENV BUILD_GRAPHS="False" ENV ORS_CONFIG_LOCATION=ors-conf/ors-config.yml From 7024213bccc60663970307976c0b653b628078c1 Mon Sep 17 00:00:00 2001 From: Julian Psotta Date: Wed, 31 Jan 2024 12:00:07 +0100 Subject: [PATCH 12/12] chore(docker): Let the docker builds run on every PR not just to main --- .github/workflows/build-and-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index b98380f1bc..be91d72cba 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -1,7 +1,6 @@ name: Build and publish the nightly Docker image on: pull_request: - branches: [ "main" ] types: [ opened, synchronize, ready_for_review ] push: branches: [ "main" ]