test: Add graph folder initialization routine

docs/documentation/Graph-Management.md

@@ -0,0 +1,71 @@
+# Graph Management 
+
+Depending on the value in the configuration property `source_file`, Openrouteservice can be started in two different modes:
+
+* **Graph generation mode**, when a OSM file is configured, e.g.: `source_file=/opt/openrouteservice/files/planet.pbf`. This is the classic mode, where Openrouteservice generates a graph locally for each configured and activated profile with the configured OSM data as input. 
+* **Graph management mode**, when the URL of a Openrouteservice Graph Repository is configured, e.g.: `source_file=https://nexus.openrouteservice.org/service/rest/ors-graphs/andorra`. Graph management mode is a new feature introduced in version TODO-Version. In this mode, openrouteservice no longer generates graphs locally, but downloads precalculated graphs from a special Openrouteservice graph repository and checks for updates on a regular basis.
+
+## Graph folder structure
+
+Before version TODO-Version, all graph files were located in a directory named like its routing profile, e.g. "car":
+
+```commandline
+car
+├── edges
+├── geometry
+├── ...
+└── turn_costs
+```
+
+Since version TODO-Version, the structure was extended by one directory level. 
+For each profile, a hash value is generated and the graph files for this graph configuration is located in a subdirectory named with this hash value.
+
+```commandline
+car
+└── 8250b2498273af0908682e2d660f279a                    # Active graph of the active profile. The graph will be backed up when a new one is activated.
+    ├── geometry
+    ├── ...
+    └── turn_costs
+```
+
+The hash value is based on all configuration properties that are relevant for a graph.
+When openrouteservice is started with a changed routing profile configuration (even if only one relevant parameter was changed), 
+openrouteservice will generate a new graph in a new hash-directory:
+
+```commandline
+car
+├── 8250b2498273af0908682e2d660f279a                    # Graph of the initial profile configuration
+│   ├── geometry
+│   ├── ...
+│   └── turn_costs
+└── d4d43791efac62e0bf75322ef17ed92c                    # Graph of the changed profile configuration
+    ├── geometry
+    ├── ...
+    └── turn_costs
+```
+
+
+## Files in the profile folders
+
+```commandline
+car
+├── 8250b2498273af0908682e2d660f279a                    # Active graph of the active profile. The graph will be backed up when a new one is activated.
+│   ├── 8250b2498273af0908682e2d660f279a.json           # Metadata about the graph and the profile configuration that was used for generating it 
+│   ├── ...                                             # Graphhopper graph files and extensions
+├── 8250b2498273af0908682e2d660f279a_new_incomplete     # Extraction folder for a downloaded graph. When extraction is done, it "_incomplete" will be removed from the folder name.  
+│   ├── ...                                             
+├── 8250b2498273af0908682e2d660f279a_new                # Extracted downloaded graph from the graph repository. This graph will be activated on the next system start.  
+│   ├── 8250b2498273af0908682e2d660f279a.json           # Metadata
+│   ├── ...                                             # Graphhopper graph files and extensions
+├── 8250b2498273af0908682e2d660f279a_2023-10-18_152345  # Backup of a previous graph. The timestamp is the date when the graph was replaced by a new one. 
+│   ├── 8250b2498273af0908682e2d660f279a.json           # Metadata
+│   ├── ...                                             # Graphhopper graph files and extensions
+├── 8250b2498273af0908682e2d660f279a.json               # Downloaded metadata of the newest graph for this profile in the repository 
+├── 8250b2498273af0908682e2d660f279a.json.incomplete    #      first the file is downloaded with extension `incomplete`, which is removed when the download (of this file) is finished
+├── 8250b2498273af0908682e2d660f279a.ghz                # Downloaded graph of this profile 
+└── 8250b2498273af0908682e2d660f279a.ghz.incomplete     #      first the file is downloaded with extension `incomplete`, which is removed when the download (of this file) is finished
+```
+
+
+```commandline
+```

ors-api/ors-config.yml

@@ -17,11 +17,20 @@ ors:
   engine:
     ### use the following line for development setup using test OSM file for Heidelberg
     #source_file: ./src/test/files/heidelberg.osm.gz
-    source_file: ./ors-core/data/osm_file.pbf
+    # we have currently no test repo! this was the former URL:
+    source_file: https://test-repo.openrouteservice.org/service/rest/ors-graphs-fastisochrones/
+    graphs_extent: andorra
     graphs_root_path: ./ors-core/data/graphs
+    max_number_of_graph_backups: 0
     elevation:
       cache_path: ./ors-core/data/elevation_cache
-
+    graphservice:
+      schedule:
+        # HINT default is never (there is no Feb 31)
+        download:
+          cron: 0 0 * 31 2 *
+        activate:
+          cron: 0 0 * 31 2 *
     profiles:
       car:
         profile: driving-car

ors-api/src/main/java/org/heigit/ors/api/Application.java

@@ -1,38 +1,59 @@
 package org.heigit.ors.api;
 
 import jakarta.servlet.ServletContextListener;
+import org.heigit.ors.api.services.GraphService;
 import org.heigit.ors.api.servlet.listeners.ORSInitContextListener;
 import org.heigit.ors.routing.RoutingProfileManagerStatus;
 import org.heigit.ors.util.StringUtility;
+import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.web.servlet.ServletComponentScan;
 import org.springframework.boot.web.servlet.ServletListenerRegistrationBean;
 import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 @ServletComponentScan("org.heigit.ors.api.servlet.listeners")
 @SpringBootApplication
+@EnableScheduling
+@EnableAsync
 public class Application extends SpringBootServletInitializer {
 
     static {
         System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
     }
 
+    private static ConfigurableApplicationContext context;
+
     public static void main(String[] args) {
         if (args.length > 0 && !StringUtility.isNullOrEmpty(args[0])) {
             System.setProperty(ORSEnvironmentPostProcessor.ORS_CONFIG_LOCATION_PROPERTY, args[0]);
         }
-        SpringApplication.run(Application.class, args);
+        context = SpringApplication.run(Application.class, args);
         if (RoutingProfileManagerStatus.hasFailed()) {
             System.exit(1);
         }
     }
 
+    public static void restart() {
+        ApplicationArguments args = context.getBean(ApplicationArguments.class);
+
+        Thread thread = new Thread(() -> {
+            context.close();
+            context = SpringApplication.run(Application.class, args.getSourceArgs());
+        });
+
+        thread.setDaemon(false);
+        thread.start();
+    }
+
     @Bean("ORSInitContextListenerBean")
-    public ServletListenerRegistrationBean<ServletContextListener> createORSInitContextListenerBean(EngineProperties engineProperties) {
+    public ServletListenerRegistrationBean<ServletContextListener> createORSInitContextListenerBean(EngineProperties engineProperties, GraphService graphService) {
         ServletListenerRegistrationBean<ServletContextListener> bean = new ServletListenerRegistrationBean<>();
-        bean.setListener(new ORSInitContextListener(engineProperties));
+        bean.setListener(new ORSInitContextListener(engineProperties, graphService));
         return bean;
     }
 }

ors-api/src/main/java/org/heigit/ors/api/EngineProperties.java

@@ -20,7 +20,9 @@ public class EngineProperties {
     private int initThreads;
     private boolean preparationMode;
     private String sourceFile;
+    private String graphsExtent;
     private String graphsRootPath;
+    private int maxNumberOfGraphBackups;
     private ElevationProperties elevation;
     private ProfileProperties profileDefault;
     private Map<String, ProfileProperties> profiles;
@@ -49,6 +51,14 @@ public void setSourceFile(String sourceFile) {
         this.sourceFile = sourceFile;
     }
 
+    public String getGraphsExtent() {
+        return graphsExtent;
+    }
+
+    public void setGraphsExtent(String graphsExtent) {
+        this.graphsExtent = graphsExtent;
+    }
+
     public String getGraphsRootPath() {
         return graphsRootPath;
     }
@@ -57,6 +67,14 @@ public void setGraphsRootPath(String graphsRootPath) {
         this.graphsRootPath = graphsRootPath;
     }
 
+    public int getMaxNumberOfGraphBackups() {
+        return maxNumberOfGraphBackups;
+    }
+
+    public void setMaxNumberOfGraphBackups(int maxNumberOfGraphBackups) {
+        this.maxNumberOfGraphBackups = maxNumberOfGraphBackups;
+    }
+
     public ElevationProperties getElevation() {
         return elevation;
     }
@@ -99,6 +117,7 @@ else if (!FileUtility.isAbsolutePath(graphPath))
                         graphPath = Paths.get(rootGraphsPath, graphPath).toString();
                 }
                 convertedProfile.setGraphPath(graphPath);
+                convertedProfile.setGraphsExtent(graphsExtent);
                 convertedProfile.setEncoderOptions(profile.getEncoderOptionsString());
                 convertedProfile.setOptimize(profile.optimize != null ? profile.optimize : profileDefault.getOptimize());
                 convertedProfile.setEncoderFlagsSize(profile.encoderFlagsSize != null ? profile.encoderFlagsSize : profileDefault.getEncoderFlagsSize());

ors-api/src/main/java/org/heigit/ors/api/services/GraphService.java

@@ -0,0 +1,77 @@
+package org.heigit.ors.api.services;
+
+import org.apache.log4j.Logger;
+import org.heigit.ors.api.Application;
+import org.heigit.ors.api.util.AppConfigMigration;
+import org.heigit.ors.routing.graphhopper.extensions.manage.ORSGraphManager;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Service
+public class GraphService {
+    private static final Logger LOGGER = Logger.getLogger(GraphService.class.getName());
+
+    public List<ORSGraphManager> graphManagers = new ArrayList<>();
+
+    public void addGraphhopperLocation(ORSGraphManager orsGraphManager) {
+        graphManagers.add(orsGraphManager);
+    }
+
+    @Async
+    @Scheduled(cron = "${ors.engine.graphservice.schedule.download.cron:0 0 0 31 2 *}")//Default is "never"
+    public void checkForUpdatesInRepo() {
+
+        LOGGER.debug("Scheduled check for updates in graph repository...");
+
+        for (ORSGraphManager orsGraphManager : graphManagers) {
+            if (orsGraphManager.isActive()) {
+                LOGGER.info("Scheduled check for updates in graph repository: [%s] Download or extraction in progress".formatted(orsGraphManager.getProfileWithHash()));
+            } else if (orsGraphManager.hasDownloadedExtractedGraph()) {
+                LOGGER.info("Scheduled check for updates in graph repository: [%s] A newer graph was already downloaded and extracted".formatted(orsGraphManager.getProfileWithHash()));
+            } else {
+                LOGGER.info("Scheduled check for updates in graph repository: [%s] Checking for update.".formatted(orsGraphManager.getProfileWithHash()));
+                orsGraphManager.downloadAndExtractLatestGraphIfNecessary();
+            }
+        }
+
+        LOGGER.debug("Scheduled check for updates in graph repository done");
+    }
+
+    @Async
+    @Scheduled(cron = "${ors.engine.graphservice.schedule.activate.cron:0 0 0 31 2 *}")//Default is "never"
+    public void checkForDownloadedGraphsToActivate() {
+
+        LOGGER.debug("Scheduled check for downloaded graphs...");
+
+        boolean restartNeeded = false;
+        boolean restartAllowed = true;
+
+        for (ORSGraphManager orsGraphManager : graphManagers) {
+            if (orsGraphManager.isActive() || orsGraphManager.hasGraphDownloadFile()) {
+                LOGGER.info("Scheduled check for downloaded graphs: [%s] Download or extraction in progress".formatted(orsGraphManager.getProfileWithHash()));
+                restartAllowed = false;
+            }
+            if (orsGraphManager.hasDownloadedExtractedGraph()) {
+                LOGGER.info("Scheduled check for downloaded graphs: [%s] Downloaded extracted graph available".formatted(orsGraphManager.getProfileWithHash()));
+                restartNeeded = true;
+            }
+        }
+
+        if (restartNeeded && restartAllowed) {
+            LOGGER.info("Scheduled check for downloaded graphs done -> restarting openrouteservice");
+            restartApplication();
+        } else {
+            LOGGER.info("Scheduled check for downloaded graphs done -> restarting openrouteservice is %s".formatted(
+                    !restartNeeded ? "not needed" : restartAllowed ? "needed and allowed" : "needed but not allowed (one or more graph managers are active)")
+            );
+        }
+    }
+
+    private void restartApplication() {
+        Application.restart();
+    }
+}

ors-api/src/main/java/org/heigit/ors/api/servlet/listeners/ORSInitContextListener.java

@@ -25,22 +25,31 @@
 import org.apache.juli.logging.LogFactory;
 import org.apache.log4j.Logger;
 import org.heigit.ors.api.EngineProperties;
+import org.heigit.ors.api.services.GraphService;
 import org.heigit.ors.config.EngineConfig;
 import org.heigit.ors.isochrones.statistics.StatisticsProviderFactory;
+import org.heigit.ors.routing.RoutingProfile;
 import org.heigit.ors.routing.RoutingProfileManager;
 import org.heigit.ors.routing.RoutingProfileManagerStatus;
+import org.heigit.ors.routing.graphhopper.extensions.ORSGraphHopper;
+import org.heigit.ors.routing.graphhopper.extensions.manage.ORSGraphManager;
 import org.heigit.ors.util.FormatUtility;
 import org.heigit.ors.util.StringUtility;
 
+import java.net.MalformedURLException;
+import java.net.URL;
+
 import static org.heigit.ors.api.ORSEnvironmentPostProcessor.ORS_CONFIG_LOCATION_ENV;
 import static org.heigit.ors.api.ORSEnvironmentPostProcessor.ORS_CONFIG_LOCATION_PROPERTY;
 
 public class ORSInitContextListener implements ServletContextListener {
     private static final Logger LOGGER = Logger.getLogger(ORSInitContextListener.class);
     private final EngineProperties engineProperties;
+    private final GraphService graphService;
 
-    public ORSInitContextListener(EngineProperties engineProperties) {
+    public ORSInitContextListener(EngineProperties engineProperties, GraphService graphService) {
         this.engineProperties = engineProperties;
+        this.graphService = graphService;
     }
 
     @Override
@@ -53,18 +62,33 @@ public void contextInitialized(ServletContextEvent contextEvent) {
                 LOGGER.debug("Configuration loaded by ARG, location: " + System.getProperty(ORS_CONFIG_LOCATION_PROPERTY));
             }
         }
+        SourceFileElements sourceFileElements = extractSourceFileElements(engineProperties.getSourceFile());
         final EngineConfig config = EngineConfig.EngineConfigBuilder.init()
-            .setInitializationThreads(engineProperties.getInitThreads())
-            .setPreparationMode(engineProperties.isPreparationMode())
-            .setElevationPreprocessed(engineProperties.getElevation().isPreprocessed())
-            .setSourceFile(engineProperties.getSourceFile())
-            .setGraphsRootPath(engineProperties.getGraphsRootPath())
-            .setProfiles(engineProperties.getConvertedProfiles())
-            .buildWithAppConfigOverride();
+                .setInitializationThreads(engineProperties.getInitThreads())
+                .setPreparationMode(engineProperties.isPreparationMode())
+                .setElevationPreprocessed(engineProperties.getElevation().isPreprocessed())
+                .setGraphsRootPath(engineProperties.getGraphsRootPath())
+                .setMaxNumberOfGraphBackups(engineProperties.getMaxNumberOfGraphBackups())
+                .setSourceFile(sourceFileElements.localOsmFilePath)
+                .setGraphsRepoUrl(sourceFileElements.repoBaseUrlString)
+                .setGraphsRepoName(sourceFileElements.repoName)
+                .setGraphsExtent(engineProperties.getGraphsExtent())
+                .setProfiles(engineProperties.getConvertedProfiles())
+                .buildWithAppConfigOverride();
         Runnable runnable = () -> {
             try {
                 LOGGER.info("Initializing ORS...");
-                new RoutingProfileManager(config);
+                RoutingProfileManager routingProfileManager = new RoutingProfileManager(config);
+                if (routingProfileManager.getProfiles() != null) {
+                    for (RoutingProfile profile : routingProfileManager.getProfiles().getUniqueProfiles()) {
+                        ORSGraphHopper orsGraphHopper = profile.getGraphhopper();
+                        ORSGraphManager orsGraphManager = orsGraphHopper.getOrsGraphManager();
+                        if (orsGraphManager != null) {
+                            LOGGER.debug("Adding orsGraphManager for profile %s to GraphService".formatted(profile.getConfiguration().getName()));
+                            graphService.addGraphhopperLocation(orsGraphManager);
+                        }
+                    }
+                }
             } catch (Exception e) {
                 LOGGER.warn("Unable to initialize ORS due to an unexpected exeception: " + e);
             }
@@ -74,6 +98,28 @@ public void contextInitialized(ServletContextEvent contextEvent) {
         thread.start();
     }
 
+    record SourceFileElements(String repoBaseUrlString, String repoName, String localOsmFilePath) {
+    }
+
+    SourceFileElements extractSourceFileElements(String sourceFilePropertyValue) {
+        String repoBaseUrlString = null;
+        String repoName = null;
+        String localOsmFilePath = "";
+        try {
+            new URL(sourceFilePropertyValue);
+            LOGGER.debug("Configuration property 'source_file' contains a URL, using value as URL for a graphs repository");
+            sourceFilePropertyValue = sourceFilePropertyValue.trim().replaceAll("/$", "");
+            String[] urlElements = sourceFilePropertyValue.split("/");
+
+            repoName = urlElements[urlElements.length - 1];
+            repoBaseUrlString = sourceFilePropertyValue.replaceAll("/%s$".formatted(repoName), "");
+        } catch (MalformedURLException e) {
+            LOGGER.debug("Configuration property 'source_file' does not contain a URL, using value as local osm file path");
+            localOsmFilePath = sourceFilePropertyValue;
+        }
+        return new SourceFileElements(repoBaseUrlString, repoName, localOsmFilePath);
+    }
+
     @Override
     public void contextDestroyed(ServletContextEvent contextEvent) {
         try {

ors-api/src/main/resources/application.yml

@@ -102,6 +102,7 @@ ors:
     init_threads: 1
     preparation_mode: false
     source_file:
+    max_number_of_graph_backups: 0
     graphs_root_path: ./graphs
     elevation:
       preprocessed: false