Merge branch '824-modification-output' into 'master'

Refactoring de la configuration de Cars

Closes #824

See merge request 3d/cars-park/cars!734

cars/applications/application_constants.py

@@ -24,3 +24,6 @@
 
 
 APPLICATION_TAG = "applications"
+
+# common parameters
+SAVE_INTERMEDIATE_DATA = "save_intermediate_data"

cars/applications/dem_generation/dem_generation.py

@@ -50,7 +50,7 @@ def __new__(cls, orchestrator=None, conf=None):  # pylint: disable=W0613
         """
 
         dem_generation_method = cls.default_application
-        if bool(conf) is False:
+        if bool(conf) is False or "method" not in conf:
             logging.info(
                 "MntGeneration method not specified, default"
                 " {} is used".format(dem_generation_method)

cars/applications/dem_generation/dem_generation_tools.py

@@ -31,9 +31,7 @@
 # CARS imports
 from cars.core import constants as cst
 from cars.core import preprocessing, projection
-from cars.pipelines.sensor_to_dense_dsm import (
-    sensor_dense_dsm_constants as sens_cst,
-)
+from cars.pipelines.parameters import sensor_inputs_constants as sens_cst
 
 
 def triangulate_sparse_matches(

cars/applications/dem_generation/dichotomic_generation.py

@@ -33,9 +33,10 @@
 import pandas
 import rasterio
 import xarray as xr
-from json_checker import And, Checker, Or
+from json_checker import And, Checker, OptionalKey, Or
 
 import cars.orchestrator.orchestrator as ocht
+from cars.applications import application_constants
 from cars.applications.dem_generation import (
     dem_generation_constants as dem_gen_cst,
 )
@@ -129,6 +130,7 @@ def check_conf(self, conf):
             "percentile": And(Or(int, float), lambda x: x >= 0),
             "min_number_matches": And(int, lambda x: x > 0),
             "fillnodata_max_search_distance": And(int, lambda x: x > 0),
+            OptionalKey(application_constants.SAVE_INTERMEDIATE_DATA): bool,
         }
 
         # Check conf

cars/applications/dense_matches_filling/dense_matches_filling.py

@@ -32,6 +32,7 @@
 from cars.applications.application_template import ApplicationTemplate
 from cars.core import constants as cst
 from cars.core import constants_disparity as cst_disp
+from cars.core.utils import safe_makedirs
 from cars.data_structures import cars_dataset
 
 
@@ -55,7 +56,7 @@ def __new__(cls, conf=None):  # pylint: disable=W0613
         """
 
         fill_method = cls.default_application
-        if bool(conf) is False:
+        if bool(conf) is False or "method" not in conf:
             logging.info(
                 "dense_matches_filling method not specified, "
                 "default {} is used".format(fill_method)
@@ -158,7 +159,7 @@ def run(
     def __register_dataset__(
         self,
         epipolar_disparity_map,
-        save_disparity_map,
+        save_intermediate_data,
         pair_folder,
         pair_key,
         app_name=None,
@@ -168,8 +169,8 @@ def __register_dataset__(
 
         :param epipolar_disparity_map:  left disparity
         :type epipolar_disparity_map: CarsDataset
-        :param save_disparity_map: true if save disparity map
-        :type save_disparity_map: bool
+        :param save_intermediate_data: true to save disparity map
+        :type save_intermediate_data: bool
         :param pair_folder: path to folder
         :type pair_folder: str
         :param pair_key: pair id
@@ -193,8 +194,9 @@ def __register_dataset__(
             epipolar_disparity_map.attributes
         )
 
-        # Save disparity maps
-        if save_disparity_map:
+        # Save intermediate data (disparity maps)
+        if save_intermediate_data:
+            safe_makedirs(pair_folder)
             self.orchestrator.add_to_save_lists(
                 os.path.join(
                     pair_folder, "epi_disp_" + app_name + "_filled.tif"

cars/applications/dense_matches_filling/plane.py

@@ -84,7 +84,7 @@ def __init__(self, conf=None):
         self.percent_to_erode = self.used_config["percent_to_erode"]
         self.classification = self.used_config["classification"]
         # Saving files
-        self.save_disparity_map = self.used_config["save_disparity_map"]
+        self.save_intermediate_data = self.used_config["save_intermediate_data"]
 
     def check_conf(self, conf):
         """
@@ -126,13 +126,13 @@ def check_conf(self, conf):
         overloaded_conf["percent_to_erode"] = conf.get("percent_to_erode", 0.2)
         overloaded_conf["classification"] = conf.get("classification", None)
         # Saving files
-        overloaded_conf["save_disparity_map"] = conf.get(
-            "save_disparity_map", False
+        overloaded_conf["save_intermediate_data"] = conf.get(
+            "save_intermediate_data", False
         )
 
         application_schema = {
             "method": str,
-            "save_disparity_map": bool,
+            "save_intermediate_data": bool,
             "interpolation_type": Or(None, str),
             "interpolation_method": Or(None, str),
             "max_search_distance": Or(None, int),
@@ -205,6 +205,7 @@ def run(
         :rtype: CarsDataset
 
         """
+
         if holes_bbox_left is None or holes_bbox_right is None:
             raise RuntimeError("Disparity holes bbox are inconsistent.")
 
@@ -236,7 +237,7 @@ def run(
             if epipolar_disparity_map.dataset_type == "arrays":
                 new_epipolar_disparity_map = self.__register_dataset__(
                     epipolar_disparity_map,
-                    self.save_disparity_map,
+                    self.save_intermediate_data,
                     pair_folder,
                     pair_key,
                     app_name="plane",

cars/applications/dense_matches_filling/zero_padding.py

@@ -67,7 +67,7 @@ def __init__(self, conf=None):
         self.classification = self.used_config["classification"]
 
         # Saving files
-        self.save_disparity_map = self.used_config["save_disparity_map"]
+        self.save_intermediate_data = self.used_config["save_intermediate_data"]
 
     def check_conf(self, conf):
         """
@@ -92,13 +92,13 @@ def check_conf(self, conf):
 
         overloaded_conf["classification"] = conf.get("classification", None)
         # Saving files
-        overloaded_conf["save_disparity_map"] = conf.get(
-            "save_disparity_map", False
+        overloaded_conf["save_intermediate_data"] = conf.get(
+            "save_intermediate_data", False
         )
 
         application_schema = {
             "method": str,
-            "save_disparity_map": bool,
+            "save_intermediate_data": bool,
             "classification": Or(None, [str]),
         }
 
@@ -173,7 +173,7 @@ def run(
                 # Save Disparity map
                 (new_epipolar_disparity_map) = self.__register_dataset__(
                     epipolar_disparity_map,
-                    self.save_disparity_map,
+                    self.save_intermediate_data,
                     pair_folder,
                     pair_key,
                     app_name="zero_padding",

cars/applications/dense_matching/census_mccnn_sgm.py

@@ -109,7 +109,7 @@ def __init__(self, conf=None):
             "disp_range_propagation_filter_size"
         ]
         # Saving files
-        self.save_disparity_map = self.used_config["save_disparity_map"]
+        self.save_intermediate_data = self.used_config["save_intermediate_data"]
 
         # init orchestrator
         self.orchestrator = None
@@ -161,13 +161,6 @@ def check_conf(self, conf):
             "disp_max_threshold", None
         )
 
-        # Permormance map parameters
-        overloaded_conf["generate_performance_map"] = conf.get(
-            "generate_performance_map", False
-        )
-        overloaded_conf["generate_confidence_intervals"] = conf.get(
-            "generate_confidence_intervals", False
-        )
         overloaded_conf["perf_eta_max_ambiguity"] = conf.get(
             "perf_eta_max_ambiguity", 0.99
         )
@@ -190,8 +183,18 @@ def check_conf(self, conf):
         )
 
         # Saving files
-        overloaded_conf["save_disparity_map"] = conf.get(
-            "save_disparity_map", False
+        overloaded_conf["save_intermediate_data"] = conf.get(
+            "save_intermediate_data", False
+        )
+
+        # Permormance map parameters
+        overloaded_conf["generate_performance_map"] = conf.get(
+            "generate_performance_map",
+            overloaded_conf["save_intermediate_data"],
+        )
+        overloaded_conf["generate_confidence_intervals"] = conf.get(
+            "generate_confidence_intervals",
+            overloaded_conf["save_intermediate_data"],
         )
 
         # check loader
@@ -227,7 +230,7 @@ def check_conf(self, conf):
             "max_elevation_offset": Or(None, int),
             "disp_min_threshold": Or(None, int),
             "disp_max_threshold": Or(None, int),
-            "save_disparity_map": bool,
+            "save_intermediate_data": bool,
             "generate_performance_map": bool,
             "generate_confidence_intervals": bool,
             "perf_eta_max_ambiguity": float,
@@ -581,7 +584,8 @@ def generate_disparity_grids(  # noqa: C901
 
         # saving infos
         # disp grids
-        if self.save_disparity_map:
+        if self.save_intermediate_data:
+            safe_makedirs(pair_folder)
             grid_min_path = os.path.join(pair_folder, "disp_min_grid.tif")
             grid_orchestrator.add_to_save_lists(
                 grid_min_path,
@@ -896,7 +900,7 @@ def generate_disparity_grids(  # noqa: C901
         )
         grid_disp_range[0, 0] = disp_range_tile
 
-        if self.save_disparity_map:
+        if self.save_intermediate_data:
             grid_orchestrator.breakpoint()
 
         if np.any(diff < 0):
@@ -995,7 +999,6 @@ def run(
 
         if pair_folder is None:
             pair_folder = os.path.join(self.orchestrator.out_dir, "tmp")
-            safe_makedirs(pair_folder)
 
         if epipolar_images_left.dataset_type == "arrays":
             # Create CarsDataset
@@ -1012,7 +1015,7 @@ def run(
             )
 
             # Save disparity maps
-            if self.save_disparity_map:
+            if self.save_intermediate_data:
                 self.orchestrator.add_to_save_lists(
                     os.path.join(pair_folder, "epi_disp.tif"),
                     cst_disp.MAP,

cars/applications/dense_matching/dense_matching.py

@@ -49,7 +49,7 @@ def __new__(cls, conf=None):  # pylint: disable=W0613
         """
 
         matching_method = cls.default_application
-        if bool(conf) is False:
+        if bool(conf) is False or "method" not in conf:
             logging.info(
                 "Dense Matching method not specified, "
                 "default {} is used".format(matching_method)

cars/applications/dense_matching/dense_matching_tools.py

@@ -703,7 +703,7 @@ def add_performance_map(
     performance_map = w_ambi * risk_max_map * disp_to_alt_ratio
 
     # Set performance map in dataset
-    performance_map_key = cst_disp.CONFIDENCE + "_performance_map"
+    performance_map_key = cst.EPI_PERFORMANCE_MAP
 
     output_dataset[performance_map_key] = xr.DataArray(
         performance_map,

cars/applications/grid_generation/epipolar_grid_generation.py

@@ -40,9 +40,7 @@
 from cars.core.utils import safe_makedirs
 from cars.data_structures import cars_dataset
 from cars.orchestrator.cluster.log_wrapper import cars_profile
-from cars.pipelines.sensor_to_dense_dsm import (
-    sensor_dense_dsm_constants as sens_cst,
-)
+from cars.pipelines.parameters import sensor_inputs_constants as sens_cst
 
 
 class EpipolarGridGeneration(GridGeneration, short_name="epipolar"):
@@ -64,8 +62,7 @@ def __init__(self, conf=None):
         self.used_method = self.used_config["method"]
         self.epi_step = self.used_config["epi_step"]
         # Saving files
-        # TODO not implemented, future work
-        self.save_grids = self.used_config["save_grids"]
+        self.save_intermediate_data = self.used_config["save_intermediate_data"]
 
         # Init orchestrator
         self.orchestrator = None
@@ -92,12 +89,14 @@ def check_conf(self, conf):
         # Overload conf
         overloaded_conf["method"] = conf.get("method", "epipolar")
         overloaded_conf["epi_step"] = conf.get("epi_step", 30)
-        overloaded_conf["save_grids"] = conf.get("save_grids", False)
+        overloaded_conf["save_intermediate_data"] = conf.get(
+            "save_intermediate_data", False
+        )
 
         grid_generation_schema = {
             "method": str,
             "epi_step": And(int, lambda x: x > 0),
-            "save_grids": bool,
+            "save_intermediate_data": bool,
         }
 
         # Check conf
@@ -106,6 +105,16 @@ def check_conf(self, conf):
 
         return overloaded_conf
 
+    def get_save_grids(self):
+        """
+        Get whether the grid will be saved
+
+        :return: true is grid saving is activated
+        :rtype: bool
+        """
+
+        return self.save_intermediate_data
+
     @cars_profile(name="Epi Grid Generation")
     def run(
         self,
@@ -248,15 +257,17 @@ def run(
         grid_origin = grid_left.attributes["grid_origin"]
         grid_spacing = grid_left.attributes["grid_spacing"]
 
-        if self.save_grids:
+        if self.save_intermediate_data:
             left_grid_path = os.path.join(pair_folder, "left_epi_grid.tif")
             right_grid_path = os.path.join(pair_folder, "right_epi_grid.tif")
+            safe_makedirs(pair_folder)
         else:
             if pair_folder is None:
                 tmp_folder = os.path.join(self.orchestrator.out_dir, "tmp")
             else:
                 tmp_folder = os.path.join(pair_folder, "tmp")
             safe_makedirs(tmp_folder)
+            self.orchestrator.add_to_clean(tmp_folder)
             left_grid_path = os.path.join(tmp_folder, "left_epi_grid.tif")
             right_grid_path = os.path.join(tmp_folder, "right_epi_grid.tif")
 

cars/applications/grid_generation/grid_correction.py

@@ -139,12 +139,13 @@ def correct_grid(grid, grid_correction, save_grid=None, pair_folder=None):
     grid_origin = grid.attributes["grid_origin"]
     grid_spacing = grid.attributes["grid_spacing"]
 
-    # Get save folder (permanent or temporay according to save_grids parameter)
+    # Get save folder (permanent or temporay according to save_grid parameter)
     if None in (pair_folder, save_grid):
         # Set path to None
         corrected_grid_right.attributes["path"] = None
     else:
         if save_grid:
+            safe_makedirs(pair_folder)
             save_folder = os.path.join(
                 pair_folder, "corrected_right_epi_grid.tif"
             )
@@ -462,6 +463,7 @@ def estimate_right_grid_correction(
         if pair_folder is None:
             logging.error("Pair folder not provided")
         else:
+            safe_makedirs(pair_folder)
             current_out_dir = pair_folder
         matches_array_path = os.path.join(
             current_out_dir, "corrected_filtered_matches.npy"