Check and verify scratch_ensembles (#161)

.travis.yml

@@ -50,12 +50,6 @@ script:
   - git clone --depth 1 --branch $TESTDATA_REPO_BRANCH https://github.com/$TESTDATA_REPO_OWNER/webviz-subsurface-testdata.git
   - pip install git+https://github.com/equinor/libecl
   - pip install git+https://github.com/equinor/fmu-ensemble
-    # Need to find full path to testdata due to https://github.com/equinor/webviz-subsurface/issues/128
-  - pushd ./webviz-subsurface-testdata
-  - sed -i "s&\.\./&$PWD/&g" webviz_examples/full_demo.yaml
-  - sed -i "s&\.\./&$PWD/&g" webviz_examples/raw_data.yaml
-  - sed -i "s&\.\./&$PWD/&g" webviz_examples/aggregated.yaml
-  - popd
   - pytest tests --forked
   - git clone https://github.com/equinor/webviz-config.git
   - pushd ./webviz-config/docs; python build_docs.py; popd

tests/integration_tests/test_parameter_corr.py

@@ -15,13 +15,13 @@ def test_parameter_corr(dash_duo):
     app.scripts.config.serve_locally = True
     app.config.suppress_callback_exceptions = True
     CACHE.init_app(app.server)
-    container_settings = {"scratch_ensembles": {"iter-0": ""}}
+    app.webviz_settings = {"shared_settings": {"scratch_ensembles": {"iter-0": ""}}}
     ensembles = ["iter-0"]
 
     with mock.patch(get_parameters) as mock_parameters:
         mock_parameters.return_value = pd.read_csv("tests/data/parameters.csv")
 
-        p = ParameterCorrelation(app, container_settings, ensembles)
+        p = ParameterCorrelation(app, ensembles)
 
         app.layout = p.layout
         dash_duo.start_server(app)

webviz_subsurface/__init__.py

@@ -1,7 +1,28 @@
+import glob
+import pathlib
 from pkg_resources import get_distribution, DistributionNotFound
 
+import webviz_config
+
+
 try:
     __version__ = get_distribution(__name__).version
 except DistributionNotFound:
     # package is not installed
     pass
+
+
+@webviz_config.SHARED_SETTINGS_SUBSCRIPTIONS.subscribe("scratch_ensembles")
+def subscribe(scratch_ensembles, config_folder):
+    if scratch_ensembles is not None:
+        for ensemble_name, ensemble_path in scratch_ensembles.items():
+            if not pathlib.Path(ensemble_path).is_absolute():
+                scratch_ensembles[ensemble_name] = str(config_folder / ensemble_path)
+
+            if not glob.glob(scratch_ensembles[ensemble_name]):
+                raise ValueError(
+                    f"Ensemble {ensemble_name} is said to be located at {ensemble_path},"
+                    "but that wildcard path does not give any matches."
+                )
+
+    return scratch_ensembles

webviz_subsurface/containers/__init__.py

@@ -6,20 +6,20 @@
 I.e. you could have
 ```yaml
 title: Reek Webviz Demonstration
-username: some_username
-password: some_password
 
-container_settings:
+shared_settings:
   scratch_ensembles:
     iter-0: /scratch/my_ensemble/realization-*/iter-0
     iter-1: /scratch/my_ensemble/realization-*/iter-1
 
 pages:
 
- - title: Front page
-   content:
-    - container: SummaryStats
-      ensemble: iter-0
+  - title: Front page
+    content:
+      - container: ReservoirSimulationTimeSeries
+        ensembles:
+          - iter-0
+          - iter-1
 ```
 """
 

webviz_subsurface/containers/_history_match.py

@@ -16,16 +16,17 @@ class HistoryMatch(WebvizContainerABC):
 
 This container visualizes the quality of the history match.
 
-* `ensembles`: List of the ensembles in `container_settings` to visualize.
+* `ensembles`: List of the ensembles in `shared_settings` to visualize.
 * `observation_File`: Path to the observation `.yaml` file.
 * `title`: Optional title for the container.
 """
 
-    def __init__(self, container_settings, ensembles, observation_file: Path):
+    def __init__(self, app, ensembles, observation_file: Path):
 
         self.observation_file = observation_file
         self.ens_paths = tuple(
-            (ens, container_settings["scratch_ensembles"][ens]) for ens in ensembles
+            (ens, app.webviz_settings["shared_settings"]["scratch_ensembles"][ens])
+            for ens in ensembles
         )
 
         data = extract_mismatch(self.ens_paths, self.observation_file)

webviz_subsurface/containers/_inplace_volumes.py

@@ -25,6 +25,7 @@ class InplaceVolumes(WebvizContainerABC):
 defined in *container_settings* and volumetric csv files stored per realizations.
 
 #### Volumetric input
+
 The volumetric csv files must follow FMU standards.
 [Example csv file](
 https://github.com/equinor/webviz-subsurface-testdata/blob/master/aggregated_data/volumes.csv)
@@ -49,7 +50,7 @@ class InplaceVolumes(WebvizContainerABC):
 - **RECOVERABLE_GAS**: Recoverable Volume (Gas)
 
 * `csvfile`: Aggregated csvfile with 'REAL', 'ENSEMBLE' and 'SOURCE' columns
-* `ensembles`: Which ensembles in `container_settings` to visualize.
+* `ensembles`: Which ensembles in `shared_settings` to visualize.
 * `volfiles`:  Key/value pair of csv files E.g. (geogrid: geogrid--oil.csv)
 * `volfolder`: Optional local folder for csv files
 * `response`: Optional initial visualized volume response
@@ -85,7 +86,6 @@ class InplaceVolumes(WebvizContainerABC):
     def __init__(
         self,
         app,
-        container_settings,
         csvfile: Path = None,
         ensembles: list = None,
         volfiles: dict = None,
@@ -103,7 +103,8 @@ def __init__(
 
         elif ensembles and volfiles:
             self.ens_paths = tuple(
-                (ens, container_settings["scratch_ensembles"][ens]) for ens in ensembles
+                (ens, app.webviz_settings["shared_settings"]["scratch_ensembles"][ens])
+                for ens in ensembles
             )
             self.volfiles = tuple(volfiles.items())
             self.volfolder = volfolder

webviz_subsurface/containers/_inplace_volumes_onebyone.py

@@ -25,7 +25,7 @@ class InplaceVolumesOneByOne(WebvizContainerABC):
 Visualizes inplace volumetrics related to a FMU ensemble with design matrix.
 
 Input can be given either as aggregated csv files for volumes and sensitivity information,
-or as an ensemble name defined in *container_settings* and volumetric csv files
+or as an ensemble name defined in *shared_settings* and volumetric csv files
 stored per realizations.
 
 #### Volumetric input
@@ -62,7 +62,7 @@ class InplaceVolumesOneByOne(WebvizContainerABC):
 
 * `csvfile_vol`: Aggregated csvfile for volumes with 'REAL', 'ENSEMBLE' and 'SOURCE' columns
 * `csvfile_reals`: Aggregated csvfile for sensitivity information
-* `ensembles`: Which ensembles in `container_settings` to visualize.
+* `ensembles`: Which ensembles in `shared_settings` to visualize.
 * `volfiles`:  Key/value pair of csv files E.g. {geogrid: geogrid--oil.csv}
 * `volfolder`: Optional local folder for csv files
 * `response`: Optional initial visualized volume response
@@ -110,7 +110,6 @@ class InplaceVolumesOneByOne(WebvizContainerABC):
     def __init__(
         self,
         app,
-        container_settings,
         csvfile_vol: Path = None,
         csvfile_reals: Path = None,
         ensembles: list = None,
@@ -133,7 +132,8 @@ def __init__(
 
         elif ensembles and volfiles:
             self.ens_paths = tuple(
-                (ens, container_settings["scratch_ensembles"][ens]) for ens in ensembles
+                (ens, app.webviz_settings["shared_settings"]["scratch_ensembles"][ens])
+                for ens in ensembles
             )
             self.volfiles = tuple(volfiles.items())
             self.volfolder = volfolder

webviz_subsurface/containers/_parameter_correlation.py

@@ -30,14 +30,15 @@ class ParameterCorrelation(WebvizContainerABC):
 This container shows parameter correlation using a correlation matrix,
 and scatter plot for any given pair of parameters.
 
-* `ensembles`: Which ensembles in `container_settings` to visualize.
+* `ensembles`: Which ensembles in `shared_settings` to visualize.
 * `drop_constants`: Drop constant parameters
 """
 
-    def __init__(self, app, container_settings, ensembles, drop_constants: bool = True):
+    def __init__(self, app, ensembles, drop_constants: bool = True):
 
         self.ensembles = {
-            ens: container_settings["scratch_ensembles"][ens] for ens in ensembles
+            ens: app.webviz_settings["shared_settings"]["scratch_ensembles"][ens]
+            for ens in ensembles
         }
         self.drop_constants = drop_constants
         self.uid = f"{uuid4()}"

webviz_subsurface/containers/_parameter_distribution.py

@@ -23,15 +23,13 @@ class ParameterDistribution(WebvizContainerABC):
 Parameters are visualized per ensemble as a histogram, and as a boxplot showing
 the parameter ranges for each ensemble.
 Input can be given either as an aggregated csv files with parameter information
-or as an ensemble name defined in `container_settings`.
+or as an ensemble name defined in `shared_settings`.
 
 * `csvfile`: Aggregated csvfile with 'REAL', 'ENSEMBLE' and parameter columns
-* `ensembles`: Which ensembles in `container_settings` to visualize.
+* `ensembles`: Which ensembles in `shared_settings` to visualize.
 """
 
-    def __init__(
-        self, app, container_settings, csvfile: Path = None, ensembles: list = None
-    ):
+    def __init__(self, app, csvfile: Path = None, ensembles: list = None):
 
         self.csvfile = csvfile if csvfile else None
 
@@ -43,7 +41,8 @@ def __init__(
             self.parameters = read_csv(csvfile)
         elif ensembles:
             self.ensembles = tuple(
-                (ens, container_settings["scratch_ensembles"][ens]) for ens in ensembles
+                (ens, app.webviz_settings["shared_settings"]["scratch_ensembles"][ens])
+                for ens in ensembles
             )
             self.parameters = load_parameters(
                 ensemble_paths=self.ensembles, ensemble_set_name="EnsembleSet"

webviz_subsurface/containers/_reservoir_simulation_timeseries.py

@@ -25,7 +25,7 @@ class ReservoirSimulationTimeSeries(WebvizContainerABC):
 in container settings.
 
 * `csvfile`: Aggregated csvfile for unsmry with 'REAL', 'ENSEMBLE', 'DATE' and vector columns
-* `ensembles`: Which ensembles in `container_settings` to visualize.
+* `ensembles`: Which ensembles in `shared_settings` to visualize.
 * `column_keys`: List of vectors to extract. If not given, all vectors
                  from the simulations will be extracted. Wild card asterisk *
                  can be used.
@@ -46,7 +46,6 @@ class ReservoirSimulationTimeSeries(WebvizContainerABC):
     def __init__(
         self,
         app,
-        container_settings,
         csvfile: Path = None,
         ensembles: list = None,
         obsfile: Path = None,
@@ -72,7 +71,12 @@ def __init__(
             self.smry = read_csv(csvfile)
         elif ensembles:
             self.ens_paths = tuple(
-                (ensemble, container_settings["scratch_ensembles"][ensemble])
+                (
+                    ensemble,
+                    app.webviz_settings["shared_settings"]["scratch_ensembles"][
+                        ensemble
+                    ],
+                )
                 for ensemble in ensembles
             )
             self.smry = load_smry(

webviz_subsurface/containers/_reservoir_simulation_timeseries_onebyone.py

@@ -30,7 +30,7 @@ class ReservoirSimulationTimeSeriesOneByOne(WebvizContainerABC):
 run with that sensitivity.
 
 Input can be given either as aggregated csv files for summary vectors and sensitivity
-information, or as an ensemble name defined in 'container_settings'.
+information, or as an ensemble name defined in 'shared_settings'.
 
 #### Time series input
 The time series input can either extracted automatically from the ensemble or
@@ -47,7 +47,7 @@ class ReservoirSimulationTimeSeriesOneByOne(WebvizContainerABC):
 
 * `csvfile_smry`: Aggregated csvfile for volumes with 'REAL', 'ENSEMBLE', 'DATE' and vector columns
 * `csvfile_reals`: Aggregated csvfile for sensitivity information
-* `ensembles`: Which ensembles in `container_settings` to visualize.
+* `ensembles`: Which ensembles in `shared_settings` to visualize.
 * `column_keys`: List of vectors to extract. If not given, all vectors
                  from the simulations will be extracted. Wild card asterisk *
                  can be used.
@@ -80,7 +80,6 @@ class ReservoirSimulationTimeSeriesOneByOne(WebvizContainerABC):
     def __init__(
         self,
         app,
-        container_settings,
         csvfile_smry: Path = None,
         csvfile_reals: Path = None,
         ensembles: list = None,
@@ -106,7 +105,12 @@ def __init__(
 
         elif ensembles:
             self.ens_paths = tuple(
-                (ensemble, container_settings["scratch_ensembles"][ensemble])
+                (
+                    ensemble,
+                    app.webviz_settings["shared_settings"]["scratch_ensembles"][
+                        ensemble
+                    ],
+                )
                 for ensemble in ensembles
             )
             # Extract realizations and sensitivity information

webviz_subsurface/containers/_subsurface_map.py

@@ -21,7 +21,7 @@ class SubsurfaceMap(WebvizContainerABC):
 a FMU ensemble.
 
 * `jsonfile`: jsonfile with data.
-* `ensemble`: Which ensemble in `container_settings` to visualize.
+* `ensemble`: Which ensemble in `shared_settings` to visualize.
 * `map_value`: Which property to show in the map (e.g. `PERMX`).
 * `flow_value`: Which property to use for the streamlines animation
   (e.g. `FLOWAT`).
@@ -31,7 +31,7 @@ class SubsurfaceMap(WebvizContainerABC):
 
     def __init__(
         self,
-        container_settings,
+        app,
         jsonfile: Path = None,
         ensemble: str = None,
         map_value: str = None,
@@ -53,7 +53,9 @@ def __init__(
             self.map_value = map_value
             self.flow_value = flow_value
             self.time_step = time_step
-            self.ensemble_path = container_settings["scratch_ensembles"][ensemble]
+            self.ensemble_path = app.webviz_settings["shared_settings"][
+                "scratch_ensembles"
+            ][ensemble]
             self.map_data = get_map_data(
                 self.ensemble_path, self.map_value, self.flow_value, self.time_step
             )