Add warning when everest-models file outputs do not match everest obj…

…ectives

src/everest/config/everest_config.py

@@ -34,7 +34,10 @@
     validate_forward_model_configs,
 )
 from everest.jobs import script_names
-from everest.util.forward_models import collect_forward_models
+from everest.util.forward_models import (
+    check_forward_model_objective,
+    collect_forward_models,
+)
 
 from ..config_file_loader import yaml_file_to_substituted_config_dict
 from ..strings import (
@@ -434,6 +437,14 @@ def validate_maintained_forward_models(self):
                 validate_forward_model_configs(self.forward_model, self.install_jobs)
         return self
 
+    @model_validator(mode="after")
+    def validate_forward_model_write_objectives(self):
+        if not self.objective_functions:
+            return self
+        objectives = {objective.name for objective in self.objective_functions}
+        check_forward_model_objective(self.forward_model, objectives)
+        return self
+
     @model_validator(mode="after")
     # pylint: disable=E0213
     def validate_input_constraints_weight_definition(self):

src/everest/plugins/hook_specs.py

@@ -1,4 +1,4 @@
-from typing import Sequence, Type, TypeVar
+from typing import List, Sequence, Set, Type, TypeVar
 
 from everest.plugins import hookspec
 
@@ -112,3 +112,11 @@ def add_log_handle_to_root():
 @hookspec
 def get_forward_model_documentations():
     """ """
+
+
+@hookspec(firstresult=True)
+def custom_forward_model_outputs(forward_model_steps: List[str]) -> Set[str]:
+    """
+    Check if the given forward model steps will output to a file maching the
+    defined everest objective
+    """

src/everest/util/forward_models.py

@@ -1,8 +1,9 @@
 from itertools import chain
-from typing import List, Type, TypeVar
+from typing import List, Set, Type, TypeVar
 
 from pydantic import BaseModel, ValidationError
 
+from ert.config import ConfigWarning
 from everest.plugins.everest_plugin_manager import EverestPluginManager
 
 pm = EverestPluginManager()
@@ -24,6 +25,22 @@ def lint_forward_model_job(job: str, args) -> List[str]:
     return pm.hook.lint_forward_model(job=job, args=args)
 
 
+def check_forward_model_objective(fm_stes: List[str], objectives: Set[str]) -> None:
+    fm_outputs = pm.hook.custom_forward_model_outputs(
+        forward_model_steps=fm_stes,
+        objectives=objectives,
+    )
+    if fm_outputs is None:
+        return
+    unaccounted_objectives = objectives.difference(fm_outputs)
+    if unaccounted_objectives:
+        add_s = "s" if len(unaccounted_objectives) > 1 else ""
+        ConfigWarning.warn(
+            f"Warning: Forward model might not write the required output file{add_s}"
+            f" for {sorted(unaccounted_objectives)}"
+        )
+
+
 def parse_forward_model_file(path: str, schema: Type[T], message: str) -> T:
     try:
         res = pm.hook.parse_forward_model_schema(path=path, schema=schema)

tests/everest/test_config_validation.py

@@ -1,16 +1,20 @@
 import os
 import pathlib
 import re
+import warnings
 from pathlib import Path
 from typing import Any, Dict, List, Union
 
 import pytest
 from pydantic import ValidationError
 
+from ert.config import ConfigWarning
 from everest.config import EverestConfig, ModelConfig
 from everest.config.control_variable_config import ControlVariableConfig
 from everest.config.sampler_config import SamplerConfig
 
+from tests.everest.utils import skipif_no_everest_models
+
 
 def has_error(error: Union[ValidationError, List[dict]], match: str):
     messages = (
@@ -944,3 +948,42 @@ def test_that_non_existing_workflow_jobs_cause_error():
                 ]
             },
         )
+
+
+@skipif_no_everest_models
+@pytest.mark.everest_models_test
+@pytest.mark.parametrize(
+    ["objective", "warning_msg"],
+    [
+        (
+            ["npv", "rf"],
+            "Warning: Forward model might not write the required output file for \\['npv'\\]",
+        ),
+        (
+            ["npv", "npv2"],
+            "Warning: Forward model might not write the required output files for \\['npv', 'npv2'\\]",
+        ),
+        (["rf"], None),
+    ],
+)
+def test_warning_forward_model_write_objectives(objective, warning_msg):
+    fm_steps = [
+        "well_constraints  -i files/well_readydate.json -c files/wc_config.yml -rc well_rate.json -o wc_wells.json",
+        "add_templates     -i wc_wells.json -c files/at_config.yml -o at_wells.json",
+        "schmerge          -s eclipse/include/schedule/schedule.tmpl -i at_wells.json -o eclipse/include/schedule/schedule.sch",
+        "eclipse100 TEST.DATA --version 2020.2",
+        "rf -s TEST -o rf",
+    ]
+    if warning_msg is not None:
+        with pytest.warns(ConfigWarning, match=warning_msg):
+            EverestConfig.with_defaults(
+                objective_functions=[{"name": o} for o in objective],
+                forward_model=fm_steps,
+            )
+    else:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", category=ConfigWarning)
+            EverestConfig.with_defaults(
+                objective_functions=[{"name": o} for o in objective],
+                forward_model=fm_steps,
+            )