Add ability to truncate report and centralise report generation. (#579)

* add last_n_evals arg to backend export * finish slider w/ text * merge main * change to update one json+html * fix notebook path typo * fix notebook path typo for los --------- Co-authored-by: Amrit Krishnan <[email protected]>
VectorInstitute · Mar 11, 2024 · c021583 · c021583
1 parent 1ea282e
commit c021583
Show file tree

Hide file tree

Showing 11 changed files with 501 additions and 455 deletions.
diff --git a/cyclops/report/model_card/fields.py b/cyclops/report/model_card/fields.py
@@ -594,11 +594,6 @@ class MetricCard(
         description="History of the metric over time.",
     )
 
-    trend: Optional[StrictStr] = Field(
-        None,
-        description="The trend of the metric over time.",
-    )
-
     timestamps: Optional[List[StrictStr]] = Field(
         None,
         description="Timestamps for each point in the history.",

diff --git a/cyclops/report/model_card/sections.py b/cyclops/report/model_card/sections.py
@@ -30,6 +30,11 @@
 class Overview(BaseModelCardSection):
     """Overview section with aggregate metrics."""
 
+    last_n_evals: Optional[int] = Field(
+        None,
+        description="The number of evaluations to display in the model card.",
+    )
+
     metric_cards: Optional[MetricCardCollection] = Field(
         None,
         description="Comparative metrics between baseline and periodic report.",

diff --git a/cyclops/report/report.py b/cyclops/report/report.py
@@ -51,7 +51,6 @@
     get_slices,
     get_thresholds,
     get_timestamps,
-    get_trends,
     regex_replace,
     regex_search,
     str_to_snake_case,
@@ -98,7 +97,7 @@ def from_json_file(
             The path to a JSON file containing model card data.
         output_dir : str, optional
             The directory to save the report to. If not provided, the report will
-            be saved in a directory called `cyclops_reports` in the current working
+            be saved in a directory called `cyclops_report` in the current working
             directory.
 
         Returns
@@ -1054,6 +1053,7 @@ def export(
         template_path: Optional[str] = None,
         interactive: bool = True,
         save_json: bool = True,
+        last_n_evals: Optional[int] = None,
         synthetic_timestamp: Optional[str] = None,
     ) -> str:
         """Export the model card report to an HTML file.
@@ -1070,6 +1070,9 @@ def export(
             Whether to create an interactive HTML report. The default is True.
         save_json : bool, optional
             Whether to save the model card as a JSON file. The default is True.
+        last_n_evals : int, optional
+            The number of most recent evaluations to include in the report and
+            calculate trends for. If not provided, all evaluations will be included.
         synthetic_timestamp : str, optional
             A synthetic timestamp to use for the report. This is useful for
             generating back-dated reports. The default is None, which uses the
@@ -1089,10 +1092,8 @@ def export(
 
         # write to file
         if synthetic_timestamp is not None:
-            today = synthetic_timestamp
             today_now = synthetic_timestamp
         else:
-            today = dt_date.today().strftime("%Y-%m-%d")
             today_now = dt_datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
         current_report_metrics: List[List[PerformanceMetric]] = []
@@ -1102,9 +1103,7 @@ def export(
         report_paths = glob.glob(
             os.path.join(
                 self.output_dir,
-                "cyclops_reports",
-                "*",
-                "*",
+                "cyclops_report",
                 "*.json",
             ),
         )
@@ -1135,6 +1134,10 @@ def export(
                 metric_cards,
             )
 
+        if self._model_card.overview is not None:
+            last_n_evals = 0 if last_n_evals is None else last_n_evals
+            self._model_card.overview.last_n_evals = last_n_evals
+
         self._validate()
         template = self._get_jinja_template(template_path=template_path)
 
@@ -1143,7 +1146,6 @@ def export(
             "sweep_graphics": sweep_graphics,
             "get_slices": get_slices,
             "get_thresholds": get_thresholds,
-            "get_trends": get_trends,
             "get_passed": get_passed,
             "get_names": get_names,
             "get_histories": get_histories,
@@ -1154,12 +1156,9 @@ def export(
         plotlyjs = get_plotlyjs() if interactive else None
         content = template.render(model_card=self._model_card, plotlyjs=plotlyjs)
 
-        now = dt_datetime.now().strftime("%H-%M-%S")
         report_path = os.path.join(
             self.output_dir,
-            "cyclops_reports",
-            today,
-            now,
+            "cyclops_report",
             output_filename or "model_card.html",
         )
         self._write_file(report_path, content)

diff --git a/cyclops/report/templates/model_report/macros.jinja b/cyclops/report/templates/model_report/macros.jinja
@@ -172,8 +172,14 @@
 
 {% macro render_perf(name, comp)%}
 <div class="card" id={{name}}>
-<h3 style="color: black; font-weight:normal;">How is your model doing?</h3><br>
-<h3 style="color: gray;  font-weight:normal;">A quick glance of your most important metrics.</h3>
+<div class="column" style="float: left; width: 80%;">
+  <h3 style="color: black; font-weight:normal;">How is your model doing?</h3><br>
+  <h3 style="color: gray;  font-weight:normal;">A quick glance of your most important metrics.</h3>
+</div>
+<span>
+  <h3 id="slider_p_title"><span style="color: black; font-weight:normal;">Last</span> <span id="slider_p_num">{{ comp.last_n_evals }}</span> <span style="color: black; font-weight:normal;">Evaluations</span></h3>
+  <input type="range" min="1" max="{{ comp.last_n_evals+10 }}" value="{{ comp.last_n_evals }}" id="n_evals_slider_p" style="width: 100%;">
+</span>
 {% for metric_card in comp.metric_cards.collection%}
   {% if metric_card.slice == 'overall' %}
     {{ render_metric_card(metric_card, loop.index-1, "subcard_overview") }}
@@ -185,15 +191,23 @@
 
 {% macro render_perf_over_time(name, comp)%}
 <div class="card" id={{name}} style="display: block;">
-  <h3 style="color: black; font-weight:normal;">How is your model doing over time?</h3><br>
-  <h3 style="color: gray;  font-weight:normal;">See how your model is performing over several metrics and subgroups over time.</h3>
-  <div style="display: flex; align-items: center; justify-content: center; padding: 10px; margin-bottom: 20px;">
-    <h4 style="padding-right: 10px;">Multi-plot Selection:</h4>
-    <div class="radio-buttons" id="plot-selection">
-      <input type="radio" id="Plot 1" name="plot" value="Plot 1" checked>
-      <label for="Plot 1">Plot 1</label>
-      <input type="radio" id="+" name="plot" value="+">
-      <label for="+" style="padding: 2.5px; font-weight:bold; font-size: 18px;">+</label>
+  <span style="float: right; width: 10%; margin-right: 10%;">
+    <h3 id="slider_pot_title"><span style="color: black; font-weight:normal;">Last</span> <span id="slider_pot_num">{{ comp.last_n_evals }}</span> <span style="color: black; font-weight:normal;">Evaluations</span></h3>
+    <input type="range" min="1" max="{{ comp.last_n_evals+10 }}" value="{{ comp.last_n_evals }}" id="n_evals_slider_pot" style="width: 100%;">
+  </span>
+  <div style="display: flex; flex-direction: column;">
+    <div class="column" style="float: left; width: 90%;">
+      <h3 style="color: black; font-weight:normal;">How is your model doing over time?</h3><br>
+      <h3 style="color: gray;  font-weight:normal;">See how your model is performing over several metrics and subgroups over time.</h3>
+    </div>
+    <div style="display: flex; align-items: center; justify-content: center; padding: 10px; margin-bottom: 20px;">
+      <h4 style="padding-right: 10px;">Multi-plot Selection:</h4>
+      <div class="radio-buttons" id="plot-selection">
+        <input type="radio" id="Plot 1" name="plot" value="Plot 1" checked>
+        <label for="Plot 1">Plot 1</label>
+        <input type="radio" id="+" name="plot" value="+">
+        <label for="+" style="padding: 2.5px; font-weight:bold; font-size: 18px;">+</label>
+      </div>
     </div>
   </div>
   <div class="column" style="float: left;">

diff --git a/cyclops/report/templates/model_report/model_report.jinja b/cyclops/report/templates/model_report/model_report.jinja
@@ -531,8 +531,53 @@
     "rgb(23, 190, 207)"
   ];
 
+  // create global variable for max_n_evals
+  var histories = JSON.parse({{ get_histories(model_card)|safe|tojson }});
+  // get max_n_evals from histories
+  var history_data = [];
+  for (let i = 0; i < histories[0].length; i++) {
+  history_data.push(parseFloat(histories[0][i]));
+  }
+  var max_n_evals = history_data.length;
+
+// Add event listeners to radio buttons
+  for (let input of inputs_all) {
+  input.addEventListener('change', updatePlot);
+  }
+  // Add event listener to update plot when window is resized
+  window.addEventListener('resize', updatePlot);
+  for (let selection of plot_selection) {
+  selection.addEventListener('change', updatePlotSelection);
+  }
+
 
   // Initial update when the page loads
   updatePlot();
   document.addEventListener('DOMContentLoaded', setCollapseButton);
+
+  function updateLastNEvals() {
+  var n_evals_slider_p = document.getElementById("n_evals_slider_p");
+  var slider_p_num = document.getElementById("slider_p_num");
+  var n_evals_slider_pot = document.getElementById("n_evals_slider_pot");
+  var slider_pot_num = document.getElementById("slider_pot_num");
+
+  n_evals_slider_p.max = max_n_evals;
+  n_evals_slider_pot.max = max_n_evals;
+
+  if (n_evals_slider_p !== null) {
+      n_evals_slider_p.oninput = function() {
+          last_n_evals = this.value;
+          slider_p_num.innerHTML = last_n_evals;
+          generate_model_card_plot();
+      }
+  }
+  if (n_evals_slider_pot !== null) {
+      n_evals_slider_pot.oninput = function() {
+          last_n_evals = this.value;
+          slider_pot_num.innerHTML = last_n_evals;
+          updatePlot();
+      }
+      }
+  }
+  document.addEventListener('DOMContentLoaded', updateLastNEvals);
 </script>