add support for heatmaps (#1547)

Adds a heatmap line style that can be used to
render a set of lines on an axis as a heatmap
instead of a line. There can only be a single
heatmap on a particular axis. The `:heatmap`
operator can be used to select this style.

The cells for the heatmap are determined based
on the step size for the time axis and the tick
grid for the value axis. For normal lines each
line that overlaps a given cell will contribute
one to the count.

For percentile approximations that would used
with the `:percentiles` operator, they can now
be used with `:percentiles-heatmap` to map the
distribution to a heatmap. In this case the
count for a cell will be determined by the
distribution for the percentile approximation.

Additional axis specific parameters have been
added for `scale`, `u`, `l`, `palette`, and
`label` with a prefix of `heatmap_`. The scale
will control how counts are mapped to colors.

Since a heatmap can combine many lines, the
default legend test is just `Heatmap`. The
`heatmap_label` parameter on the URL can be
used to customize the label.

atlas-chart/src/main/scala/com/netflix/atlas/chart/DefaultGraphEngine.scala

@@ -94,11 +94,25 @@ class DefaultGraphEngine extends PngGraphEngine {
         config.plots.zipWithIndex.foreach {
           case (plot, i) =>
             val label = plot.ylabel.map(s => s"Axis $i: $s").getOrElse(s"Axis $i")
-            belowCanvas += Legend(config.theme.legend, plot, Some(label), showStats, entriesPerPlot)
+            belowCanvas += Legend(
+              config.theme.legend,
+              plot,
+              graph.heatmaps.get(i),
+              Some(label),
+              showStats,
+              entriesPerPlot
+            )
         }
       } else {
         config.plots.foreach { plot =>
-          belowCanvas += Legend(config.theme.legend, plot, None, showStats, entriesPerPlot)
+          belowCanvas += Legend(
+            config.theme.legend,
+            plot,
+            graph.heatmaps.get(0),
+            None,
+            showStats,
+            entriesPerPlot
+          )
         }
       }
 

atlas-chart/src/main/scala/com/netflix/atlas/chart/JsonCodec.scala

@@ -91,6 +91,7 @@ private[chart] object JsonCodec {
     config.plots.zipWithIndex.foreach {
       case (plot, i) =>
         writePlotDefMetadata(gen, plot, i)
+        writeHeatmapDef(gen, config, plot, i)
     }
     config.plots.zipWithIndex.foreach {
       case (plot, i) =>
@@ -186,6 +187,76 @@ private[chart] object JsonCodec {
     gen.writeEndObject()
   }
 
+  private def writeHeatmapDef(gen: JsonGenerator, graph: GraphDef, plot: PlotDef, id: Int): Unit = {
+    plot.heatmapData(graph).foreach { heatmap =>
+      gen.writeStartObject()
+      gen.writeStringField("type", "heatmap")
+      gen.writeNumberField("plot", id)
+      gen.writeStringField("colorScale", heatmap.settings.colorScale.name())
+      gen.writeStringField("upper", heatmap.settings.upper.toString)
+      gen.writeStringField("lower", heatmap.settings.lower.toString)
+      heatmap.settings.label.foreach { label =>
+        gen.writeStringField("label", label)
+      }
+
+      // Y-tick information, used to define the vertical buckets for heatmap counts. Included
+      // so the result can be reproduced in a dynamic rendering.
+      gen.writeArrayFieldStart("yTicks")
+      var min = heatmap.yaxis.min
+      var i = 0
+      while (i < heatmap.yTicks.size) {
+        val max = heatmap.yTicks(i).v
+        gen.writeStartObject()
+        gen.writeNumberField("min", min)
+        gen.writeNumberField("max", max)
+        gen.writeStringField("label", heatmap.yTicks(i).label)
+        gen.writeEndObject()
+        min = max
+        i += 1
+      }
+      gen.writeEndArray()
+
+      // Color ticks used to map counts to a color
+      gen.writeArrayFieldStart("colorTicks")
+      val colorTicks = heatmap.colorTicks
+      min = heatmap.minCount
+      i = 1
+      while (i < colorTicks.size) {
+        val max = colorTicks(i).v
+        gen.writeStartObject()
+        gen.writeFieldName("color")
+        writeColor(gen, heatmap.palette.colors(i - 1))
+        gen.writeNumberField("min", min)
+        gen.writeNumberField("max", max)
+        gen.writeStringField("label", colorTicks(i).label)
+        gen.writeEndObject()
+        min = max
+        i += 1
+      }
+      gen.writeEndArray()
+
+      // Output the counts associated with each cell
+      gen.writeObjectFieldStart("data")
+      gen.writeStringField("type", "heatmap")
+      gen.writeArrayFieldStart("values")
+      var t = heatmap.xaxis.start
+      while (t < heatmap.xaxis.end) {
+        gen.writeStartArray()
+        var y = 0
+        while (y < heatmap.numberOfValueBuckets) {
+          gen.writeNumber(heatmap.count(t, y))
+          y += 1
+        }
+        gen.writeEndArray()
+        t += graph.step
+      }
+      gen.writeEndArray()
+      gen.writeEndObject()
+
+      gen.writeEndObject()
+    }
+  }
+
   private def writeDataDef(
     gen: JsonGenerator,
     plot: Int,
@@ -283,6 +354,7 @@ private[chart] object JsonCodec {
   private def readGraphDef(parser: JsonParser): GraphDef = {
     var gdef: GraphDef = null
     val plots = Map.newBuilder[Int, PlotDef]
+    val heatmaps = Map.newBuilder[Int, HeatmapDef]
     val data = List.newBuilder[(Int, DataDef)]
     foreachItem(parser) {
       val node = mapper.readTree[JsonNode](parser)
@@ -295,6 +367,9 @@ private[chart] object JsonCodec {
           gdef = toGraphDef(node)
         case "plot-metadata" =>
           plots += node.get("id").asInt(0) -> toPlotDef(node)
+        case "heatmap" =>
+          val plot = node.get("plot").asInt(0)
+          heatmaps += plot -> toHeatmapDef(node)
         case "timeseries" =>
           val plot = node.get("plot").asInt(0)
           data += plot -> toLineDef(gdef, node)
@@ -310,13 +385,14 @@ private[chart] object JsonCodec {
       }
     }
 
+    val heatmapData = heatmaps.result()
     val groupedData = data.result().groupBy(_._1)
 
     val sortedPlots = plots.result().toList.sortWith(_._1 < _._1)
     val plotList = sortedPlots.map {
       case (id, plot) =>
         val plotLines = groupedData.get(id).map(_.map(_._2)).getOrElse(Nil)
-        plot.copy(data = plotLines)
+        plot.copy(data = plotLines, heatmap = heatmapData.get(id))
     }
 
     gdef.copy(plots = plotList)
@@ -410,6 +486,25 @@ private[chart] object JsonCodec {
     }
   }
 
+  private def toHeatmapDef(node: JsonNode): HeatmapDef = {
+    import scala.jdk.CollectionConverters._
+    val colors = node
+      .get("colorTicks")
+      .elements()
+      .asScala
+      .map { node =>
+        toColor(node.get("color"))
+      }
+      .toArray
+    HeatmapDef(
+      colorScale = Scale.valueOf(node.get("colorScale").asText()),
+      upper = PlotBound(node.get("upper").asText()),
+      lower = PlotBound(node.get("lower").asText()),
+      palette = Some(Palette.fromArray("heatmap", colors)),
+      label = Option(node.get("label")).map(_.asText())
+    )
+  }
+
   private def toLineDef(gdef: GraphDef, node: JsonNode): LineDef = {
     LineDef(
       data = toTimeSeries(gdef, node),