api: introduce latency observation for http endpoint

This patch enables monitoring of HTTP endpoint latency.
The `metrics` object is added to the endpoint configuration to facilitate this.
When `metrics.enabled` is set to `true`, the endpoint handler is wrapped
with `metrics.http_middleware` to capture latency data.

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- Introduce latency observation for http endpoint (#17).
 
 ### Fixed
 

README.md

@@ -89,6 +89,18 @@ incoming requests. An individual endpoint can be described as:
   format: format_to_export
 ```
 
+Optionally, you can enable
+[http metrics](https://www.tarantool.io/en/doc/latest/reference/reference_lua/metrics/#collecting-http-metrics)
+for each endpoint. For this you should
+set `metrics.enabled` to `true`:
+
+```yaml
+- path: /path/to/export/on/the/server
+  format: format_to_export
+  metrics:
+    enabled: true
+```
+
 For now only `json` and `prometheus` formats are supported.
 
 Let's put it all together now:
@@ -107,6 +119,8 @@ roles_cfg:
       endpoints:
       - path: /metrics/
         format: json
+        metrics:
+          enabled: true
 ```
 
 With this configuration, metrics can be obtained on this machine with the

roles/metrics-export.lua

@@ -125,6 +125,20 @@ local http_handlers = {
 -- It is used as an error string with the predefined order.
 local http_supported_formats_str = "json, prometheus"
 
+local function validate_endpoint_metrics(metrics)
+    if type(metrics) ~= 'table' then
+        error("http endpoint 'metrics' must be a table, got " .. type(metrics), 5)
+    end
+
+    if is_array(metrics) then
+        error("http endpoint 'metrics' must be a map, not an array", 5)
+    end
+
+    if metrics.enabled ~= nil and type(metrics.enabled) ~= 'boolean' then
+        error("http endpoint metrics 'enabled' must be a boolean, got " .. type(metrics.enabled), 5)
+    end
+end
+
 local function validate_http_endpoint(endpoint)
     if type(endpoint) ~= "table" then
         error("http endpoint must be a table, got " .. type(endpoint), 4)
@@ -150,6 +164,10 @@ local function validate_http_endpoint(endpoint)
         error("http endpoint 'format' must be one of: " ..
               http_supported_formats_str .. ", got " .. endpoint.format, 4)
     end
+
+    if endpoint.metrics ~= nil then
+        validate_endpoint_metrics(endpoint.metrics)
+    end
 end
 
 local function validate_http_node(node)
@@ -217,6 +235,25 @@ local function validate_http(conf)
     end
 end
 
+local function wrap_handler(handler, metrics)
+    if metrics ~= nil and metrics.enabled == true then
+        local http_middleware = require('metrics.http_middleware')
+        return http_middleware.v1(handler)
+    end
+    return handler
+end
+
+local function routes_equal(old, new)
+    assert(type(old.metrics) == 'table')
+    assert(type(new.metrics) == 'table')
+
+    if old.format ~= new.format or old.metrics.enabled ~= new.metrics.enabled then
+        return false
+    end
+
+    return true
+end
+
 local function apply_http(conf)
     local enabled = {}
     for _, node in ipairs(conf) do
@@ -245,28 +282,31 @@ local function apply_http(conf)
             local new_routes = {}
             for _, endpoint in ipairs(node.endpoints) do
                 local path = remove_side_slashes(endpoint.path)
-                new_routes[path] = endpoint.format
+                new_routes[path] = {
+                    format = endpoint.format,
+                    metrics = endpoint.metrics or {},
+                }
             end
 
             -- Remove old routes.
-            for path, format in pairs(old_routes) do
-                if new_routes[path] == nil or new_routes[path] ~= format then
+            for path, e in pairs(old_routes) do
+                if new_routes[path] == nil or not routes_equal(e, new_routes[path]) then
                     delete_route(httpd, path)
                     old_routes[path] = nil
                 end
             end
 
             -- Add new routes.
-            for path, format in pairs(new_routes) do
+            for path, endpoint in pairs(new_routes) do
                 if old_routes[path] == nil then
                     httpd:route({
                         method = "GET",
                         path = path,
                         name = path,
-                    }, http_handlers[format])
+                    }, wrap_handler(http_handlers[endpoint.format], endpoint.metrics))
                 else
                     assert(old_routes[path] == nil
-                           or old_routes[path] == new_routes[path])
+                           or routes_equal(old_routes[path], new_routes[path]))
                 end
             end
 

test/entrypoint/config.yaml

@@ -25,6 +25,10 @@ groups:
                     format: prometheus
                   - path: /metrics/json/
                     format: json
+                  - path: /metrics/observed/prometheus
+                    format: prometheus
+                    metrics:
+                      enabled: true
             iproto:
               listen:
                 - uri: '127.0.0.1:3313'

test/integration/role_test.lua

@@ -66,6 +66,34 @@ local function assert_prometheus(uri)
     t.assert_not(ok)
 end
 
+local function assert_observed(host, path)
+    -- Trigger observation.
+    http_client.get(host .. path)
+
+    local response = http_client.get(host .. path)
+    t.assert_equals(response.status, 200)
+    t.assert(response.body)
+
+    local pattern = "http_server_request_latency_count.*" .. path
+    t.assert_str_contains(response.body, pattern, true)
+    local ok = pcall(json.decode, response.body)
+    t.assert_not(ok)
+end
+
+local function assert_not_observed(host, path)
+    -- Trigger observation.
+    http_client.get(host .. path)
+
+    local response = http_client.get(host .. path)
+    t.assert_equals(response.status, 200)
+    t.assert(response.body)
+
+    local pattern = "http_server_request_latency_count.*" .. path
+    t.assert_not_str_contains(response.body, pattern, true)
+    local ok = pcall(json.decode, response.body)
+    t.assert_not(ok)
+end
+
 g.test_endpoints = function()
     assert_json("http://127.0.0.1:8081/metrics/json")
     assert_json("http://127.0.0.1:8081/metrics/json/")
@@ -76,4 +104,8 @@ g.test_endpoints = function()
     assert_prometheus("http://127.0.0.1:8082/metrics/prometheus/")
     assert_json("http://127.0.0.1:8082/metrics/json")
     assert_json("http://127.0.0.1:8082/metrics/json/")
+
+    assert_not_observed("http://127.0.0.1:8081", "/metrics/prometheus")
+    assert_not_observed("http://127.0.0.1:8082", "/metrics/prometheus")
+    assert_observed("http://127.0.0.1:8082", "/metrics/observed/prometheus")
 end

test/unit/middleware_test.lua

@@ -0,0 +1,138 @@
+local http_client = require('http.client')
+local http_middleware = require('metrics.http_middleware')
+local metrics = require('metrics')
+
+local t = require('luatest')
+local g = t.group()
+
+g.before_all(function(cg)
+    cg.role = require('roles.metrics-export')
+end)
+
+g.before_each(function(cg)
+    cg.collector_name = http_middleware.get_default_collector().name
+end)
+
+g.after_each(function(cg)
+    metrics.clear()
+    http_middleware.set_default_collector(nil)
+
+    cg.role.stop()
+end)
+
+local function assert_contains_http(cg, uri, path_pattern)
+    local response = http_client.get(uri)
+    t.assert(response.body)
+
+    local data = response.body
+    local expected = ("%s_count"):format(cg.collector_name)
+    if path_pattern ~= nil then
+        expected = expected .. ".*" .. path_pattern
+    end
+    t.assert_str_contains(data, expected, true)
+end
+
+local function assert_not_contains_http(cg, uri, path_pattern)
+    local response = http_client.get(uri)
+    t.assert(response.body)
+
+    local data = response.body
+    local expected = ("%s_count"):format(cg.collector_name)
+    if path_pattern ~= nil then
+        expected = expected .. ".*" .. path_pattern
+    end
+    t.assert_not_str_contains(data, expected, true)
+end
+
+local function trigger_http(uri)
+    local response = http_client.get(uri)
+    t.assert(response.body)
+end
+
+g.test_http_metrics_disabled_by_default = function(cg)
+    cg.role.apply({
+        http = {
+            {
+                listen = 8081,
+                endpoints = {
+                    {
+                        path = "/prometheus",
+                        format = "prometheus",
+                    },
+                    {
+                        path = "/json",
+                        format = "json",
+                    },
+                },
+            },
+        },
+    })
+
+    trigger_http("http://127.0.0.1:8081/prometheus")
+    trigger_http("http://127.0.0.1:8081/json")
+
+
+    assert_not_contains_http(cg, "http://127.0.0.1:8081/prometheus")
+    assert_not_contains_http(cg, "http://127.0.0.1:8081/json")
+end
+
+g.test_enabled_http_metrics = function(cg)
+    cg.role.apply({
+        http = {
+            {
+                listen = 8081,
+                endpoints = {
+                    {
+                        path = "/prometheus",
+                        format = "prometheus",
+                        metrics = {enabled = true},
+                    },
+                    {
+                        path = "/json",
+                        format = "json",
+                        metrics = {enabled = true},
+                    },
+                },
+            },
+        },
+    })
+
+    trigger_http("http://127.0.0.1:8081/prometheus")
+    trigger_http("http://127.0.0.1:8081/json")
+
+    assert_contains_http(cg, "http://127.0.0.1:8081/prometheus", [[path="/prometheus"]])
+    assert_contains_http(cg, "http://127.0.0.1:8081/prometheus", [[path="/json"]])
+
+    assert_contains_http(cg, "http://127.0.0.1:8081/json", [["path":"/prometheus"]])
+    assert_contains_http(cg, "http://127.0.0.1:8081/json", [["path":"/json"]])
+end
+
+g.test_enabled_http_metrics_for_one_endpoint = function(cg)
+    cg.role.apply({
+        http = {
+            {
+                listen = 8081,
+                endpoints = {
+                    {
+                        path = "/prometheus",
+                        format = "prometheus",
+                        metrics = {enabled = true},
+                    },
+                    {
+                        path = "/json",
+                        format = "json",
+                    },
+                },
+            },
+        },
+    })
+
+    trigger_http("http://127.0.0.1:8081/prometheus")
+    trigger_http("http://127.0.0.1:8081/json")
+
+    assert_contains_http(cg, "http://127.0.0.1:8081/prometheus", [[path="/prometheus"]])
+    assert_contains_http(cg, "http://127.0.0.1:8081/json", [["path":"/prometheus"]])
+
+    assert_not_contains_http(cg, "http://127.0.0.1:8081/prometheus", [[path="/json"]])
+    assert_not_contains_http(cg, "http://127.0.0.1:8081/json", [["path":"/json"]])
+end