feat: add option to include request body and response body in log util (

#10888)

apisix/plugins/elasticsearch-logger.lua

@@ -75,7 +75,23 @@ local schema = {
         ssl_verify = {
             type = "boolean",
             default = true
-        }
+        },
+        include_req_body = {type = "boolean", default = false},
+        include_req_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+        include_resp_body = { type = "boolean", default = false },
+        include_resp_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
     },
     encrypt_fields = {"auth.password"},
     oneOf = {
@@ -171,6 +187,11 @@ local function send_to_elasticsearch(conf, entries)
 end
 
 
+function _M.body_filter(conf, ctx)
+    log_util.collect_body(conf, ctx)
+end
+
+
 function _M.log(conf, ctx)
     local entry = get_logger_entry(conf, ctx)
 

apisix/plugins/skywalking-logger.lua

@@ -39,6 +39,21 @@ local schema = {
         log_format = {type = "object"},
         timeout = {type = "integer", minimum = 1, default = 3},
         include_req_body = {type = "boolean", default = false},
+        include_req_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+        include_resp_body = { type = "boolean", default = false },
+        include_resp_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
     },
     required = {"endpoint_addr"},
 }
@@ -116,6 +131,11 @@ local function send_http_data(conf, log_message)
 end
 
 
+function _M.body_filter(conf, ctx)
+    log_util.collect_body(conf, ctx)
+end
+
+
 function _M.log(conf, ctx)
     local log_body = log_util.get_log_entry(plugin_name, conf, ctx)
     local trace_context

apisix/plugins/sls-logger.lua

@@ -33,6 +33,21 @@ local schema = {
     type = "object",
     properties = {
         include_req_body = {type = "boolean", default = false},
+        include_req_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+        include_resp_body = { type = "boolean", default = false },
+        include_resp_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
         timeout = {type = "integer", minimum = 1, default= 5000},
         log_format = {type = "object"},
         host = {type = "string"},
@@ -129,6 +144,12 @@ local function handle_log(entries)
     return send_tcp_data(entries[1].route_conf, data)
 end
 
+
+function _M.body_filter(conf, ctx)
+    log_util.collect_body(conf, ctx)
+end
+
+
 -- log phase in APISIX
 function _M.log(conf, ctx)
     local entry = log_util.get_log_entry(plugin_name, conf, ctx)
@@ -146,7 +167,7 @@ function _M.log(conf, ctx)
     }
     local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix",
                                       ctx.var.pid, json_str, structured_data)
-
+    core.log.info("collect_data:" .. rf5424_data)
     local process_context = {
         data = rf5424_data,
         route_conf = conf

apisix/plugins/syslog.lua

@@ -34,7 +34,22 @@ local schema = {
         pool_size = {type = "integer", minimum = 5, default = 5},
         tls = {type = "boolean", default = false},
         log_format = {type = "object"},
-        include_req_body = {type = "boolean", default = false}
+        include_req_body = {type = "boolean", default = false},
+        include_req_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+        include_resp_body = { type = "boolean", default = false },
+        include_resp_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
     },
     required = {"host", "port"}
 }
@@ -69,6 +84,11 @@ function _M.check_schema(conf, schema_type)
 end
 
 
+function _M.body_filter(conf, ctx)
+    log_util.collect_body(conf, ctx)
+end
+
+
 function _M.log(conf, ctx)
     local entry = log_util.get_log_entry(plugin_name, conf, ctx)
     syslog.push_entry(conf, ctx, entry)

apisix/plugins/syslog/init.lua

@@ -88,7 +88,7 @@ function _M.push_entry(conf, ctx, entry)
 
     local rfc5424_data = rfc5424.encode("SYSLOG", "INFO", ctx.var.host,
                                 "apisix", ctx.var.pid, json_str)
-
+    core.log.info("collect_data:" .. rfc5424_data)
     if batch_processor_manager:add_entry(conf, rfc5424_data) then
         return
     end

apisix/plugins/tcp-logger.lua

@@ -33,7 +33,22 @@ local schema = {
         tls_options = {type = "string"},
         timeout = {type = "integer", minimum = 1, default= 1000},
         log_format = {type = "object"},
-        include_req_body = {type = "boolean", default = false}
+        include_req_body = {type = "boolean", default = false},
+        include_req_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+        include_resp_body = { type = "boolean", default = false },
+        include_resp_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
     },
     required = {"host", "port"}
 }
@@ -77,6 +92,7 @@ local function send_tcp_data(conf, log_message)
     sock:settimeout(conf.timeout)
 
     core.log.info("sending a batch logs to ", conf.host, ":", conf.port)
+    core.log.info("sending log_message: ", log_message)
 
     local ok, err = sock:connect(conf.host, conf.port)
     if not ok then
@@ -109,6 +125,11 @@ local function send_tcp_data(conf, log_message)
 end
 
 
+function _M.body_filter(conf, ctx)
+    log_util.collect_body(conf, ctx)
+end
+
+
 function _M.log(conf, ctx)
     local entry = log_util.get_log_entry(plugin_name, conf, ctx)
 

apisix/plugins/udp-logger.lua

@@ -31,7 +31,22 @@ local schema = {
         port = {type = "integer", minimum = 0},
         timeout = {type = "integer", minimum = 1, default = 3},
         log_format = {type = "object"},
-        include_req_body = {type = "boolean", default = false}
+        include_req_body = {type = "boolean", default = false},
+        include_req_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
+        include_resp_body = { type = "boolean", default = false },
+        include_resp_body_expr = {
+            type = "array",
+            minItems = 1,
+            items = {
+                type = "array"
+            }
+        },
     },
     required = {"host", "port"}
 }
@@ -70,6 +85,7 @@ local function send_udp_data(conf, log_message)
     sock:settimeout(conf.timeout * 1000)
 
     core.log.info("sending a batch logs to ", conf.host, ":", conf.port)
+    core.log.info("sending log_message: ", log_message)
 
     local ok, err = sock:setpeername(conf.host, conf.port)
 
@@ -95,6 +111,11 @@ local function send_udp_data(conf, log_message)
 end
 
 
+function _M.body_filter(conf, ctx)
+    log_util.collect_body(conf, ctx)
+end
+
+
 function _M.log(conf, ctx)
     local entry = log_util.get_log_entry(plugin_name, conf, ctx)
 

docs/en/latest/plugins/elasticsearch-logger.md

@@ -48,6 +48,10 @@ When the Plugin is enabled, APISIX will serialize the request context informatio
 | auth.password | string  | True     |                             | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) password. |
 | ssl_verify    | boolean | False    | true                        | When set to `true` enables SSL verification as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake). |
 | timeout       | integer | False    | 10                          | Elasticsearch send data timeout in seconds.                  |
+| include_req_body       | boolean       | False    | false   | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations.                                                               |
+| include_req_body_expr  | array         | False    |         | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.        |
+| include_resp_body      | boolean       | False    | false   | When set to `true` includes the response body in the log.                                                                                                                                                                      |
+| include_resp_body_expr | array         | False    |         | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`.       |
 
 NOTE: `encrypt_fields = {"auth.password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).
 

docs/en/latest/plugins/skywalking-logger.md

@@ -44,6 +44,9 @@ If there is an existing tracing context, it sets up the trace-log correlation au
 | timeout               | integer | False    | 3                      | [1,...]       | Time to keep the connection alive for after sending a request.                                               |
 | name                  | string  | False    | "skywalking logger"    |               | Unique identifier to identify the logger. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`.                                                                    |
 | include_req_body      | boolean | False    | false                  | [false, true] | When set to `true` includes the request body in the log.                                                     |
+| include_req_body_expr  | array         | False    |         |               | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.        |
+| include_resp_body      | boolean       | False    | false   | [false, true] | When set to `true` includes the response body in the log.                                                                                                                                                                      |
+| include_resp_body_expr | array         | False    |         |               | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`.       |
 
 This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration.
 

docs/en/latest/plugins/sls-logger.md

@@ -46,6 +46,9 @@ It might take some time to receive the log data. It will be automatically sent a
 | access_key_id     | True     | AccessKey ID in Alibaba Cloud. See [Authorization](https://www.alibabacloud.com/help/en/log-service/latest/create-a-ram-user-and-authorize-the-ram-user-to-access-log-service) for more details.                                                                     |
 | access_key_secret | True     | AccessKey Secret in Alibaba Cloud. See [Authorization](https://www.alibabacloud.com/help/en/log-service/latest/create-a-ram-user-and-authorize-the-ram-user-to-access-log-service) for more details.                                                                 |
 | include_req_body  | True     | When set to `true`, includes the request body in the log.                                                                                                                                                                                       |
+| include_req_body_expr | No      | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.                                                                                                                          |
+| include_resp_body  | No  | When set to `true` includes the response body in the log.                                                                                                        |
+| include_resp_body_expr | No  |  Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.                                                                                                                        |
 | name              | False    | Unique identifier for the batch processor. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`.                                                                                                                                                                                                      |
 
 NOTE: `encrypt_fields = {"access_key_secret"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields).

docs/en/latest/plugins/syslog.md

@@ -46,7 +46,10 @@ Logs can be set as JSON objects.
 | sock_type        | string  | False    | "tcp"        | ["tcp", "udp] | Transport layer protocol to use.                                                                                         |
 | pool_size        | integer | False    | 5            | [5, ...]      | Keep-alive pool size used by `sock:keepalive`.                                                                           |
 | log_format       | object  | False    |  |              | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
-| include_req_body | boolean | False    | false        |               | When set to `true` includes the request body in the log.                                                                 |
+| include_req_body | boolean | False    | false        | [false, true] | When set to `true` includes the request body in the log.                                                                 |
+| include_req_body_expr  | array         | False    |         |               | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.        |
+| include_resp_body      | boolean       | False    | false   | [false, true] | When set to `true` includes the response body in the log.                                                                                                                                                                      |
+| include_resp_body_expr | array         | False    |         |               | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`.       |
 
 This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration.
 

docs/en/latest/plugins/tcp-logger.md

@@ -46,7 +46,10 @@ This plugin also allows to push logs as a batch to your external TCP server. It
 | log_format       | object  | False    |  |              | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. |
 | tls              | boolean | False    | false   |              | When set to `true` performs SSL verification.            |
 | tls_options      | string  | False    |         |              | TLS options.                                             |
-| include_req_body | boolean | False    | false   |              | When set to `true` includes the request body in the log. |
+| include_req_body | boolean | False    | false   | [false, true] | When set to `true` includes the request body in the log. |
+| include_req_body_expr  | array   | No       |         |               | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.                                                                                                                          |
+| include_resp_body | boolean | No       | false   | [false, true] | When set to `true` includes the response body in the log.                                                                                                        |
+| include_resp_body_expr | array   | No  |         |               | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more.                                                                                                                        |
 
 This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration.