diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..6918180 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,24 @@ +name: Tests + +on: [pull_request] + + +jobs: + test: + env: + GO111MODULE: on + name: Test + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + - name: Before install + run: | + go get golang.org/x/tools/cmd/cover + go get github.com/mattn/goveralls + - name: Test + working-directory: ./logzio-lambda-extensions-logs/utils + run: go test -v -race -covermode=atomic -coverprofile=coverage.out \ No newline at end of file diff --git a/logzio-lambda-extensions-logs/README.md b/logzio-lambda-extensions-logs/README.md index c99235a..5de610e 100644 --- a/logzio-lambda-extensions-logs/README.md +++ b/logzio-lambda-extensions-logs/README.md @@ -2,22 +2,23 @@ Lambda extensions enable tools to integrate deeply into the Lambda execution environment to control and participate in Lambda’s lifecycle. To read more about Lambda Extensions, [click here](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-extensions-api.html). -The Logz.io Lambda extension for logs, uses the AWS Extensions API and [AWS Logs API](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-logs-api.html), and sends your Lambda Function Logs directly to your Logz.io account. +The Logz.io Lambda extension for logs uses the AWS Extensions API and [AWS Logs API](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-logs-api.html), and sends your Lambda Function Logs directly to your Logz.io account. This repo is based on the [AWS lambda extensions sample](https://github.com/aws-samples/aws-lambda-extensions). -This extension is written in Go, but can be run with runtimes that support extensions](https://docs.aws.amazon.com/lambda/latest/dg/using-extensions.html). +This extension is written in Go, but can be run with runtimes that support [extensions](https://docs.aws.amazon.com/lambda/latest/dg/using-extensions.html). ### Prerequisites * Lambda function with [supported runtime](https://docs.aws.amazon.com/lambda/latest/dg/using-extensions.html) for extensions. * AWS Lambda limitations: A function can use up to five layers at a time. The total unzipped size of the function and all layers cannot exceed the unzipped deployment package size limit of 250 MB. -### Important notes: -* If the extension won't have enough time to receive logs from AWS Logs API, it may send the logs in the next invocation of the Lambda function. -So if you want that all the logs will be sent by the end of your function's run, you'll need to add at the end of your Lambda function code a sleep interval that will allow the extension enough time to do it's job. +### Important notes + +* If the extension doesn't have enough time to receive logs from AWS Logs API, it may send the logs in the next invocation of the Lambda function. + So if you want that all the logs are sent by the end of your function's run, you'll need to add at the end of your Lambda function code a sleep interval that will allow the extension enough time to do its job. * Due to [Lambda's execution environment lifecycle](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-context.html), the extension is being invoked on two events - `INVOKE` and `SHUTDOWN`. -That means that if your Lambda function goes into the `SHUTDOWN` phase, the extension will run and if there are logs in it's queue, it will send them. + This means that if your Lambda function goes into the `SHUTDOWN` phase, the extension will run and if there are logs in its queue, it will send them. ### Extension deployment options @@ -44,7 +45,7 @@ aws lambda update-function-configuration \ | Placeholder | Description | |---|---| -| `<>` | Name of the Lambda Function you want to monitor. | +| `<>` | Name of the Lambda function you want to monitor. | | `<>` | A space-separated list of function layers to add to the function's execution environment. Specify each layer by its ARN, including the version. For the ARN, see the [**Lambda extension versions** table](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#lambda-extension-versions).| | `<>` | Key-value pairs containing environment variables that are accessible from function code during execution. Should appear in the following format: `KeyName1=string,KeyName2=string`. For a list of all the environment variables for the extension, see the [**Lambda environment variables** table](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#environment-variables).| @@ -60,7 +61,7 @@ aws lambda update-function-configuration \ ``` **NOTE:** This command overwrites the existing function configuration. If you already have your own layers and environment variables for your function, include them in the list. - + #### Deleting the extension @@ -80,13 +81,13 @@ aws lambda update-function-configuration \ ##### Add the extension to your Lambda Function 1. In the Lambda Functions screen, choose the function you want to monitor. -![Pick lambda function](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-1.jpg) + ![Pick lambda function](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-1.jpg) 2. In the page for the function, scroll down to the `Layers` section and choose `Add Layer`. -![Add layer](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-2.jpg) + ![Add layer](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-2.jpg) 3. Select the `Specify an ARN` option, then choose the ARN of the extension with the region code that matches your Lambda Function region from the [**Lambda extension versions** table](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#lambda-extension-versions), and click the `Add` button. -![Add ARN extension](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-3.jpg) + ![Add ARN extension](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-3.jpg) ##### Configure the extension parameters @@ -101,6 +102,65 @@ Run the function. It may take more than one run of the function for the logs to - To delete the **extension layer**: In your function page, go to the **layers** panel. Click `edit`, select the extension layer, and click `save`. - To delete the extension's **environment variables**: In your function page, select the `Configuration` tab, select `Environment variables`, click `edit`, and remove the variables that you added for the extension. +### Parsing logs + +By default, the extension sends the logs as strings. +If your logs are formatted, and you wish to parse them to separate fields, the extension will use the [grok library](https://github.com/vjeantet/grok) to parse grok patterns. +You can see all the pre-built grok patterns (for example `COMMONAPACHELOG` is already a known pattern in the library) [here](https://github.com/vjeantet/grok/tree/master/patterns). +If you need to use a custom pattern, you can use the environment variables `GROK_PATTERNS` and `LOGS_FORMAT`. + +#### Example + +For logs that are formatted like this: + +```python +%(app_name)s : %(message)s +``` + +we will use `cool app` as the `app_name` and the `message` will have strings containing whitespaces, letters and numbers. + +In Logz.io we wish to have `app_name`, `message` in their own fields, named `my_app` and `my_message`, respectively. +To do so, we'll set the environment variables as follows: + +##### GROK_PATTERNS + +The `GROK_PATTERNS` variable should be in a JSON format. +The key is used as the pattern name, and the value should be the regex that captures the pattern. +In our case, while `app_name` always stays `cool app`, we don't know what `message` will be, so we need to set `GROK_PATTERNS` as: `{"app_name":"cool app","message":".*"}` + +##### LOGS_FORMAT + +The `LOGS_FORMAT` variable will contain the same format as the logs, according to the pattern names that we used in `GROK_PATTERNS`. +The variable should be in a grok format for each pattern name: `${PATTERN_NAME:FIELD_NAME}` where `PATTERN_NAME` is the pattern name from `GROK_PATTERNS`, and `FIELD_NAME` is the name of the field you want the pattern to be parsed to. +**Note** that the `FIELD_NAME` cannot contain a dot (`.`) in it. +In our case, we want `app_name` to appear under the field `my_app`, and `message` to appear under the field `my_message`. Since we know that the logs format is as mentioned above, we will set `LOGS_FORMAT` as: `%{app_name:my_app} : %{message:my_message}`. + +The logs that match the configuration above will appear in Logz.io with the fields `lambda.record.my_app`, `lambda.record.my_message`. +The log: `"cool app : The sky is so blue"`, will be parsed to look like this: +``` +my_app: cool app +my_message: The sky is so blue +``` + +To learn more about grok, read the [grok library](https://github.com/vjeantet/grok), [Logz.io's blog post](https://logz.io/blog/logstash-grok/), or watch [this introduction to grok video](https://logz.io/learn/introduction-to-the-logstash-grok/). + +### Nested fields + +As of v0.2.0 the extension can detect if a log is in a JSON format, and to parse the fields to appear as nested fields in the Logz.io app. +For example, the following log: + +``` +{ "foo": "bar", "field2": "val2" } +``` + +Will appear under the fields: +``` +message_nested.foo: bar +message_nested.field2: val2 +``` + +**Note:** The user must insert a valid JSON. Sending a dictionary or any key-value data structure that is not in a JSON format will cause the log to be sent as a string. + ### Environment Variables | Name | Description |Required/Default| @@ -110,11 +170,14 @@ Run the function. It may take more than one run of the function for the logs to | `LOGS_EXT_LOG_LEVEL` | Log level of the extension. Can be set to one of the following: `debug`, `info`, `warn`, `error`, `fatal`, `panic`. |Default: `info` | | `ENABLE_EXTENSION_LOGS` | Set to `true` if you wish the extension logs will be shipped to your Logz.io account. | Default: `false` | | `ENABLE_PLATFORM_LOGS` | The platform log captures runtime or execution environment errors. Set to `true` if you wish the platform logs will be shipped to your Logz.io account. | Default: `false` | +| `GROK_PATTERNS` | Must be set with `LOGS_FORMAT`. Use this if you want to parse your logs into fields. A minified JSON list that contains the field name and the regex that will match the field. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - | +| `LOGS_FORMAT` | Must be set with `GROK_PATTERNS`. Use this if you want to parse your logs into fields. The format in which the logs will appear, in accordance to grok conventions. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - | ### Lambda extension versions | Version | Supported Runtimes | AWS ARN | | --- | --- | --- | +| 0.2.0 | `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Python 3.9`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Custom runtime` | `arn:aws:lambda:<>:486140753397:layer:LogzioLambdaExtensionLogs:3` | | 0.1.0| `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Node.js 10.x`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Ruby 2.5`, `Custom runtime`| `arn:aws:lambda:<>:486140753397:layer:LogzioLambdaExtensionLogs:2` | | 0.0.1 | `Python 3.7`, `Python 3.8` | `arn:aws:lambda:<>:486140753397:layer:LogzioLambdaExtensionLogs:1` | @@ -130,7 +193,7 @@ Run the function. It may take more than one run of the function for the logs to | Europe (Ireland) | `eu-west-1` | - | | Europe (Stockholm) | `eu-north-1` | - | | Asia Pacific (Sydney) | `ap-southeast-2` | Available from v0.1.0 | -| Canada (Central) ) | `ca-central-1` | Available from v0.1.0 | +| Canada (Central) | `ca-central-1` | Available from v0.1.0 | **NOTE:** If your AWS region is not in the list, please reach out to Logz.io's support or open an issue in this repo. @@ -141,6 +204,10 @@ Note: the dependencies layer is deprecated. | 0.0.1 | `requests` | `arn:aws:lambda:<>:486140753397:layer:LogzioLambdaExtensionLogsLibs:1` | ### Changelog: + +- **0.2.0**: + - Allow parsing log into fields. To learn more see [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. + - Allow nested JSON within logs. To learn more see [nested fields](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#nested-fields) section. - **0.1.0**: - - **BREAKING CHANGES**: Written in Go, supports multiple runtimes. Compatible with the GA version of the Extensions API. + - **BREAKING CHANGES**: Written in Go, supports multiple runtimes. Compatible with the GA version of the Extensions API. - **0.0.1**: Initial release. Supports only python 3.7, python 3.8 runtimes. Compatible with the beta version of the Extensions API. diff --git a/logzio-lambda-extensions-logs/agent/http.go b/logzio-lambda-extensions-logs/agent/http.go index 26c9f30..8f61c19 100644 --- a/logzio-lambda-extensions-logs/agent/http.go +++ b/logzio-lambda-extensions-logs/agent/http.go @@ -179,4 +179,4 @@ func getEventTypesToRegister() []logsapi.EventType { } return eventTypes -} \ No newline at end of file +} diff --git a/logzio-lambda-extensions-logs/agent/logger.go b/logzio-lambda-extensions-logs/agent/logger.go index f5ea4b3..1d45e60 100644 --- a/logzio-lambda-extensions-logs/agent/logger.go +++ b/logzio-lambda-extensions-logs/agent/logger.go @@ -11,7 +11,7 @@ import ( ) const ( - maxBulkSizeBytes = 10 * 1024 * 1024 // 10 MB + maxBulkSizeBytes = 10 * 1024 * 1024 // 10 MB ) var logger = log.WithFields(log.Fields{"agent": "logsApiAgent"}) @@ -35,7 +35,7 @@ func NewLogzioLogger() (*logzio.LogzioSender, error) { logzio.SetInMemoryQueue(true), logzio.SetDebug(os.Stdout), logzio.SetinMemoryCapacity(maxBulkSizeBytes), //bytes - logzio.SetDrainDuration(time.Second * 5), + logzio.SetDrainDuration(time.Second*5), logzio.SetDebug(os.Stdout), ) } else { @@ -45,7 +45,7 @@ func NewLogzioLogger() (*logzio.LogzioSender, error) { logzio.SetInMemoryQueue(true), logzio.SetDebug(os.Stdout), logzio.SetinMemoryCapacity(maxBulkSizeBytes), //bytes - logzio.SetDrainDuration(time.Second * 5), + logzio.SetDrainDuration(time.Second*5), ) } @@ -54,4 +54,4 @@ func NewLogzioLogger() (*logzio.LogzioSender, error) { } return logzioLogger, nil -} \ No newline at end of file +} diff --git a/logzio-lambda-extensions-logs/extension/client.go b/logzio-lambda-extensions-logs/extension/client.go index 0257d0e..1ee9aa5 100644 --- a/logzio-lambda-extensions-logs/extension/client.go +++ b/logzio-lambda-extensions-logs/extension/client.go @@ -198,4 +198,4 @@ func (e *Client) ExitError(ctx context.Context, errorType string) (*StatusRespon return nil, err } return &res, nil -} \ No newline at end of file +} diff --git a/logzio-lambda-extensions-logs/go.mod b/logzio-lambda-extensions-logs/go.mod index 5291645..b6228fc 100644 --- a/logzio-lambda-extensions-logs/go.mod +++ b/logzio-lambda-extensions-logs/go.mod @@ -4,7 +4,9 @@ go 1.15 require ( github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259 - github.com/logzio/logzio-go v1.0.2 + github.com/logzio/logzio-go v1.0.3 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.6.1 + github.com/vjeantet/grok v1.0.1 ) diff --git a/logzio-lambda-extensions-logs/logsapi/client.go b/logzio-lambda-extensions-logs/logsapi/client.go index 9c85cee..db21527 100644 --- a/logzio-lambda-extensions-logs/logsapi/client.go +++ b/logzio-lambda-extensions-logs/logsapi/client.go @@ -172,4 +172,4 @@ func httpPutWithHeaders(client *http.Client, url string, data []byte, headers *m } return resp, nil -} \ No newline at end of file +} diff --git a/logzio-lambda-extensions-logs/main.go b/logzio-lambda-extensions-logs/main.go index c825550..f06ad4d 100644 --- a/logzio-lambda-extensions-logs/main.go +++ b/logzio-lambda-extensions-logs/main.go @@ -20,6 +20,7 @@ import ( // INITIAL_QUEUE_SIZE is the initial size set for the synchronous logQueue const INITIAL_QUEUE_SIZE = 5 + func main() { extensionName := path.Base(os.Args[0]) printPrefix := fmt.Sprintf("[%s]", extensionName) @@ -167,4 +168,4 @@ func setLogLevel() { // otherwise, we'll use the logger's default log level (info) log.SetLevel(logLevel) } -} \ No newline at end of file +} diff --git a/logzio-lambda-extensions-logs/utils/converter.go b/logzio-lambda-extensions-logs/utils/converter.go index b3d1b79..577e896 100644 --- a/logzio-lambda-extensions-logs/utils/converter.go +++ b/logzio-lambda-extensions-logs/utils/converter.go @@ -1,31 +1,144 @@ package utils +import ( + "encoding/json" + "fmt" + "github.com/vjeantet/grok" +) + const ( - fldLogzioTimestamp = "@timestamp" - fldLambdaTime = "time" - fldLogzioType = "type" - fldLambdaType = "type" - fldLogzioLambdaType = "lambda.log.type" - fldLambdaRecord = "record" - fldLogzioMsg = "message" - fldLogzioLambdaRecord = "lambda.record" - - extensionType = "lambda-extension-logs" + FldLogzioTimestamp = "@timestamp" + FldLambdaTime = "time" + FldLogzioType = "type" + FldLambdaType = "type" + FldLogzioLambdaType = "lambda.log.type" + FldLambdaRecord = "record" + FldLogzioMsg = "message" + FldLogzioMsgNested = "message_nested" + FldLogzioLambdaRecord = "lambda.record" + + ExtensionType = "lambda-extension-logs" + + grokKeyLogFormat = "LOG_FORMAT" ) // ConvertLambdaLogToLogzioLog converts a log that was sent from AWS Logs API to a log in a Logz.io format -func ConvertLambdaLogToLogzioLog(lambdaLog map[string]interface{}) map[string]interface{}{ +func ConvertLambdaLogToLogzioLog(lambdaLog map[string]interface{}) map[string]interface{} { + sendAsString := false logzioLog := make(map[string]interface{}) - logzioLog[fldLogzioTimestamp] = lambdaLog[fldLambdaTime] - logzioLog[fldLogzioType] = extensionType - logzioLog[fldLogzioLambdaType] = lambdaLog[fldLambdaType] + logzioLog[FldLogzioTimestamp] = lambdaLog[FldLambdaTime] + logzioLog[FldLogzioType] = ExtensionType + logzioLog[FldLogzioLambdaType] = lambdaLog[FldLambdaType] + logger.Debugf("working on: %v", lambdaLog[FldLambdaRecord]) - switch lambdaLog[fldLambdaRecord].(type) { + switch lambdaLog[FldLambdaRecord].(type) { case string: - logzioLog[fldLogzioMsg] = lambdaLog[fldLambdaRecord] + grokPattern := GetGrokPatterns() + logsFormat := GetLogsFormat() + if len(grokPattern) > 0 && len(logsFormat) > 0 { + logger.Debugf("grok pattern: %s", grokPattern) + logger.Debugf("logs format: %s", logsFormat) + logger.Info("detected grok pattern and logs format. trying to parse log") + err := parseFields(logzioLog, lambdaLog[FldLambdaRecord].(string), grokPattern, logsFormat) + if err != nil { + logger.Errorf("error occurred while trying to parse fields. sedning log as a string: %s", err.Error()) + sendAsString = true + } + } else { + if len(grokPattern) > 0 || len(logsFormat) > 0 { + logger.Error("grok pattern and logs format must be set in order to parse fields. sending log as string.") + } + + sendAsString = true + } + + if sendAsString { + var nested map[string]interface{} + err := json.Unmarshal([]byte(fmt.Sprintf(`%s`, lambdaLog[FldLambdaRecord])), &nested) + if err != nil { + logger.Infof("error occurred while checking if log %s is JSON. ignore if this is not JSON: %s", lambdaLog[FldLambdaRecord], err.Error()) + logzioLog[FldLogzioMsg] = lambdaLog[FldLambdaRecord] + } else { + logger.Debugf("detected JSON: %s", lambdaLog[FldLambdaRecord]) + logzioLog[FldLogzioMsgNested] = nested + } + } default: - logzioLog[fldLogzioLambdaRecord] = lambdaLog[fldLambdaRecord] + logzioLog[FldLogzioLambdaRecord] = lambdaLog[FldLambdaRecord] } return logzioLog } + +func parseFields(logMap map[string]interface{}, fieldsToParse, grokPatterns, logsFormat string) error { + g, err := grok.NewWithConfig(&grok.Config{NamedCapturesOnly: true}) + if err != nil { + return err + } + + err = addGrokPatterns(g, grokPatterns, logsFormat) + if err != nil { + return err + } + + logger.Debugf("about to parse: %s", fieldsToParse) + fields, err := g.Parse(fmt.Sprintf(`%%{%s}`, grokKeyLogFormat), fmt.Sprintf(`%s`, fieldsToParse)) + logger.Debugf("number of fields after grok: %d", len(fields)) + if err != nil { + return err + } + + if len(fields) == 0 { + return fmt.Errorf("could not parse fields with the current patterns & format") + } + + addFields(logMap, fields) + + return nil +} + +func addGrokPatterns(g *grok.Grok, patternsStr, logFormat string) error { + var grokPatterns map[string]string + err := json.Unmarshal([]byte(patternsStr), &grokPatterns) + if err != nil { + return err + } + + for key, val := range grokPatterns { + fVal := fmt.Sprintf(`%s`, val) + logger.Debugf("adding pattern %s", fVal) + g.AddPattern(key, fVal) + } + + logger.Debugf("added patterns from user") + + err = g.AddPattern(grokKeyLogFormat, fmt.Sprintf(`%s`, logFormat)) + if err != nil { + return err + } + + logger.Debugf("added %s: %s", grokKeyLogFormat, logFormat) + + return nil +} + +func addFields(logsMap map[string]interface{}, fields map[string]string) { + var nested map[string]interface{} + for key, val := range fields { + logger.Debugf("adding field: %s to logzio log", key) + // Trying to see if the string is in JSON format. + // If so - add the nested version to the log + err := json.Unmarshal([]byte(fmt.Sprintf(`%s`, val)), &nested) + if err != nil { + logger.Infof("error occurred while checking if log %s is JSON. ignore if this is not JSON: %s", val, err.Error()) + } else { + logger.Debugf("detected JSON: %s", val) + } + + if nested != nil && len(nested) > 0 { + logsMap[key] = nested + } else { + logsMap[key] = val + } + } +} diff --git a/logzio-lambda-extensions-logs/utils/converter_test.go b/logzio-lambda-extensions-logs/utils/converter_test.go new file mode 100644 index 0000000..02d6b31 --- /dev/null +++ b/logzio-lambda-extensions-logs/utils/converter_test.go @@ -0,0 +1,152 @@ +package utils_test + +import ( + "github.com/stretchr/testify/assert" + "logzio-lambda-extensions-logs/utils" + "os" + "testing" +) + +func TestConverterSimpleLog(t *testing.T) { + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "this is a simple log\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Equal(t, lambdaLog[utils.FldLambdaRecord], logzioLog[utils.FldLogzioMsg]) +} + +func TestConverterSimpleJsonLog(t *testing.T) { + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "{\"foo\": \"bar\"}\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Equal(t, "bar", logzioLog[utils.FldLogzioMsgNested].(map[string]interface{})["foo"]) +} + +func TestConverterGrokFormattedLog(t *testing.T) { + os.Setenv("GROK_PATTERNS", "{\"app_name\":\"cool app\",\"my_message\":\".*\"}") + os.Setenv("LOGS_FORMAT", "%{app_name:my_app} : %{my_message:my_message}") + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "cool app : this is a formatted log\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Equal(t, "cool app", logzioLog["my_app"]) + assert.Equal(t, "this is a formatted log", logzioLog["my_message"]) +} + +func TestConverterGrokFormattedLogWithJson(t *testing.T) { + os.Setenv("GROK_PATTERNS", "{\"app_name\":\"cool app\",\"my_message\":\".*\"}") + os.Setenv("LOGS_FORMAT", "%{app_name:my_app} : %{my_message:my_message}") + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "cool app : {\"foo\": \"bar\"}\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Equal(t, "cool app", logzioLog["my_app"]) + assert.Equal(t, "bar", logzioLog["my_message"].(map[string]interface{})["foo"]) +} + +func TestConverterGrokFormattedLogIncorrectLogsFormat(t *testing.T) { + os.Setenv("GROK_PATTERNS", "{\"app_name\":\"cool app\",\"my_message\":\".*\"}") + os.Setenv("LOGS_FORMAT", "%{app_name:my_app} = %{my_message:my_message}") + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "cool app : this is a formatted log\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Nil(t, logzioLog["my_app"]) + assert.Nil(t, logzioLog["my_message"]) + assert.NotNil(t, logzioLog[utils.FldLogzioMsg]) + assert.Equal(t, "cool app : this is a formatted log\n" , logzioLog[utils.FldLogzioMsg]) +} + +func TestConverterGrokFormattedLogIncorrectGrokPattern(t *testing.T) { + os.Setenv("GROK_PATTERNS", "{\"app_name\":\"some app\",\"my_message\":\".*\"}") + os.Setenv("LOGS_FORMAT", "%{app_name:my_app} : %{my_message:my_message}") + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "cool app : this is a formatted log\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Nil(t, logzioLog["my_app"]) + assert.Nil(t, logzioLog["my_message"]) + assert.NotNil(t, logzioLog[utils.FldLogzioMsg]) + assert.Equal(t, "cool app : this is a formatted log\n" , logzioLog[utils.FldLogzioMsg]) +} + +func TestConverterGrokFormattedLogNoGrokPattern(t *testing.T) { + os.Setenv("LOGS_FORMAT", "%{app_name:my_app} : %{my_message:my_message}") + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "cool app : this is a formatted log\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Nil(t, logzioLog["my_app"]) + assert.Nil(t, logzioLog["my_message"]) + assert.NotNil(t, logzioLog[utils.FldLogzioMsg]) + assert.Equal(t, "cool app : this is a formatted log\n" , logzioLog[utils.FldLogzioMsg]) +} + +func TestConverterGrokFormattedLogNoLogsFormat(t *testing.T) { + os.Setenv("LOGS_FORMAT", "%{app_name:my_app} : %{my_message:my_message}") + lambdaLog := map[string]interface{} { + utils.FldLambdaTime: "2021-11-11T08:28:16.870Z", + utils.FldLambdaType: "function", + utils.FldLambdaRecord: "cool app : this is a formatted log\n", + } + + logzioLog := utils.ConvertLambdaLogToLogzioLog(lambdaLog) + assert.NotNil(t, logzioLog) + assert.NotZero(t, len(logzioLog)) + assert.Equal(t, lambdaLog[utils.FldLambdaTime], logzioLog[utils.FldLogzioTimestamp]) + assert.Equal(t, lambdaLog[utils.FldLambdaType], logzioLog[utils.FldLogzioLambdaType]) + assert.Nil(t, logzioLog["my_app"]) + assert.Nil(t, logzioLog["my_message"]) + assert.NotNil(t, logzioLog[utils.FldLogzioMsg]) + assert.Equal(t, "cool app : this is a formatted log\n" , logzioLog[utils.FldLogzioMsg]) +} \ No newline at end of file diff --git a/logzio-lambda-extensions-logs/utils/getters.go b/logzio-lambda-extensions-logs/utils/getters.go index 7a55014..6165e5b 100644 --- a/logzio-lambda-extensions-logs/utils/getters.go +++ b/logzio-lambda-extensions-logs/utils/getters.go @@ -15,6 +15,8 @@ const ( envEnablePlatformLogs = "ENABLE_PLATFORM_LOGS" envEnableExtensionLogs = "ENABLE_EXTENSION_LOGS" envExtensionLogLevel = "LOGS_EXT_LOG_LEVEL" + envGrokPatterns = "GROK_PATTERNS" + envLogsFormat = "LOGS_FORMAT" LogLevelDebug = "debug" LogLevelInfo = "info" LogLevelWarn = "warn" @@ -24,7 +26,7 @@ const ( defaultEnablePlatformLogs = false defaultEnableExtensionLogs = false defaultExtensionLogLevel = LogLevelInfo - ) +) func GetToken() (string, error) { token := os.Getenv(envLogzioLogsToken) @@ -86,4 +88,12 @@ func GetExtensionLogLevel() string { logger.Infof("Reverting to default log level: %s", defaultExtensionLogLevel) return defaultExtensionLogLevel -} \ No newline at end of file +} + +func GetGrokPatterns() string { + return os.Getenv(envGrokPatterns) +} + +func GetLogsFormat() string { + return os.Getenv(envLogsFormat) +}