0.2 (#10)

.github/workflows/tests.yml

@@ -0,0 +1,24 @@
+name: Tests
+
+on: [pull_request]
+
+
+jobs:
+  test:
+    env:
+      GO111MODULE: on
+    name: Test
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+      - name: Before install
+        run: |
+          go get golang.org/x/tools/cmd/cover
+          go get github.com/mattn/goveralls
+      - name: Test
+        working-directory: ./logzio-lambda-extensions-logs/utils
+        run: go test -v -race -covermode=atomic -coverprofile=coverage.out

logzio-lambda-extensions-logs/README.md

@@ -2,22 +2,23 @@
 
 Lambda extensions enable tools to integrate deeply into the Lambda execution environment to control and participate in Lambda’s lifecycle.
 To read more about Lambda Extensions, [click here](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-extensions-api.html).  
-The Logz.io Lambda extension for logs, uses the AWS Extensions API and [AWS Logs API](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-logs-api.html), and sends your Lambda Function Logs directly to your Logz.io account.
+The Logz.io Lambda extension for logs uses the AWS Extensions API and [AWS Logs API](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-logs-api.html), and sends your Lambda Function Logs directly to your Logz.io account.
 
 This repo is based on the [AWS lambda extensions sample](https://github.com/aws-samples/aws-lambda-extensions).
 
-This extension is written in Go, but can be run with runtimes that support extensions](https://docs.aws.amazon.com/lambda/latest/dg/using-extensions.html).
+This extension is written in Go, but can be run with runtimes that support [extensions](https://docs.aws.amazon.com/lambda/latest/dg/using-extensions.html).
 
 ### Prerequisites
 
 * Lambda function with [supported runtime](https://docs.aws.amazon.com/lambda/latest/dg/using-extensions.html) for extensions.
 * AWS Lambda limitations: A function can use up to five layers at a time. The total unzipped size of the function and all layers cannot exceed the unzipped deployment package size limit of 250 MB.
 
-### Important notes:
-* If the extension won't have enough time to receive logs from AWS Logs API, it may send the logs in the next invocation of the Lambda function.
-So if you want that all the logs will be sent by the end of your function's run, you'll need to add at the end of your Lambda function code a sleep interval that will allow the extension enough time to do it's job.
+### Important notes
+
+* If the extension doesn't have enough time to receive logs from AWS Logs API, it may send the logs in the next invocation of the Lambda function.
+  So if you want that all the logs are sent by the end of your function's run, you'll need to add at the end of your Lambda function code a sleep interval that will allow the extension enough time to do its job.
 * Due to [Lambda's execution environment lifecycle](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-context.html), the extension is being invoked on two events - `INVOKE` and `SHUTDOWN`.
-That means that if your Lambda function goes into the `SHUTDOWN` phase, the extension will run and if there are logs in it's queue, it will send them.
+  This means that if your Lambda function goes into the `SHUTDOWN` phase, the extension will run and if there are logs in its queue, it will send them.
 
 ### Extension deployment options
 
@@ -44,7 +45,7 @@ aws lambda update-function-configuration \
 
 | Placeholder | Description |
 |---|---|
-| `<<FUNCTION-NAME>>` |  Name of the Lambda Function you want to monitor. |
+| `<<FUNCTION-NAME>>` |  Name of the Lambda function you want to monitor. |
 | `<<LAYERS>>` | A space-separated list of function layers to add to the function's execution environment. Specify each layer by its ARN, including the version.  For the ARN, see the [**Lambda extension versions** table](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#lambda-extension-versions).|
 | `<<ENV-VARS>>`  | Key-value pairs containing environment variables that are accessible from function code during execution. Should appear in the following format: `KeyName1=string,KeyName2=string`.  For a list of all the environment variables for the extension, see the [**Lambda environment variables** table](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#environment-variables).|
 
@@ -60,7 +61,7 @@ aws lambda update-function-configuration \
 ```
 
 **NOTE:** This command overwrites the existing function configuration. If you already have your own layers and environment variables for your function, include them in the list.
-  
+
 
 #### Deleting the extension
 
@@ -80,13 +81,13 @@ aws lambda update-function-configuration \
 ##### Add the extension to your Lambda Function
 
 1. In the Lambda Functions screen, choose the function you want to monitor.
-![Pick lambda function](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-1.jpg)
+   ![Pick lambda function](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-1.jpg)
 
 2. In the page for the function, scroll down to the `Layers` section and choose `Add Layer`.
-![Add layer](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-2.jpg)
+   ![Add layer](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-2.jpg)
 
 3. Select the `Specify an ARN` option, then choose the ARN of the extension with the region code that matches your Lambda Function region from the [**Lambda extension versions** table](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#lambda-extension-versions), and click the `Add` button.
-![Add ARN extension](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-3.jpg)
+   ![Add ARN extension](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lambda_extensions/lambda-x_1-3.jpg)
 
 ##### Configure the extension parameters
 
@@ -101,6 +102,65 @@ Run the function. It may take more than one run of the function for the logs to
 - To delete the **extension layer**: In your function page, go to the **layers** panel. Click `edit`, select the extension layer, and click `save`.
 - To delete the extension's **environment variables**: In your function page, select the `Configuration` tab, select `Environment variables`, click `edit`, and remove the variables that you added for the extension.
 
+### Parsing logs
+
+By default, the extension sends the logs as strings.  
+If your logs are formatted, and you wish to parse them to separate fields, the extension will use the [grok library](https://github.com/vjeantet/grok) to parse grok patterns.
+You can see all the pre-built grok patterns (for example `COMMONAPACHELOG` is already a known pattern in the library) [here](https://github.com/vjeantet/grok/tree/master/patterns).
+If you need to use a custom pattern, you can use the environment variables `GROK_PATTERNS` and `LOGS_FORMAT`.
+
+#### Example
+
+For logs that are formatted like this:
+
+```python
+%(app_name)s : %(message)s
+```
+
+we will use `cool app` as the `app_name` and the `message` will have strings containing whitespaces, letters and numbers.
+
+In Logz.io we wish to have `app_name`, `message` in their own fields, named `my_app` and `my_message`, respectively.
+To do so, we'll set the environment variables as follows:
+
+##### GROK_PATTERNS
+
+The `GROK_PATTERNS` variable should be in a JSON format.
+The key is used as the pattern name, and the value should be the regex that captures the pattern.  
+In our case, while `app_name` always stays `cool app`, we don't know what `message` will be, so we need to set `GROK_PATTERNS` as: `{"app_name":"cool app","message":".*"}`
+
+##### LOGS_FORMAT
+
+The `LOGS_FORMAT` variable will contain the same format as the logs, according to the pattern names that we used in `GROK_PATTERNS`.  
+The variable should be in a grok format for each pattern name: `${PATTERN_NAME:FIELD_NAME}` where `PATTERN_NAME` is the pattern name from `GROK_PATTERNS`, and `FIELD_NAME` is the name of the field you want the pattern to be parsed to.  
+**Note** that the `FIELD_NAME` cannot contain a dot (`.`) in it.
+In our case, we want `app_name` to appear under the field `my_app`, and `message` to appear under the field `my_message`. Since we know that the logs format is as mentioned above, we will set `LOGS_FORMAT` as: `%{app_name:my_app} : %{message:my_message}`.
+
+The logs that match the configuration above will appear in Logz.io with the fields `lambda.record.my_app`, `lambda.record.my_message`.  
+The log: `"cool app : The sky is so blue"`, will be parsed to look like this:
+```
+my_app: cool app
+my_message: The sky is so blue
+```
+
+To learn more about grok, read the [grok library](https://github.com/vjeantet/grok), [Logz.io's blog post](https://logz.io/blog/logstash-grok/), or watch [this introduction to grok video](https://logz.io/learn/introduction-to-the-logstash-grok/).
+
+### Nested fields
+
+As of v0.2.0 the extension can detect if a log is in a JSON format, and to parse the fields to appear as nested fields in the Logz.io app.
+For example, the following log:
+
+```
+{ "foo": "bar", "field2": "val2" }
+```
+
+Will appear under the fields:
+```
+message_nested.foo: bar
+message_nested.field2: val2
+```
+
+**Note:** The user must insert a valid JSON. Sending a dictionary or any key-value data structure that is not in a JSON format will cause the log to be sent as a string.
+
 ### Environment Variables
 
 | Name | Description |Required/Default|
@@ -110,11 +170,14 @@ Run the function. It may take more than one run of the function for the logs to
 | `LOGS_EXT_LOG_LEVEL` |  Log level of the extension. Can be set to one of the following: `debug`, `info`, `warn`, `error`, `fatal`, `panic`. |Default: `info` |
 | `ENABLE_EXTENSION_LOGS` |  Set to `true` if you wish the extension logs will be shipped to your Logz.io account. | Default: `false` |
 | `ENABLE_PLATFORM_LOGS` | The platform log captures runtime or execution environment errors. Set to `true` if you wish the platform logs will be shipped to your Logz.io account. | Default: `false` |
+| `GROK_PATTERNS` | Must be set with `LOGS_FORMAT`. Use this if you want to parse your logs into fields. A minified JSON list that contains the field name and the regex that will match the field. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - |
+| `LOGS_FORMAT` | Must be set with `GROK_PATTERNS`. Use this if you want to parse your logs into fields. The format in which the logs will appear, in accordance to grok conventions. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - |
 
 ### Lambda extension versions
 
 | Version | Supported Runtimes | AWS ARN |
 | --- | --- | --- |
+| 0.2.0 | `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Python 3.9`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Custom runtime` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:3` |
 | 0.1.0| `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Node.js 10.x`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Ruby 2.5`, `Custom runtime`| `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:2` |
 | 0.0.1 | `Python 3.7`, `Python 3.8` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:1` |
 
@@ -130,7 +193,7 @@ Run the function. It may take more than one run of the function for the logs to
 | Europe (Ireland) | `eu-west-1` | - |
 | Europe (Stockholm) | `eu-north-1` | - |
 | Asia Pacific (Sydney) | `ap-southeast-2` | Available from v0.1.0 |
-| Canada (Central) ) | `ca-central-1` | Available from v0.1.0 |
+| Canada (Central) | `ca-central-1` | Available from v0.1.0 |
 
 **NOTE:** If your AWS region is not in the list, please reach out to Logz.io's support or open an issue in this repo.
 
@@ -141,6 +204,10 @@ Note: the dependencies layer is deprecated.
 | 0.0.1 | `requests` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogsLibs:1` |
 
 ### Changelog:
+
+- **0.2.0**:
+  - Allow parsing log into fields. To learn more see [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section.
+  - Allow nested JSON within logs. To learn more see [nested fields](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#nested-fields) section.
 - **0.1.0**:
-    - **BREAKING CHANGES**: Written in Go, supports multiple runtimes. Compatible with the GA version of the Extensions API.
+  - **BREAKING CHANGES**: Written in Go, supports multiple runtimes. Compatible with the GA version of the Extensions API.
 - **0.0.1**: Initial release. Supports only python 3.7, python 3.8 runtimes. Compatible with the beta version of the Extensions API.

logzio-lambda-extensions-logs/agent/http.go

@@ -179,4 +179,4 @@ func getEventTypesToRegister() []logsapi.EventType {
 	}
 
 	return eventTypes
-}
+}

logzio-lambda-extensions-logs/agent/logger.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	maxBulkSizeBytes = 10 * 1024 * 1024  // 10 MB
+	maxBulkSizeBytes = 10 * 1024 * 1024 // 10 MB
 )
 
 var logger = log.WithFields(log.Fields{"agent": "logsApiAgent"})
@@ -35,7 +35,7 @@ func NewLogzioLogger() (*logzio.LogzioSender, error) {
 			logzio.SetInMemoryQueue(true),
 			logzio.SetDebug(os.Stdout),
 			logzio.SetinMemoryCapacity(maxBulkSizeBytes), //bytes
-			logzio.SetDrainDuration(time.Second * 5),
+			logzio.SetDrainDuration(time.Second*5),
 			logzio.SetDebug(os.Stdout),
 		)
 	} else {
@@ -45,7 +45,7 @@ func NewLogzioLogger() (*logzio.LogzioSender, error) {
 			logzio.SetInMemoryQueue(true),
 			logzio.SetDebug(os.Stdout),
 			logzio.SetinMemoryCapacity(maxBulkSizeBytes), //bytes
-			logzio.SetDrainDuration(time.Second * 5),
+			logzio.SetDrainDuration(time.Second*5),
 		)
 	}
 
@@ -54,4 +54,4 @@ func NewLogzioLogger() (*logzio.LogzioSender, error) {
 	}
 
 	return logzioLogger, nil
-}
+}

logzio-lambda-extensions-logs/extension/client.go

@@ -198,4 +198,4 @@ func (e *Client) ExitError(ctx context.Context, errorType string) (*StatusRespon
 		return nil, err
 	}
 	return &res, nil
-}
+}

logzio-lambda-extensions-logs/go.mod

@@ -4,7 +4,9 @@ go 1.15
 
 require (
 	github.com/golang-collections/go-datastructures v0.0.0-20150211160725-59788d5eb259
-	github.com/logzio/logzio-go v1.0.2
+	github.com/logzio/logzio-go v1.0.3
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.6.1
+	github.com/vjeantet/grok v1.0.1
 )

logzio-lambda-extensions-logs/logsapi/client.go

@@ -172,4 +172,4 @@ func httpPutWithHeaders(client *http.Client, url string, data []byte, headers *m
 	}
 
 	return resp, nil
-}
+}

logzio-lambda-extensions-logs/main.go

@@ -20,6 +20,7 @@ import (
 
 // INITIAL_QUEUE_SIZE is the initial size set for the synchronous logQueue
 const INITIAL_QUEUE_SIZE = 5
+
 func main() {
 	extensionName := path.Base(os.Args[0])
 	printPrefix := fmt.Sprintf("[%s]", extensionName)
@@ -167,4 +168,4 @@ func setLogLevel() {
 		// otherwise, we'll use the logger's default log level (info)
 		log.SetLevel(logLevel)
 	}
-}
+}