From 431e451630dc9c76da0a03928d0e513687bf1994 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 17 Mar 2021 14:51:58 +0100 Subject: [PATCH] Add k8s container logs collection (#36) Fixes https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/2536 --- charts/opentelemetry-collector/Chart.yaml | 4 +- charts/opentelemetry-collector/README.md | 75 ++++++++++++++++- .../templates/_config.tpl | 82 +++++++++++++++++++ .../templates/_pod.tpl | 16 ++++ charts/opentelemetry-collector/values.yaml | 14 +++- 5 files changed, 185 insertions(+), 6 deletions(-) diff --git a/charts/opentelemetry-collector/Chart.yaml b/charts/opentelemetry-collector/Chart.yaml index c5521213e..d3b600887 100644 --- a/charts/opentelemetry-collector/Chart.yaml +++ b/charts/opentelemetry-collector/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: opentelemetry-collector -version: 0.4.3 +version: 0.5.0 description: OpenTelemetry Collector Helm chart for Kubernetes type: application home: https://opentelemetry.io/ @@ -13,4 +13,4 @@ maintainers: - name: pjanotti - name: tigrannajaryan icon: https://opentelemetry.io/img/logos/opentelemetry-logo-nav.png -appVersion: 0.20.0 +appVersion: 0.22.0 diff --git a/charts/opentelemetry-collector/README.md b/charts/opentelemetry-collector/README.md index 22c0df471..6a17de9e0 100644 --- a/charts/opentelemetry-collector/README.md +++ b/charts/opentelemetry-collector/README.md @@ -25,9 +25,9 @@ helm install my-opentelemetry-collector open-telemetry/opentelemetry-collector ### Default configuration -By default this chart will deploy an OpenTelemetry Collector as daemonset with two pipelines (traces and metrics) +By default this chart will deploy an OpenTelemetry Collector as daemonset with three pipelines (logs, metrics and traces) and logging exporter enabled by default. Besides daemonset (agent), it can be also installed as standalone deployment. -Both modes can be enabled together, in that case metrics and traces will be flowing from agents to standalone collectors. +Both modes can be enabled together, in that case logs, metrics and traces will be flowing from agents to standalone collectors. *Example*: Install collector as a standalone deployment, and do not run it as an agent. @@ -42,6 +42,7 @@ By default collector has the following receivers enabled: - **metrics**: OTLP and prometheus. Prometheus is configured only for scraping collector's own metrics. - **traces**: OTLP, zipkin and jaeger (thrift and grpc). +- **logs**: OTLP (to enable container logs, see [Configuration for Kubernetes container logs](#configuration-for-kubernetes-container-logs)). There are two ways to configure collector pipelines, which can be used together as well. @@ -107,6 +108,76 @@ agentCollector: mountPropagation: HostToContainer ``` +### Configuration for Kubernetes container logs + +The collector can be used to collect logs sent to standard output by Kubernetes containers. +This feature is disabled by default. It has the following requirements: + +- It needs agent collector to be deployed, which means it will not work if only standalone collector is enabled. +- It requires the [contrib](https://github.com/open-telemetry/opentelemetry-collector-contrib) version +of the collector image. + +To enable this feature, set the `agentCollector.containerLogs.enabled` property to `true` and replace the collector image. +Here is an example `values.yaml`: + +```yaml +agentCollector: + containerLogs: + enabled: true + +image: + repository: otel/opentelemetry-collector-contrib + +command: + name: otelcontribcol +``` + +The way this feature works is it adds a `filelog` receiver on the `logs` pipeline. This receiver is preconfigured +to read the files where Kubernetes container runtime writes all containers' console output to. + +#### :warning: Warning: Risk of looping the exported logs back into the receiver, causing "log explosion" + +The container logs pipeline uses the `logging` console exporter by default. +Paired with the default `filelog` receiver that receives all containers' console output, +it is easy to accidentally feed the exported logs back into the receiver. + +Also note that using the `--log-level=debug` option for the `logging` exporter causes it to output +multiple lines per single received log, which when looped, would amplify the logs exponentially. + +To prevent the looping, the default configuration of the receiver excludes logs from the collector's containers. + +If you want to include the collector's logs, make sure to replace the `logging` exporter +with an exporter that does not send logs to collector's standard output. + +Here's an example `values.yaml` file that replaces the default `logging` exporter on the `logs` pipeline +with an `otlphttp` exporter that sends the container logs to `https://example.com:55681` endpoint. +It also clears the `filelog` receiver's `exclude` property, for collector logs to be included in the pipeline. + +```yaml +agentCollector: + containerLogs: + enabled: true + + configOverride: + exporters: + otlphttp: + endpoint: https://example.com:55681 + receivers: + filelog: + exclude: [] + service: + pipelines: + logs: + exporters: + - otlphttp + +image: + repository: otel/opentelemetry-collector-contrib + +command: + name: otelcontribcol +``` + ### Other configuration options The [values.yaml](./values.yaml) file contains information about all other configuration diff --git a/charts/opentelemetry-collector/templates/_config.tpl b/charts/opentelemetry-collector/templates/_config.tpl index 7937f4b64..f4c20e873 100644 --- a/charts/opentelemetry-collector/templates/_config.tpl +++ b/charts/opentelemetry-collector/templates/_config.tpl @@ -32,6 +32,7 @@ Build config file for agent OpenTelemetry Collector {{- $values := deepCopy .Values.agentCollector | mustMergeOverwrite (deepCopy .Values) }} {{- $data := dict "Values" $values | mustMergeOverwrite (deepCopy .) }} {{- $config := include "opentelemetry-collector.baseConfig" $data | fromYaml }} +{{- $config := include "opentelemetry-collector.agent.containerLogsConfig" $data | fromYaml | mustMergeOverwrite $config }} {{- $config := include "opentelemetry-collector.agentConfigOverride" $data | fromYaml | mustMergeOverwrite $config }} {{- .Values.agentCollector.configOverride | mustMergeOverwrite $config | toYaml }} {{- end }} @@ -115,9 +116,90 @@ exporters: {{- if .Values.standaloneCollector.enabled }} service: pipelines: + logs: + exporters: [otlp] metrics: exporters: [otlp] traces: exporters: [otlp] {{- end }} {{- end }} + +{{- define "opentelemetry-collector.agent.containerLogsConfig" -}} +{{- if .Values.agentCollector.containerLogs.enabled }} +receivers: + filelog: + include: [ /var/log/pods/*/*/*.log ] + # Exclude collector container's logs. The file format is /var/log/pods/__//.log + exclude: [ /var/log/pods/{{ .Release.Namespace }}_{{ include "opentelemetry-collector.fullname" . }}*_*/{{ .Chart.Name }}/*.log ] + start_at: beginning + include_file_path: true + include_file_name: false + operators: + # Find out which format is used by kubernetes + - type: router + id: get-format + routes: + - output: parser-docker + expr: '$$record matches "^\\{"' + - output: parser-crio + expr: '$$record matches "^[^ Z]+ "' + - output: parser-containerd + expr: '$$record matches "^[^ Z]+Z"' + # Parse CRI-O format + - type: regex_parser + id: parser-crio + regex: '^(?P