diff --git a/Makefile b/Makefile index 61f16b646f..80d29b8bf3 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,8 @@ ALL_IMAGES := \ v1.3/debian-papertrail:v1.3.2-debian-papertrail,v1.3-debian-papertrail \ v1.3/debian-logzio:v1.3.2-debian-logzio,v1.3-debian-logzio,debian-logzio \ v1.3/debian-kafka:v1.3.2-debian-kafka,v1.3-debian-kafka \ - v1.3/debian-kinesis:v1.3.2-debian-kinesis,v1.3-debian-kinesis + v1.3/debian-kinesis:v1.3.2-debian-kinesis,v1.3-debian-kinesis \ + v1.3/debian-amqp:v1.3.2-debian-amqp,v1.3-debian-amqp # :,,,... diff --git a/README.md b/README.md index c43596b014..add33115b3 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ - `v1.3-debian-logzio` [docker-image/v1.3/debian-logzio/Dockerfile](docker-image/v1.3/debian-logzio/Dockerfile) - `v1.3-debian-kafka` [docker-image/v1.3/debian-kafka/Dockerfile](docker-image/v1.3/debian-kafka/Dockerfile) - `v1.3-debian-kinesis` [docker-image/v1.3/debian-kinesis/Dockerfile](docker-image/v1.3/debian-kinesis/Dockerfile) +- `v1.3-debian-amqp` [docker-image/v1.3/debian-amqp/Dockerfile](docker-image/v1.3/debian-amqp/Dockerfile) ### Alpine Linux (This is deprecated. Use Debian images instead) @@ -95,6 +96,7 @@ Some images are contributed by users. If you have a problem/question for followi - Amazon Kinesis : @shiftky - logz.io : @SaMnCo / @jamielennox - splunkhec: @FutureSharks +- amqp: @restorando ### References diff --git a/docker-image/v1.3/debian-amqp/.dockerignore b/docker-image/v1.3/debian-amqp/.dockerignore new file mode 100644 index 0000000000..921ed3fe9e --- /dev/null +++ b/docker-image/v1.3/debian-amqp/.dockerignore @@ -0,0 +1 @@ +**/*.gitkeep \ No newline at end of file diff --git a/docker-image/v1.3/debian-amqp/Dockerfile b/docker-image/v1.3/debian-amqp/Dockerfile new file mode 100644 index 0000000000..5c27b04259 --- /dev/null +++ b/docker-image/v1.3/debian-amqp/Dockerfile @@ -0,0 +1,50 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb + +FROM fluent/fluentd:v1.3.2-debian + +LABEL maintainer="Eduardo Silva " +USER root +WORKDIR /home/fluent +ENV PATH /fluentd/vendor/bundle/ruby/2.3.0/bin:$PATH +ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.3.0 +ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.3.0 +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +COPY Gemfile* /fluentd/ + RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \ + && apt-get update \ + && apt-get upgrade -y \ + && apt-get install \ + -y --no-install-recommends \ + $buildDeps net-tools libjemalloc1 \ + && gem install bundler --version 1.16.2 \ + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ + && SUDO_FORCE_REMOVE=yes \ + apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && gem sources --clear-all \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem + +# Copy configuration files +COPY ./conf/fluent.conf /fluentd/etc/ +COPY ./conf/systemd.conf /fluentd/etc/ +COPY ./conf/kubernetes.conf /fluentd/etc/ + +# Copy plugins +COPY plugins /fluentd/plugins/ +COPY entrypoint.sh /fluentd/entrypoint.sh + +# Environment variables +ENV FLUENTD_OPT="" +ENV FLUENTD_CONF="fluent.conf" + +# See https://packages.debian.org/stretch/amd64/libjemalloc1/filelist +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.1" + +# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib +ENTRYPOINT ["/fluentd/entrypoint.sh"] diff --git a/docker-image/v1.3/debian-amqp/Gemfile b/docker-image/v1.3/debian-amqp/Gemfile new file mode 100644 index 0000000000..68028b21bd --- /dev/null +++ b/docker-image/v1.3/debian-amqp/Gemfile @@ -0,0 +1,12 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "fluentd", "1.3.2" +gem "oj", "3.5.1" +gem "fluent-plugin-rewrite-tag-filter", "~> 2.1.0" +gem 'fluent-plugin-amqp2' +gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.1.4" +gem "ffi" +gem "fluent-plugin-systemd", "~> 1.0.1" diff --git a/docker-image/v1.3/debian-amqp/Gemfile.lock b/docker-image/v1.3/debian-amqp/Gemfile.lock new file mode 100644 index 0000000000..b4c88bc431 --- /dev/null +++ b/docker-image/v1.3/debian-amqp/Gemfile.lock @@ -0,0 +1,106 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (5.2.2) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) + amq-protocol (2.3.0) + bunny (2.13.0) + amq-protocol (~> 2.3, >= 2.3.0) + concurrent-ruby (1.1.4) + cool.io (1.5.3) + dig_rb (1.0.1) + domain_name (0.5.20180417) + unf (>= 0.0.5, < 1.0.0) + ffi (1.10.0) + fluent-config-regexp-type (1.0.0) + fluentd (> 1.0.0, < 2) + fluent-plugin-amqp2 (0.2.0) + bunny (>= 0.10.8) + fluentd (>= 0.10.0, < 2) + yajl-ruby (~> 1.0) + fluent-plugin-kubernetes_metadata_filter (2.1.6) + fluentd (>= 0.14.0, < 2) + kubeclient (~> 1.1.4) + lru_redux + fluent-plugin-rewrite-tag-filter (2.1.1) + fluent-config-regexp-type + fluentd (>= 0.14.2, < 2) + fluent-plugin-systemd (1.0.1) + fluentd (>= 0.14.11, < 2) + systemd-journal (~> 1.3.2) + fluentd (1.3.2) + cool.io (>= 1.4.5, < 2.0.0) + dig_rb (~> 1.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 0.7.0, < 2.0.0) + serverengine (>= 2.0.4, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (~> 1.0) + tzinfo-data (~> 1.0) + yajl-ruby (~> 1.0) + http (0.9.8) + addressable (~> 2.3) + http-cookie (~> 1.0) + http-form_data (~> 1.0.1) + http_parser.rb (~> 0.6.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http-form_data (1.0.3) + http_parser.rb (0.6.0) + i18n (1.5.1) + concurrent-ruby (~> 1.0) + kubeclient (1.1.4) + activesupport + http (= 0.9.8) + recursive-open-struct (= 1.0.0) + rest-client + lru_redux (1.1.0) + mime-types (3.2.2) + mime-types-data (~> 3.2015) + mime-types-data (3.2018.0812) + minitest (5.11.3) + msgpack (1.2.6) + netrc (0.11.0) + oj (3.5.1) + public_suffix (3.0.3) + recursive-open-struct (1.0.0) + rest-client (2.0.2) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + serverengine (2.1.0) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.3) + systemd-journal (1.3.3) + ffi (~> 1.9) + thread_safe (0.3.6) + tzinfo (1.2.5) + thread_safe (~> 0.1) + tzinfo-data (1.2018.9) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.5) + yajl-ruby (1.4.1) + +PLATFORMS + ruby + +DEPENDENCIES + ffi + fluent-plugin-amqp2 + fluent-plugin-kubernetes_metadata_filter (~> 2.1.4) + fluent-plugin-rewrite-tag-filter (~> 2.1.0) + fluent-plugin-systemd (~> 1.0.1) + fluentd (= 1.3.2) + oj (= 3.5.1) + +BUNDLED WITH + 1.17.3 diff --git a/docker-image/v1.3/debian-amqp/conf/fluent.conf b/docker-image/v1.3/debian-amqp/conf/fluent.conf new file mode 100644 index 0000000000..9bf4252cc2 --- /dev/null +++ b/docker-image/v1.3/debian-amqp/conf/fluent.conf @@ -0,0 +1,34 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb + +@include systemd.conf +@include kubernetes.conf +@include conf.d/*.conf + + + type amqp + + # Set broker host and port + host "#{ENV['FLUENT_AMQP_HOST']}" + port "#{ENV['FLUENT_AMQP_PORT'] || 5672 }" + + # Set user and password for authentication + user "#{ENV['FLUENT_AMQP_USER'] || guest }" + password "{ENV['FLUENT_AMQP_PASSWORD']}" + + # Configure amqp entities vhost, exchange id and type + vhost "#{ENV['FLUENT_AMQP_VHOST'] || '/'}" + exchange "#{ENV['FLUENT_AMQP_EXCHANGE'] || 'my_exchange'}" + exchange_type "#{ENV['FLUENT_AMQP_EXCHANGE_TYPE'] || 'topic'}" + # optionally set exchange durability - default is true. + exchange_durable "#{ENV['FLUENT_AMQP_EXCHANGE_DURABLE'] || 'true'}" + # If true, will not try to create the exchange - default is false. + passive "#{ENV['FLUENT_AMQP_EXCHANGE'] || 'false'}" + # optional - default is false. if true, only the payload will be sent. if false, data format is { "key" => tag, "timestamp" => time, "payload" => record }. + payload_only "#{ENV['FLUENT_AMQP_PAYLOAD_ONLY'] || 'false'}" + # optional - default is application/octet-stream. some amqp consumers will expect application/json. + content_type "#{ENV['FLUENT_AMQP_CONTENT_TYPE'] || 'application/octet-stream'}" + # the priority for the message - requires bunny >= 1.1.6 and rabbitmq >= 3.5 + priority "#{ENV['FLUENT_AMQP_PRIORITY'] || '0'}" + diff --git a/docker-image/v1.3/debian-amqp/conf/kubernetes.conf b/docker-image/v1.3/debian-amqp/conf/kubernetes.conf new file mode 100644 index 0000000000..b1b6cb5797 --- /dev/null +++ b/docker-image/v1.3/debian-amqp/conf/kubernetes.conf @@ -0,0 +1,192 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/kubernetes.conf.erb + + + @type null + + + + @type tail + @id in_tail_container_logs + path /var/log/containers/*.log + pos_file /var/log/fluentd-containers.log.pos + tag kubernetes.* + read_from_head true + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + + + @type tail + @id in_tail_minion + path /var/log/salt/minion + pos_file /var/log/fluentd-salt.pos + tag salt + + @type regexp + expression /^(? + + + + @type tail + @id in_tail_startupscript + path /var/log/startupscript.log + pos_file /var/log/fluentd-startupscript.log.pos + tag startupscript + + @type syslog + + + + + @type tail + @id in_tail_docker + path /var/log/docker.log + pos_file /var/log/fluentd-docker.log.pos + tag docker + + @type regexp + expression /^time="(? + + + + @type tail + @id in_tail_etcd + path /var/log/etcd.log + pos_file /var/log/fluentd-etcd.log.pos + tag etcd + + @type none + + + + + @type tail + @id in_tail_kubelet + multiline_flush_interval 5s + path /var/log/kubelet.log + pos_file /var/log/fluentd-kubelet.log.pos + tag kubelet + + @type kubernetes + + + + + @type tail + @id in_tail_kube_proxy + multiline_flush_interval 5s + path /var/log/kube-proxy.log + pos_file /var/log/fluentd-kube-proxy.log.pos + tag kube-proxy + + @type kubernetes + + + + + @type tail + @id in_tail_kube_apiserver + multiline_flush_interval 5s + path /var/log/kube-apiserver.log + pos_file /var/log/fluentd-kube-apiserver.log.pos + tag kube-apiserver + + @type kubernetes + + + + + @type tail + @id in_tail_kube_controller_manager + multiline_flush_interval 5s + path /var/log/kube-controller-manager.log + pos_file /var/log/fluentd-kube-controller-manager.log.pos + tag kube-controller-manager + + @type kubernetes + + + + + @type tail + @id in_tail_kube_scheduler + multiline_flush_interval 5s + path /var/log/kube-scheduler.log + pos_file /var/log/fluentd-kube-scheduler.log.pos + tag kube-scheduler + + @type kubernetes + + + + + @type tail + @id in_tail_rescheduler + multiline_flush_interval 5s + path /var/log/rescheduler.log + pos_file /var/log/fluentd-rescheduler.log.pos + tag rescheduler + + @type kubernetes + + + + + @type tail + @id in_tail_glbc + multiline_flush_interval 5s + path /var/log/glbc.log + pos_file /var/log/fluentd-glbc.log.pos + tag glbc + + @type kubernetes + + + + + @type tail + @id in_tail_cluster_autoscaler + multiline_flush_interval 5s + path /var/log/cluster-autoscaler.log + pos_file /var/log/fluentd-cluster-autoscaler.log.pos + tag cluster-autoscaler + + @type kubernetes + + + +# Example: +# 2017-02-09T00:15:57.992775796Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" ip="104.132.1.72" method="GET" user="kubecfg" as="" asgroups="" namespace="default" uri="/api/v1/namespaces/default/pods" +# 2017-02-09T00:15:57.993528822Z AUDIT: id="90c73c7c-97d6-4b65-9461-f94606ff825f" response="200" + + @type tail + @id in_tail_kube_apiserver_audit + multiline_flush_interval 5s + path /var/log/kubernetes/kube-apiserver-audit.log + pos_file /var/log/kube-apiserver-audit.log.pos + tag kube-apiserver-audit + + @type multiline + format_firstline /^\S+\s+AUDIT:/ + # Fields must be explicitly captured by name to be parsed into the record. + # Fields may not always be present, and order may change, so this just looks + # for a list of key="\"quoted\" value" pairs separated by spaces. + # Unknown fields are ignored. + # Note: We can't separate query/response lines as format1/format2 because + # they don't always come one after the other for a given query. + format1 /^(? + + + + @type kubernetes_metadata + @id filter_kube_metadata + + diff --git a/docker-image/v1.3/debian-amqp/conf/systemd.conf b/docker-image/v1.3/debian-amqp/conf/systemd.conf new file mode 100644 index 0000000000..cfc73eba80 --- /dev/null +++ b/docker-image/v1.3/debian-amqp/conf/systemd.conf @@ -0,0 +1,46 @@ + +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/systemd.conf.erb + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_kubelet + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-kubelet-cursor.json + + read_from_head true + tag kubelet + + +# Logs from docker-systemd + + @type systemd + @id in_systemd_docker + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-docker-cursor.json + + read_from_head true + tag docker.systemd + + +# Logs from systemd-journal for interesting services. + + @type systemd + @id in_systemd_bootkube + matches [{ "_SYSTEMD_UNIT": "bootkube.service" }] + + @type local + persistent true + path /var/log/fluentd-journald-bootkube-cursor.json + + read_from_head true + tag bootkube + + diff --git a/docker-image/v1.3/debian-amqp/entrypoint.sh b/docker-image/v1.3/debian-amqp/entrypoint.sh new file mode 100755 index 0000000000..a1f111acff --- /dev/null +++ b/docker-image/v1.3/debian-amqp/entrypoint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/dumb-init /bin/sh + + +exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} diff --git a/docker-image/v1.3/debian-amqp/hooks/post_push b/docker-image/v1.3/debian-amqp/hooks/post_push new file mode 100644 index 0000000000..33a2f2b509 --- /dev/null +++ b/docker-image/v1.3/debian-amqp/hooks/post_push @@ -0,0 +1,15 @@ +#!/bin/bash +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/post_push.erb + +set -e + +# Parse image name for repo name +tagStart=$(expr index "$IMAGE_NAME" :) +repoName=${IMAGE_NAME:0:tagStart-1} + +# Tag and push image for each additional tag +for tag in {v1.3.2-debian-amqp,v1.3-debian-amqp}; do + docker tag $IMAGE_NAME ${repoName}:${tag} + docker push ${repoName}:${tag} +done diff --git a/docker-image/v1.3/debian-amqp/plugins/.gitkeep b/docker-image/v1.3/debian-amqp/plugins/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker-image/v1.3/debian-amqp/plugins/parser_kubernetes.rb b/docker-image/v1.3/debian-amqp/plugins/parser_kubernetes.rb new file mode 100644 index 0000000000..0f593caafa --- /dev/null +++ b/docker-image/v1.3/debian-amqp/plugins/parser_kubernetes.rb @@ -0,0 +1,68 @@ +# +# Fluentd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The following Fluentd parser plugin, aims to simplify the parsing of multiline +# logs found in Kubernetes nodes. Since many log files shared the same format and +# in order to simplify the configuration, this plugin provides a 'kubernetes' format +# parser (built on top of MultilineParser). +# +# When tailing files, this 'kubernetes' format should be applied to the following +# log file sources: +# +# - /var/log/kubelet.log +# - /var/log/kube-proxy.log +# - /var/log/kube-apiserver.log +# - /var/log/kube-controller-manager.log +# - /var/log/kube-scheduler.log +# - /var/log/rescheduler.log +# - /var/log/glbc.log +# - /var/log/cluster-autoscaler.log +# +# Usage: +# +# ---- fluentd.conf ---- +# +# +# @type tail +# path ./kubelet.log +# read_from_head yes +# tag kubelet +# +# @type kubernetes +# +# +# +# ---- EOF --- + +require 'fluent/plugin/parser_regexp' + +module Fluent + module Plugin + class KubernetesParser < RegexpParser + Fluent::Plugin.register_parser("kubernetes", self) + + CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/} + CONF_FORMAT1 = %q{/^(?\w)(?