Support for AMQP

Makefile

@@ -38,7 +38,8 @@ ALL_IMAGES := \
 	v1.3/debian-papertrail:v1.3.2-debian-papertrail,v1.3-debian-papertrail \
 	v1.3/debian-logzio:v1.3.2-debian-logzio,v1.3-debian-logzio,debian-logzio \
 	v1.3/debian-kafka:v1.3.2-debian-kafka,v1.3-debian-kafka \
-	v1.3/debian-kinesis:v1.3.2-debian-kinesis,v1.3-debian-kinesis
+	v1.3/debian-kinesis:v1.3.2-debian-kinesis,v1.3-debian-kinesis \
+	v1.3/debian-amqp:v1.3.2-debian-amqp,v1.3-debian-amqp
 
 #	<Dockerfile>:<version>,<tag1>,<tag2>,...
 

README.md

@@ -37,6 +37,7 @@
 - `v1.3-debian-logzio` [docker-image/v1.3/debian-logzio/Dockerfile](docker-image/v1.3/debian-logzio/Dockerfile)
 - `v1.3-debian-kafka` [docker-image/v1.3/debian-kafka/Dockerfile](docker-image/v1.3/debian-kafka/Dockerfile)
 - `v1.3-debian-kinesis` [docker-image/v1.3/debian-kinesis/Dockerfile](docker-image/v1.3/debian-kinesis/Dockerfile)
+- `v1.3-debian-amqp` [docker-image/v1.3/debian-amqp/Dockerfile](docker-image/v1.3/debian-amqp/Dockerfile)
 
 ### Alpine Linux (This is deprecated. Use Debian images instead)
 
@@ -95,6 +96,7 @@ Some images are contributed by users. If you have a problem/question for followi
 - Amazon Kinesis : @shiftky
 - logz.io : @SaMnCo / @jamielennox
 - splunkhec: @FutureSharks
+- amqp: @restorando
 
 ### References
 

docker-image/v1.3/debian-amqp/.dockerignore

@@ -0,0 +1 @@
+**/*.gitkeep

docker-image/v1.3/debian-amqp/Dockerfile

@@ -0,0 +1,50 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb
+
+FROM fluent/fluentd:v1.3.2-debian
+
+LABEL maintainer="Eduardo Silva <[email protected]>"
+USER root
+WORKDIR /home/fluent
+ENV PATH /fluentd/vendor/bundle/ruby/2.3.0/bin:$PATH
+ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.3.0
+ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.3.0
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+COPY Gemfile* /fluentd/
+  RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \
+     && apt-get update \
+     && apt-get upgrade -y \
+     && apt-get install \
+     -y --no-install-recommends \
+     $buildDeps net-tools libjemalloc1 \
+    && gem install bundler --version 1.16.2 \
+    && bundle config silence_root_warning true \
+    && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
+    && SUDO_FORCE_REMOVE=yes \
+    apt-get purge -y --auto-remove \
+                  -o APT::AutoRemove::RecommendsImportant=false \
+                  $buildDeps \
+ && rm -rf /var/lib/apt/lists/* \
+    && gem sources --clear-all \
+    && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+# Copy configuration files
+COPY ./conf/fluent.conf /fluentd/etc/
+COPY ./conf/systemd.conf /fluentd/etc/
+COPY ./conf/kubernetes.conf /fluentd/etc/
+
+# Copy plugins
+COPY plugins /fluentd/plugins/
+COPY entrypoint.sh /fluentd/entrypoint.sh
+
+# Environment variables
+ENV FLUENTD_OPT=""
+ENV FLUENTD_CONF="fluent.conf"
+
+# See https://packages.debian.org/stretch/amd64/libjemalloc1/filelist
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.1"
+
+# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib
+ENTRYPOINT ["/fluentd/entrypoint.sh"]

docker-image/v1.3/debian-amqp/Gemfile

@@ -0,0 +1,12 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb
+
+source "https://rubygems.org"
+
+gem "fluentd", "1.3.2"
+gem "oj", "3.5.1"
+gem "fluent-plugin-rewrite-tag-filter", "~> 2.1.0"
+gem 'fluent-plugin-amqp2'
+gem "fluent-plugin-kubernetes_metadata_filter", "~> 2.1.4"
+gem "ffi"
+gem "fluent-plugin-systemd", "~> 1.0.1"

docker-image/v1.3/debian-amqp/Gemfile.lock

@@ -0,0 +1,106 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    amq-protocol (2.3.0)
+    bunny (2.13.0)
+      amq-protocol (~> 2.3, >= 2.3.0)
+    concurrent-ruby (1.1.4)
+    cool.io (1.5.3)
+    dig_rb (1.0.1)
+    domain_name (0.5.20180417)
+      unf (>= 0.0.5, < 1.0.0)
+    ffi (1.10.0)
+    fluent-config-regexp-type (1.0.0)
+      fluentd (> 1.0.0, < 2)
+    fluent-plugin-amqp2 (0.2.0)
+      bunny (>= 0.10.8)
+      fluentd (>= 0.10.0, < 2)
+      yajl-ruby (~> 1.0)
+    fluent-plugin-kubernetes_metadata_filter (2.1.6)
+      fluentd (>= 0.14.0, < 2)
+      kubeclient (~> 1.1.4)
+      lru_redux
+    fluent-plugin-rewrite-tag-filter (2.1.1)
+      fluent-config-regexp-type
+      fluentd (>= 0.14.2, < 2)
+    fluent-plugin-systemd (1.0.1)
+      fluentd (>= 0.14.11, < 2)
+      systemd-journal (~> 1.3.2)
+    fluentd (1.3.2)
+      cool.io (>= 1.4.5, < 2.0.0)
+      dig_rb (~> 1.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http (0.9.8)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http-form_data (1.0.3)
+    http_parser.rb (0.6.0)
+    i18n (1.5.1)
+      concurrent-ruby (~> 1.0)
+    kubeclient (1.1.4)
+      activesupport
+      http (= 0.9.8)
+      recursive-open-struct (= 1.0.0)
+      rest-client
+    lru_redux (1.1.0)
+    mime-types (3.2.2)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2018.0812)
+    minitest (5.11.3)
+    msgpack (1.2.6)
+    netrc (0.11.0)
+    oj (3.5.1)
+    public_suffix (3.0.3)
+    recursive-open-struct (1.0.0)
+    rest-client (2.0.2)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    serverengine (2.1.0)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.3)
+    systemd-journal (1.3.3)
+      ffi (~> 1.9)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2018.9)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.5)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ffi
+  fluent-plugin-amqp2
+  fluent-plugin-kubernetes_metadata_filter (~> 2.1.4)
+  fluent-plugin-rewrite-tag-filter (~> 2.1.0)
+  fluent-plugin-systemd (~> 1.0.1)
+  fluentd (= 1.3.2)
+  oj (= 3.5.1)
+
+BUNDLED WITH
+   1.17.3

docker-image/v1.3/debian-amqp/conf/fluent.conf

@@ -0,0 +1,34 @@
+
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/conf/fluent.conf.erb
+
+@include systemd.conf
+@include kubernetes.conf
+@include conf.d/*.conf
+
+<match **>
+  @type amqp
+  @id out_amqp
+
+  host "#{ENV['FLUENT_AMQP_HOST']}"
+  port "#{ENV['FLUENT_AMQP_PORT'] || 5672 }"
+
+  # Set user and password for authentication
+  user "#{ENV['FLUENT_AMQP_USER'] || guest }"
+  password "#{ENV['FLUENT_AMQP_PASSWORD']}"
+
+  # Configure amqp entities vhost, exchange id and type
+  vhost "#{ENV['FLUENT_AMQP_VHOST'] || '/'}"
+  exchange "#{ENV['FLUENT_AMQP_EXCHANGE'] || 'my_exchange'}"
+  exchange_type "#{ENV['FLUENT_AMQP_EXCHANGE_TYPE'] || 'exchange_type'}"
+  # optionally set exchange durability - default is true.
+  exchange_durable "#{ENV['FLUENT_AMQP_EXCHANGE_DURABLE'] || 'true'}"
+  # If true, will not try to create the exchange - default is false.
+  passive "#{ENV['FLUENT_AMQP_PASSIVE'] || 'false'}"
+  # optional - default is false. if true, only the payload will be sent. if false, data format is { "key" => tag, "timestamp" => time, "payload" => record }.
+  payload_only "#{ENV['FLUENT_AMQP_PAYLOAD_ONLY'] || 'false'}"
+  # optional - default is application/octet-stream. some amqp consumers will expect application/json.
+  content_type "#{ENV['FLUENT_AMQP_CONTENT_TYPE'] || 'application/octet-stream'}"
+  # the priority for the message - requires bunny >= 1.1.6 and rabbitmq >= 3.5
+  priority "#{ENV['FLUENT_AMQP_PRIORITY'] || '0'}"
+</match>