From 23edf2227fa135c0511a96b981fbbfdef4f48f11 Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sat, 7 Dec 2024 14:44:29 -0800 Subject: [PATCH 1/3] Adding gems conditionally --- Gemfile | 11 +++++++++-- fluent-plugin-kafka.gemspec | 2 -- lib/fluent/plugin/out_rdkafka2.rb | 4 +--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index 0e3cf2b..45898ca 100644 --- a/Gemfile +++ b/Gemfile @@ -3,5 +3,12 @@ source 'https://rubygems.org' # Specify your gem's dependencies in fluent-plugin-kafka.gemspec gemspec -gem 'json', '2.7.3' # override of 2.7.4 version -gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] if ENV['USE_RDKAFKA'] +if ENV['USE_RDKAFKA'] + gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] + min_version = Gem::Version.new('0.16.0') + min_range_version = Gem::Version.new(ENV['RDKAFKA_VERSION_MIN_RANGE'].split(' ').last) + if min_range_version >= min_version + gem 'aws-msk-iam-sasl-signer' + gem 'json', '2.7.3' # override of 2.7.4 version + end +end diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index ad57ea9..3620ddb 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -19,8 +19,6 @@ Gem::Specification.new do |gem| gem.add_dependency "fluentd", [">= 0.10.58", "< 2"] gem.add_dependency 'ltsv' gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2' - gem.add_dependency 'rdkafka' - gem.add_dependency 'aws-msk-iam-sasl-signer' gem.add_development_dependency "rake", ">= 0.9.2" gem.add_development_dependency "test-unit", ">= 3.0.8" gem.add_development_dependency "test-unit-rr", "~> 1.0" diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index c76e3f5..9a09ce0 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -4,7 +4,6 @@ require 'fluent/plugin/kafka_plugin_util' require 'rdkafka' -require 'aws_msk_iam_sasl_signer' begin rdkafka_version = Gem::Version::create(Rdkafka::VERSION) @@ -16,6 +15,7 @@ require_relative 'rdkafka_patch/0_14_0' elsif rdkafka_version >= Gem::Version.create('0.16.0') require_relative 'rdkafka_patch/0_16_0' + require 'aws_msk_iam_sasl_signer' end rescue LoadError, NameError raise "unable to patch rdkafka." @@ -208,7 +208,6 @@ def add(level, message = nil) end end } - # HERE ----------------- Rdkafka::Config.logger = log config = build_config @rdkafka = Rdkafka::Config.new(config) @@ -217,7 +216,6 @@ def add(level, message = nil) if config[:"security.protocol"] == "sasl_ssl" && config[:"sasl.mechanisms"] == "OAUTHBEARER" Rdkafka::Config.oauthbearer_token_refresh_callback = method(:refresh_token) end - # HERE ----------------- if @default_topic.nil? if @use_default_for_unknown_topic || @use_default_for_unknown_partition_error From 3d63b3fdf3e91cbd3b5e4d5e357f7d4c928a1bbf Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sat, 7 Dec 2024 19:16:10 -0800 Subject: [PATCH 2/3] Moving required gems into gemspec --- Gemfile | 10 ---------- fluent-plugin-kafka.gemspec | 6 ++++++ lib/fluent/plugin/out_rdkafka2.rb | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index 45898ca..d5c272e 100644 --- a/Gemfile +++ b/Gemfile @@ -2,13 +2,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in fluent-plugin-kafka.gemspec gemspec - -if ENV['USE_RDKAFKA'] - gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] - min_version = Gem::Version.new('0.16.0') - min_range_version = Gem::Version.new(ENV['RDKAFKA_VERSION_MIN_RANGE'].split(' ').last) - if min_range_version >= min_version - gem 'aws-msk-iam-sasl-signer' - gem 'json', '2.7.3' # override of 2.7.4 version - end -end diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index 3620ddb..d6d3371 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -19,6 +19,12 @@ Gem::Specification.new do |gem| gem.add_dependency "fluentd", [">= 0.10.58", "< 2"] gem.add_dependency 'ltsv' gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2' + + if ENV['USE_RDKAFKA'] + gem.add_dependency 'rdkafka', [ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE']] + gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' + end + gem.add_development_dependency "rake", ">= 0.9.2" gem.add_development_dependency "test-unit", ">= 3.0.8" gem.add_development_dependency "test-unit-rr", "~> 1.0" diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index 9a09ce0..f3bfbdd 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -15,8 +15,8 @@ require_relative 'rdkafka_patch/0_14_0' elsif rdkafka_version >= Gem::Version.create('0.16.0') require_relative 'rdkafka_patch/0_16_0' - require 'aws_msk_iam_sasl_signer' end + require 'aws_msk_iam_sasl_signer' if rdkafka_version >= Gem::Version.create('0.16.0') rescue LoadError, NameError raise "unable to patch rdkafka." end From a0fac67e364d57520c80ffd4b78a0eef05bbe99a Mon Sep 17 00:00:00 2001 From: Andrea Singh Date: Sat, 7 Dec 2024 19:31:31 -0800 Subject: [PATCH 3/3] Make aws signer lib dependent on min ruby version --- fluent-plugin-kafka.gemspec | 4 +++- lib/fluent/plugin/out_rdkafka2.rb | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fluent-plugin-kafka.gemspec b/fluent-plugin-kafka.gemspec index d6d3371..c7700a4 100644 --- a/fluent-plugin-kafka.gemspec +++ b/fluent-plugin-kafka.gemspec @@ -22,7 +22,9 @@ Gem::Specification.new do |gem| if ENV['USE_RDKAFKA'] gem.add_dependency 'rdkafka', [ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE']] - gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' + if Gem::Version.new('3.0' >= Gem::Version.new(RUBY_VERSION) + gem.add_dependency 'aws-msk-iam-sasl-signer', '~> 0.1.1' + end end gem.add_development_dependency "rake", ">= 0.9.2" diff --git a/lib/fluent/plugin/out_rdkafka2.rb b/lib/fluent/plugin/out_rdkafka2.rb index f3bfbdd..6301e19 100644 --- a/lib/fluent/plugin/out_rdkafka2.rb +++ b/lib/fluent/plugin/out_rdkafka2.rb @@ -16,11 +16,14 @@ elsif rdkafka_version >= Gem::Version.create('0.16.0') require_relative 'rdkafka_patch/0_16_0' end - require 'aws_msk_iam_sasl_signer' if rdkafka_version >= Gem::Version.create('0.16.0') rescue LoadError, NameError raise "unable to patch rdkafka." end +if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.0') + require 'aws-msk-iam-sasl-signer' +end + module Fluent::Plugin class Fluent::Rdkafka2Output < Output Fluent::Plugin.register_output('rdkafka2', self)