Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 8.17] Reimplement LogStash::String setting in Java (#16576) #16947

Closed
wants to merge 1 commit into from
Closed

[Backport 8.17] Reimplement LogStash::String setting in Java (#16576) #16947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions logstash-core/lib/logstash/environment.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ module Environment

[
Setting::Boolean.new("allow_superuser", true),
Setting::String.new("node.name", Socket.gethostname),
Setting::NullableString.new("path.config", nil, false),
Setting::SettingString.new("node.name", Socket.gethostname),
Setting::SettingNullableString.new("path.config", nil, false),
Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
Setting::NullableString.new("config.string", nil, false),
Setting::SettingNullableString.new("config.string", nil, false),
Setting::Modules.new("modules.cli", LogStash::Util::ModulesSettingArray, []),
Setting::Modules.new("modules", LogStash::Util::ModulesSettingArray, []),
Setting.new("modules_list", Array, []),
Expand All @@ -50,10 +50,10 @@ module Environment
Setting::Boolean.new("config.reload.automatic", false),
Setting::TimeValue.new("config.reload.interval", "3s"), # in seconds
Setting::Boolean.new("config.support_escapes", false),
Setting::String.new("config.field_reference.escape_style", "none", true, %w(none percent ampersand)),
Setting::String.new("event_api.tags.illegal", "rename", true, %w(rename warn)),
Setting::SettingString.new("config.field_reference.escape_style", "none", true, %w(none percent ampersand)),
Setting::SettingString.new("event_api.tags.illegal", "rename", true, %w(rename warn)),
Setting::Boolean.new("metric.collect", true),
Setting::String.new("pipeline.id", "main"),
Setting::SettingString.new("pipeline.id", "main"),
Setting::Boolean.new("pipeline.system", false),
Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
Setting::PositiveInteger.new("pipeline.batch.size", 125),
Expand All @@ -65,32 +65,32 @@ module Environment
Setting::CoercibleString.new("pipeline.ordered", "auto", true, ["auto", "true", "false"]),
Setting::CoercibleString.new("pipeline.ecs_compatibility", "v8", true, %w(disabled v1 v8)),
Setting.new("path.plugins", Array, []),
Setting::NullableString.new("interactive", nil, false),
Setting::SettingNullableString.new("interactive", nil, false),
Setting::Boolean.new("config.debug", false),
Setting::String.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
Setting::SettingString.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
Setting::Boolean.new("version", false),
Setting::Boolean.new("help", false),
Setting::Boolean.new("enable-local-plugin-development", false),
Setting::String.new("log.format", "plain", true, ["json", "plain"]),
Setting::SettingString.new("log.format", "plain", true, ["json", "plain"]),
Setting::Boolean.new("log.format.json.fix_duplicate_message_fields", false),
Setting::Boolean.new("api.enabled", true).with_deprecated_alias("http.enabled", "9"),
Setting::String.new("api.http.host", "127.0.0.1").with_deprecated_alias("http.host", "9"),
Setting::SettingString.new("api.http.host", "127.0.0.1").with_deprecated_alias("http.host", "9"),
Setting::PortRange.new("api.http.port", 9600..9700).with_deprecated_alias("http.port", "9"),
Setting::String.new("api.environment", "production").with_deprecated_alias("http.environment", "9"),
Setting::String.new("api.auth.type", "none", true, %w(none basic)),
Setting::String.new("api.auth.basic.username", nil, false).nullable,
Setting::SettingString.new("api.environment", "production").with_deprecated_alias("http.environment", "9"),
Setting::SettingString.new("api.auth.type", "none", true, %w(none basic)),
Setting::SettingString.new("api.auth.basic.username", nil, false).nullable,
Setting::Password.new("api.auth.basic.password", nil, false).nullable,
Setting::String.new("api.auth.basic.password_policy.mode", "WARN", true, %w[WARN ERROR]),
Setting::SettingString.new("api.auth.basic.password_policy.mode", "WARN", true, %w[WARN ERROR]),
Setting::Numeric.new("api.auth.basic.password_policy.length.minimum", 8),
Setting::String.new("api.auth.basic.password_policy.include.upper", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
Setting::String.new("api.auth.basic.password_policy.include.lower", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
Setting::String.new("api.auth.basic.password_policy.include.digit", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
Setting::String.new("api.auth.basic.password_policy.include.symbol", "OPTIONAL", true, %w[REQUIRED OPTIONAL]),
Setting::SettingString.new("api.auth.basic.password_policy.include.upper", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
Setting::SettingString.new("api.auth.basic.password_policy.include.lower", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
Setting::SettingString.new("api.auth.basic.password_policy.include.digit", "REQUIRED", true, %w[REQUIRED OPTIONAL]),
Setting::SettingString.new("api.auth.basic.password_policy.include.symbol", "OPTIONAL", true, %w[REQUIRED OPTIONAL]),
Setting::Boolean.new("api.ssl.enabled", false),
Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable,
Setting::Password.new("api.ssl.keystore.password", nil, false).nullable,
Setting::StringArray.new("api.ssl.supported_protocols", nil, true, %w[TLSv1 TLSv1.1 TLSv1.2 TLSv1.3]),
Setting::String.new("queue.type", "memory", true, ["persisted", "memory"]),
Setting::SettingString.new("queue.type", "memory", true, ["persisted", "memory"]),
Setting::Boolean.new("queue.drain", false),
Setting::Bytes.new("queue.page_capacity", "64mb"),
Setting::Bytes.new("queue.max_bytes", "1024mb"),
Expand All @@ -102,16 +102,16 @@ module Environment
Setting::Boolean.new("dead_letter_queue.enable", false),
Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"),
Setting::Numeric.new("dead_letter_queue.flush_interval", 5000),
Setting::String.new("dead_letter_queue.storage_policy", "drop_newer", true, ["drop_newer", "drop_older"]),
Setting::NullableString.new("dead_letter_queue.retain.age"), # example 5d
Setting::SettingString.new("dead_letter_queue.storage_policy", "drop_newer", true, ["drop_newer", "drop_older"]),
Setting::SettingNullableString.new("dead_letter_queue.retain.age"), # example 5d
Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
Setting::TimeValue.new("slowlog.threshold.info", "-1"),
Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
Setting::TimeValue.new("slowlog.threshold.trace", "-1"),
Setting::String.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"),
Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on
Setting::NullableString.new("monitoring.cluster_uuid"),
Setting::String.new("pipeline.buffer.type", nil, false, ["direct", "heap"])
Setting::SettingString.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"),
Setting::SettingString.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on
Setting::SettingNullableString.new("monitoring.cluster_uuid"),
Setting::SettingString.new("pipeline.buffer.type", nil, false, ["direct", "heap"])
# post_process
].each {|setting| SETTINGS.register(setting) }

Expand Down
25 changes: 5 additions & 20 deletions logstash-core/lib/logstash/settings.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,27 +523,10 @@ def validate(value)
@validator_class.validate(value)
end
end

java_import org.logstash.settings.SettingString

class String < Setting
def initialize(name, default = nil, strict = true, possible_strings = [])
@possible_strings = possible_strings
super(name, ::String, default, strict)
end

def validate(value)
super(value)
unless @possible_strings.empty? || @possible_strings.include?(value)
raise ArgumentError.new("Invalid value \"#{name}: #{value}\". Options are: #{@possible_strings.inspect}")
end
end
end

class NullableString < String
def validate(value)
return if value.nil?
super(value)
end
end
java_import org.logstash.settings.SettingNullableString

class Password < Coercible
def initialize(name, default = nil, strict = true)
Expand Down Expand Up @@ -824,6 +807,8 @@ def validate(value)
end
end

java_import org.logstash.settings.NullableSetting

# @see Setting#nullable
# @api internal
class Nullable < SimpleDelegator
Expand Down
4 changes: 2 additions & 2 deletions logstash-core/lib/logstash/util/settings_helper.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ module LogStash::Util::SettingsHelper
# The `path.settings` and `path.logs` can not be defined in "logstash/environment" since the `Environment::LOGSTASH_HOME` doesn't
# exist unless launched via "bootstrap/environment"
def self.pre_process
LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
LogStash::SETTINGS.register(LogStash::Setting::String.new("path.logs", ::File.join(LogStash::Environment::LOGSTASH_HOME, "logs")))
LogStash::SETTINGS.register(LogStash::Setting::SettingString.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
LogStash::SETTINGS.register(LogStash::Setting::SettingString.new("path.logs", ::File.join(LogStash::Environment::LOGSTASH_HOME, "logs")))
end

# Ensure that any settings are re-calculated after loading yaml
Expand Down
4 changes: 2 additions & 2 deletions logstash-core/spec/logstash/api/commands/default_metadata_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ def registerIfNot(setting)
before :all do
registerIfNot(LogStash::Setting::Boolean.new("xpack.monitoring.enabled", false))
registerIfNot(LogStash::Setting::ArrayCoercible.new("xpack.monitoring.elasticsearch.hosts", String, ["http://localhost:9200"]))
registerIfNot(LogStash::Setting::NullableString.new("xpack.monitoring.elasticsearch.username", "logstash_TEST system"))
registerIfNot(LogStash::Setting::NullableString.new("xpack.monitoring.elasticsearch.username", "logstash_TEST system"))
registerIfNot(LogStash::Setting::SettingNullableString.new("xpack.monitoring.elasticsearch.username", "logstash_TEST system"))
registerIfNot(LogStash::Setting::SettingNullableString.new("xpack.monitoring.elasticsearch.username", "logstash_TEST system"))
end

after :each do
Expand Down
4 changes: 2 additions & 2 deletions logstash-core/spec/logstash/queue_factory_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@
let(:settings_array) do
[
LogStash::Setting::WritableDirectory.new("path.queue", Stud::Temporary.pathname),
LogStash::Setting::String.new("queue.type", "memory", true, ["persisted", "memory"]),
LogStash::Setting::SettingString.new("queue.type", "memory", true, ["persisted", "memory"]),
LogStash::Setting::Bytes.new("queue.page_capacity", "8mb"),
LogStash::Setting::Bytes.new("queue.max_bytes", "64mb"),
LogStash::Setting::Numeric.new("queue.max_events", 0),
LogStash::Setting::Numeric.new("queue.checkpoint.acks", 1024),
LogStash::Setting::Numeric.new("queue.checkpoint.writes", 1024),
LogStash::Setting::Numeric.new("queue.checkpoint.interval", 1000),
LogStash::Setting::Boolean.new("queue.checkpoint.retry", false),
LogStash::Setting::String.new("pipeline.id", pipeline_id),
LogStash::Setting::SettingString.new("pipeline.id", pipeline_id),
LogStash::Setting::PositiveInteger.new("pipeline.batch.size", 125),
LogStash::Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum)
]
Expand Down
8 changes: 4 additions & 4 deletions logstash-core/spec/logstash/settings/nullable_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@

describe LogStash::Setting::Nullable do
let(:setting_name) { "this.that" }
let(:normal_setting) { LogStash::Setting::String.new(setting_name, nil, false, possible_strings) }
let(:normal_setting) { LogStash::Setting::SettingString.new(setting_name, nil, false, possible_strings) }
let(:possible_strings) { [] } # empty means any string passes

subject(:nullable_setting) { normal_setting.nullable }

it 'is a kind of Nullable' do
expect(nullable_setting).to be_a_kind_of(described_class)
expect(nullable_setting).to be_a_kind_of(LogStash::Setting::NullableSetting)
end

it "retains the wrapped setting's name" do
Expand Down Expand Up @@ -56,14 +56,14 @@
context 'to an invalid wrong-type value' do
let(:candidate_value) { 127 } # wrong type, expects String
it 'is an invalid setting' do
expect { nullable_setting.validate_value }.to raise_error(ArgumentError, a_string_including("Setting \"#{setting_name}\" must be a "))
expect { nullable_setting.validate_value }.to raise_error(java.lang.ClassCastException, a_string_including("class java.lang.Long cannot be cast to class java.lang.String"))
end
end
context 'to an invalid value not in the allow-list' do
let(:possible_strings) { %w(this that)}
let(:candidate_value) { "another" } # wrong type, expects String
it 'is an invalid setting' do
expect { nullable_setting.validate_value }.to raise_error(ArgumentError, a_string_including("Invalid value"))
expect { nullable_setting.validate_value }.to raise_error(java.lang.IllegalArgumentException, a_string_including("Invalid value"))
end
end
context 'to a valid value' do
Expand Down
55 changes: 44 additions & 11 deletions logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,44 @@
let(:default_value) { "DeFaUlT" }

let(:settings) { LogStash::Settings.new }
let(:canonical_setting) { LogStash::Setting::String.new(canonical_setting_name, default_value, true) }
let(:canonical_setting) { LogStash::Setting::SettingString.new(canonical_setting_name, default_value, true) }

let(:events) { [] }

before(:each) do
java_import org.apache.logging.log4j.LogManager
logger = LogManager.getLogger("org.logstash.settings.DeprecatedAlias")
deprecated_logger = LogManager.getLogger("org.logstash.deprecation.settings.DeprecatedAlias")

@custom_appender = CustomAppender.new(events).tap {|appender| appender.start }

java_import org.apache.logging.log4j.Level
logger.addAppender(@custom_appender)
deprecated_logger.addAppender(@custom_appender)
# had to set level after appending as it was "error" for some reason
logger.setLevel(Level::INFO)
deprecated_logger.setLevel(Level::INFO)

expect(@custom_appender.started?).to be_truthy

allow(LogStash::Settings).to receive(:logger).and_return(double("SettingsLogger").as_null_object)
allow(LogStash::Settings).to receive(:deprecation_logger).and_return(double("SettingsDeprecationLogger").as_null_object)

settings.register(canonical_setting.with_deprecated_alias(deprecated_setting_name))
end

after(:each) do
events.clear
java_import org.apache.logging.log4j.LogManager
logger = LogManager.getLogger("org.logstash.settings.DeprecatedAlias")
deprecated_logger = LogManager.getLogger("org.logstash.deprecation.settings.DeprecatedAlias")
# The Logger's AbstractConfiguration contains a cache of Appender, by class name. The cache is updated
# iff is absent, so to make subsequent add_appender effective we have cleanup on teardown, else the new
# appender instance is simply not used by the logger
logger.remove_appender(@custom_appender)
deprecated_logger.remove_appender(@custom_appender)
end

shared_examples '#validate_value success' do
context '#validate_value' do
it "returns without raising" do
Expand All @@ -57,6 +86,7 @@
it 'does not emit a deprecation warning' do
expect(LogStash::Settings.deprecation_logger).to_not receive(:deprecated).with(a_string_including(deprecated_setting_name))
settings.get_setting(deprecated_setting_name).observe_post_process
expect(events).to be_empty
end
end
end
Expand All @@ -66,22 +96,23 @@

before(:each) do
settings.set(deprecated_setting_name, value)
settings.get_setting(deprecated_setting_name).observe_post_process
end

it 'resolves to the value provided for the deprecated alias' do
expect(settings.get(canonical_setting_name)).to eq(value)
end

it 'logs a deprecation warning' do
expect(LogStash::Settings.deprecation_logger).to have_received(:deprecated).with(a_string_including(deprecated_setting_name))
expect(events[0]).to include(deprecated_setting_name)
end

include_examples '#validate_value success'

context "#observe_post_process" do
it 're-emits the deprecation warning' do
expect(LogStash::Settings.deprecation_logger).to receive(:deprecated).with(a_string_including(deprecated_setting_name))
settings.get_setting(deprecated_setting_name).observe_post_process
expect(events[0]).to include(deprecated_setting_name)
end
end

Expand Down Expand Up @@ -116,13 +147,14 @@
let(:old_value) { "iron man" }
let(:canonical_name) { "iron.setting" }
let(:deprecated_name) { "iron.oxide.setting" }
subject { LogStash::Setting::String.new(canonical_name, old_value, true) }
subject { LogStash::Setting::SettingString.new(canonical_name, old_value, true) }

it 'logs a deprecation warning with target remove version' do
settings.set(deprecated_name, new_value)
expect(LogStash::Settings.deprecation_logger).to have_received(:deprecated)
.with(a_string_including(deprecated_name))
.with(a_string_including("version 9"))
settings.get_setting(deprecated_name).observe_post_process
expect(events.length).to be 2
expect(events[1]).to include(deprecated_name)
expect(events[1]).to include("version 9")
end
end
describe "java boolean setting" do
Expand All @@ -149,15 +181,16 @@
end

it 'does not produce a relevant deprecation warning' do
expect(LogStash::Settings.deprecation_logger).to_not have_received(:deprecated).with(a_string_including(deprecated_setting_name))
settings.get_setting(deprecated_setting_name).observe_post_process
expect(events).to be_empty
end

include_examples '#validate_value success'

context "#observe_post_process" do
it 'does not emit a deprecation warning' do
expect(LogStash::Settings.deprecation_logger).to_not receive(:deprecated).with(a_string_including(deprecated_setting_name))
settings.get_setting(deprecated_setting_name).observe_post_process
expect(events).to be_empty
end
end
end
Expand All @@ -171,15 +204,15 @@
context '#validate_value' do
it "raises helpful exception" do
expect { settings.get_setting(canonical_setting_name).validate_value }
.to raise_exception(ArgumentError, a_string_including("Both `#{canonical_setting_name}` and its deprecated alias `#{deprecated_setting_name}` have been set. Please only set `#{canonical_setting_name}`"))
.to raise_exception(java.lang.IllegalStateException, a_string_including("Both `#{canonical_setting_name}` and its deprecated alias `#{deprecated_setting_name}` have been set. Please only set `#{canonical_setting_name}`"))
end
end
end

context 'Settings#get on deprecated alias' do
it 'produces a WARN-level message to the logger' do
expect(LogStash::Settings.logger).to receive(:warn).with(a_string_including "setting `#{canonical_setting_name}` has been queried by its deprecated alias `#{deprecated_setting_name}`")
settings.get(deprecated_setting_name)
expect(events[0]).to include("setting `#{canonical_setting_name}` has been queried by its deprecated alias `#{deprecated_setting_name}`")
end
end
end
Loading
Loading