diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..96cc43e --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml new file mode 100644 index 0000000..e7bedf3 --- /dev/null +++ b/.idea/copyright/profiles_settings.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/.idea/logstash-filter-elasticsearch.iml b/.idea/logstash-filter-elasticsearch.iml new file mode 100644 index 0000000..d6ebd48 --- /dev/null +++ b/.idea/logstash-filter-elasticsearch.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..3add8ec --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + Android + + + Android Lint + + + CorrectnessLintAndroid + + + GSPGrailsGroovy + + + General + + + GrailsGroovy + + + Groovy + + + LintAndroid + + + Maven + + + Plugin DevKit + + + XPath + + + + + Android + + + + + + + + + + + + + + + + + + + + + Go 1.6 + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..8f8700b --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..75858c4 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,1218 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1470047684133 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb index 7fe38d7..46535ec 100644 --- a/lib/logstash/filters/elasticsearch.rb +++ b/lib/logstash/filters/elasticsearch.rb @@ -2,7 +2,7 @@ require "logstash/filters/base" require "logstash/namespace" require_relative "elasticsearch/client" - +require "logstash/json" # Search Elasticsearch for a previous log event and copy some fields from it # into the current event. @@ -44,7 +44,11 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base # Elasticsearch query string. Read the Elasticsearch query string documentation # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax - config :query, :validate => :string + config :query_string, :validate => :string + + # File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation + # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html + config :query_template, :validate => :string # Comma-delimited list of `:` pairs that define the sort order config :sort, :validate => :string, :default => "@timestamp:desc" @@ -81,16 +85,36 @@ def register :logger => @logger } @client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options) + + #Load query if it exists + if @query_template + if File.zero?(@query_template) + raise "template is empty" + end + file = File.open(@query_template, "rb") + @query_dsl = file.read + end + end # def register def filter(event) begin - query_str = event.sprintf(@query) - params = { :q => query_str, :size => result_size, :index => @index } - params[:sort] = @sort if @enable_sort - results = @client.search(params) + params = {:index => @index } + + if @query_dsl + query = LogStash::Json.load(event.sprintf(@query_dsl)) + params[:body] = query + else + query = event.sprintf(@query_string) + params[:q] = query + params[:size] = result_size + params[:sort] = @sort if @enable_sort + end + @logger.info("Querying elasticsearch for lookup", :params => params) + + results = @client.search(params) @fields.each do |old_key, new_key| if !results['hits']['hits'].empty? set = [] @@ -101,7 +125,7 @@ def filter(event) end end rescue => e - @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query_str, :event => event, :error => e) + @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e) @tag_on_failure.each{|tag| event.tag(tag)} end filter_matched(event) diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index 43104c4..4b8d81b 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -2,6 +2,7 @@ require "elasticsearch" require "base64" + module LogStash module Filters class ElasticsearchClient @@ -30,6 +31,11 @@ def search(params) @client.search(params) end + def percolate(params) + + + end + end end end diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index dfb159c..b20c5e1 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-elasticsearch' - s.version = '3.0.3' + s.version = '3.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Search elasticsearch for a previous log event and copy some fields from it into the current event" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index cc3c246..190dc2f 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -19,7 +19,7 @@ let(:config) do { "hosts" => ["localhost:9200"], - "query" => "response: 404", + "query_string" => "response: 404", "fields" => [ ["response", "code"] ], } end @@ -54,7 +54,7 @@ { "index" => "foo*", "hosts" => ["localhost:9200"], - "query" => "response: 404", + "query_string" => "response: 404", "fields" => [ ["response", "code"] ], } end @@ -70,7 +70,7 @@ let(:config) do { "hosts" => ["localhost:9200"], - "query" => "response: 404", + "query_string" => "response: 404", "fields" => [ ["response", "code"] ], "result_size" => 10 } @@ -100,6 +100,28 @@ expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure") end end + + context "testing a simple query template" do + let(:config) do + { + "hosts" => ["localhost:9200"], + "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"), + "fields" => [ ["response", "code"] ], + "result_size" => 1 + } + end + + let(:response) do + LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json"))) + end + + it "should enhance the current event with new data" do + plugin.filter(event) + expect(event.get("code")).to eq(404) + end + + end + end end diff --git a/spec/filters/fixtures/query_template.json b/spec/filters/fixtures/query_template.json new file mode 100644 index 0000000..0661d86 --- /dev/null +++ b/spec/filters/fixtures/query_template.json @@ -0,0 +1,7 @@ +{ + "query": { + "query_string": { + "query": "response: 404" + } + } +} \ No newline at end of file diff --git a/spec/filters/integration/elasticsearch_spec.rb b/spec/filters/integration/elasticsearch_spec.rb index 4b7d1ce..33248a2 100644 --- a/spec/filters/integration/elasticsearch_spec.rb +++ b/spec/filters/integration/elasticsearch_spec.rb @@ -8,7 +8,7 @@ let(:config) do { "hosts" => ["localhost:9200"], - "query" => "response: 404", + "query_string" => "response: 404", "fields" => [ ["response", "code"] ], } end @@ -29,7 +29,7 @@ let(:config) do { "hosts" => ["localhost:9200"], - "query" => "response: 404", + "query_string" => "response: 404", "fields" => [ ["response", "code"] ], "result_size" => 10 }