diff --git a/.idea/compiler.xml b/.idea/compiler.xml
new file mode 100644
index 0000000..96cc43e
--- /dev/null
+++ b/.idea/compiler.xml
@@ -0,0 +1,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml
new file mode 100644
index 0000000..e7bedf3
--- /dev/null
+++ b/.idea/copyright/profiles_settings.xml
@@ -0,0 +1,3 @@
+
+
+
\ No newline at end of file
diff --git a/.idea/logstash-filter-elasticsearch.iml b/.idea/logstash-filter-elasticsearch.iml
new file mode 100644
index 0000000..d6ebd48
--- /dev/null
+++ b/.idea/logstash-filter-elasticsearch.iml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..3add8ec
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,86 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Android
+
+
+ Android Lint
+
+
+ CorrectnessLintAndroid
+
+
+ GSPGrailsGroovy
+
+
+ General
+
+
+ GrailsGroovy
+
+
+ Groovy
+
+
+ LintAndroid
+
+
+ Maven
+
+
+ Plugin DevKit
+
+
+ XPath
+
+
+
+
+ Android
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Go 1.6
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..8f8700b
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..35eb1dd
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
new file mode 100644
index 0000000..75858c4
--- /dev/null
+++ b/.idea/workspace.xml
@@ -0,0 +1,1218 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1470047684133
+
+
+ 1470047684133
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb
index 7fe38d7..46535ec 100644
--- a/lib/logstash/filters/elasticsearch.rb
+++ b/lib/logstash/filters/elasticsearch.rb
@@ -2,7 +2,7 @@
require "logstash/filters/base"
require "logstash/namespace"
require_relative "elasticsearch/client"
-
+require "logstash/json"
# Search Elasticsearch for a previous log event and copy some fields from it
# into the current event.
@@ -44,7 +44,11 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
# Elasticsearch query string. Read the Elasticsearch query string documentation
# for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
- config :query, :validate => :string
+ config :query_string, :validate => :string
+
+ # File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
+ # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+ config :query_template, :validate => :string
# Comma-delimited list of `:` pairs that define the sort order
config :sort, :validate => :string, :default => "@timestamp:desc"
@@ -81,16 +85,36 @@ def register
:logger => @logger
}
@client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
+
+ #Load query if it exists
+ if @query_template
+ if File.zero?(@query_template)
+ raise "template is empty"
+ end
+ file = File.open(@query_template, "rb")
+ @query_dsl = file.read
+ end
+
end # def register
def filter(event)
begin
- query_str = event.sprintf(@query)
- params = { :q => query_str, :size => result_size, :index => @index }
- params[:sort] = @sort if @enable_sort
- results = @client.search(params)
+ params = {:index => @index }
+
+ if @query_dsl
+ query = LogStash::Json.load(event.sprintf(@query_dsl))
+ params[:body] = query
+ else
+ query = event.sprintf(@query_string)
+ params[:q] = query
+ params[:size] = result_size
+ params[:sort] = @sort if @enable_sort
+ end
+ @logger.info("Querying elasticsearch for lookup", :params => params)
+
+ results = @client.search(params)
@fields.each do |old_key, new_key|
if !results['hits']['hits'].empty?
set = []
@@ -101,7 +125,7 @@ def filter(event)
end
end
rescue => e
- @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query_str, :event => event, :error => e)
+ @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
@tag_on_failure.each{|tag| event.tag(tag)}
end
filter_matched(event)
diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb
index 43104c4..4b8d81b 100644
--- a/lib/logstash/filters/elasticsearch/client.rb
+++ b/lib/logstash/filters/elasticsearch/client.rb
@@ -2,6 +2,7 @@
require "elasticsearch"
require "base64"
+
module LogStash
module Filters
class ElasticsearchClient
@@ -30,6 +31,11 @@ def search(params)
@client.search(params)
end
+ def percolate(params)
+
+
+ end
+
end
end
end
diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec
index dfb159c..b20c5e1 100644
--- a/logstash-filter-elasticsearch.gemspec
+++ b/logstash-filter-elasticsearch.gemspec
@@ -1,7 +1,7 @@
Gem::Specification.new do |s|
s.name = 'logstash-filter-elasticsearch'
- s.version = '3.0.3'
+ s.version = '3.1.0'
s.licenses = ['Apache License (2.0)']
s.summary = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb
index cc3c246..190dc2f 100644
--- a/spec/filters/elasticsearch_spec.rb
+++ b/spec/filters/elasticsearch_spec.rb
@@ -19,7 +19,7 @@
let(:config) do
{
"hosts" => ["localhost:9200"],
- "query" => "response: 404",
+ "query_string" => "response: 404",
"fields" => [ ["response", "code"] ],
}
end
@@ -54,7 +54,7 @@
{
"index" => "foo*",
"hosts" => ["localhost:9200"],
- "query" => "response: 404",
+ "query_string" => "response: 404",
"fields" => [ ["response", "code"] ],
}
end
@@ -70,7 +70,7 @@
let(:config) do
{
"hosts" => ["localhost:9200"],
- "query" => "response: 404",
+ "query_string" => "response: 404",
"fields" => [ ["response", "code"] ],
"result_size" => 10
}
@@ -100,6 +100,28 @@
expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
end
end
+
+ context "testing a simple query template" do
+ let(:config) do
+ {
+ "hosts" => ["localhost:9200"],
+ "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
+ "fields" => [ ["response", "code"] ],
+ "result_size" => 1
+ }
+ end
+
+ let(:response) do
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
+ end
+
+ it "should enhance the current event with new data" do
+ plugin.filter(event)
+ expect(event.get("code")).to eq(404)
+ end
+
+ end
+
end
end
diff --git a/spec/filters/fixtures/query_template.json b/spec/filters/fixtures/query_template.json
new file mode 100644
index 0000000..0661d86
--- /dev/null
+++ b/spec/filters/fixtures/query_template.json
@@ -0,0 +1,7 @@
+{
+ "query": {
+ "query_string": {
+ "query": "response: 404"
+ }
+ }
+}
\ No newline at end of file
diff --git a/spec/filters/integration/elasticsearch_spec.rb b/spec/filters/integration/elasticsearch_spec.rb
index 4b7d1ce..33248a2 100644
--- a/spec/filters/integration/elasticsearch_spec.rb
+++ b/spec/filters/integration/elasticsearch_spec.rb
@@ -8,7 +8,7 @@
let(:config) do
{
"hosts" => ["localhost:9200"],
- "query" => "response: 404",
+ "query_string" => "response: 404",
"fields" => [ ["response", "code"] ],
}
end
@@ -29,7 +29,7 @@
let(:config) do
{
"hosts" => ["localhost:9200"],
- "query" => "response: 404",
+ "query_string" => "response: 404",
"fields" => [ ["response", "code"] ],
"result_size" => 10
}