diff --git a/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb index da90b24f63e..63176d4ff74 100644 --- a/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb +++ b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.rb @@ -80,7 +80,91 @@ def _nt_config r0 end + module NewlineOrEoi0 + end + + def _nt_newline_or_eoi + start_index = index + if node_cache[:newline_or_eoi].has_key?(index) + cached = node_cache[:newline_or_eoi][index] + if cached + node_cache[:newline_or_eoi][index] = cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + i1, s1 = index, [] + if (match_len = has_terminal?("\r", false, index)) + r3 = true + @index += match_len + else + terminal_parse_failure('"\\r"') + r3 = nil + end + if r3 + r2 = r3 + else + r2 = instantiate_node(SyntaxNode,input, index...index) + end + s1 << r2 + if r2 + if (match_len = has_terminal?("\n", false, index)) + r4 = true + @index += match_len + else + terminal_parse_failure('"\\n"') + r4 = nil + end + s1 << r4 + end + if s1.last + r1 = instantiate_node(SyntaxNode,input, i1...index, s1) + r1.extend(NewlineOrEoi0) + else + @index = i1 + r1 = nil + end + if r1 + r1 = SyntaxNode.new(input, (index-1)...index) if r1 == true + r0 = r1 + else + i5 = index + if index < input_length + r6 = true + @index += 1 + else + terminal_parse_failure("any character") + r6 = nil + end + if r6 + @index = i5 + r5 = nil + terminal_parse_failure("any character", true) + else + @terminal_failures.pop + @index = i5 + r5 = instantiate_node(SyntaxNode,input, index...index) + end + if r5 + r5 = SyntaxNode.new(input, (index-1)...index) if r5 == true + r0 = r5 + else + @index = i0 + r0 = nil + end + end + + node_cache[:newline_or_eoi][start_index] = r0 + + r0 + end + module Comment0 + def newline_or_eoi + elements[3] + end end def _nt_comment @@ -132,29 +216,8 @@ def _nt_comment r5 = instantiate_node(SyntaxNode,input, i5...index, s5) s1 << r5 if r5 - if (match_len = has_terminal?("\r", false, index)) - r8 = true - @index += match_len - else - terminal_parse_failure('"\\r"') - r8 = nil - end - if r8 - r7 = r8 - else - r7 = instantiate_node(SyntaxNode,input, index...index) - end + r7 = _nt_newline_or_eoi s1 << r7 - if r7 - if (match_len = has_terminal?("\n", false, index)) - r9 = true - @index += match_len - else - terminal_parse_failure('"\\n"') - r9 = nil - end - s1 << r9 - end end end end @@ -1674,7 +1737,7 @@ def _nt_hash end module Hashentries0 - def whitespace + def cs elements[0] end @@ -1708,7 +1771,7 @@ def _nt_hashentries s2, i2 = [], index loop do i3, s3 = index, [] - r4 = _nt_whitespace + r4 = _nt_cs s3 << r4 if r4 r5 = _nt_hashentry diff --git a/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop index 9af43f608ea..d7af0af60fb 100644 --- a/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop +++ b/logstash-core/lib/logstash/compiler/lscl/lscl_grammar.treetop @@ -6,8 +6,13 @@ grammar LogStashCompilerLSCLGrammar (cs plugin_section)* cs end + rule newline_or_eoi + # `!.` is a negative lookahead for 'anything', i.e. it matches at the end of input. + ("\r"? "\n") / !. + end + rule comment - (whitespace? "#" [^\r\n]* "\r"? "\n")+ + (whitespace? "#" [^\r\n]* newline_or_eoi)+ end rule cs @@ -114,7 +119,7 @@ grammar LogStashCompilerLSCLGrammar end rule hashentries - hashentry (whitespace hashentry)* + hashentry (cs hashentry)* end diff --git a/logstash-core/lib/logstash/config/grammar.rb b/logstash-core/lib/logstash/config/grammar.rb index 008878c8ca6..e7de111e818 100644 --- a/logstash-core/lib/logstash/config/grammar.rb +++ b/logstash-core/lib/logstash/config/grammar.rb @@ -104,7 +104,91 @@ def _nt_config r0 end + module NewlineOrEoi0 + end + + def _nt_newline_or_eoi + start_index = index + if node_cache[:newline_or_eoi].has_key?(index) + cached = node_cache[:newline_or_eoi][index] + if cached + node_cache[:newline_or_eoi][index] = cached = SyntaxNode.new(input, index...(index + 1)) if cached == true + @index = cached.interval.end + end + return cached + end + + i0 = index + i1, s1 = index, [] + if (match_len = has_terminal?("\r", false, index)) + r3 = true + @index += match_len + else + terminal_parse_failure('"\\r"') + r3 = nil + end + if r3 + r2 = r3 + else + r2 = instantiate_node(SyntaxNode,input, index...index) + end + s1 << r2 + if r2 + if (match_len = has_terminal?("\n", false, index)) + r4 = true + @index += match_len + else + terminal_parse_failure('"\\n"') + r4 = nil + end + s1 << r4 + end + if s1.last + r1 = instantiate_node(SyntaxNode,input, i1...index, s1) + r1.extend(NewlineOrEoi0) + else + @index = i1 + r1 = nil + end + if r1 + r1 = SyntaxNode.new(input, (index-1)...index) if r1 == true + r0 = r1 + else + i5 = index + if index < input_length + r6 = true + @index += 1 + else + terminal_parse_failure("any character") + r6 = nil + end + if r6 + @index = i5 + r5 = nil + terminal_parse_failure("any character", true) + else + @terminal_failures.pop + @index = i5 + r5 = instantiate_node(SyntaxNode,input, index...index) + end + if r5 + r5 = SyntaxNode.new(input, (index-1)...index) if r5 == true + r0 = r5 + else + @index = i0 + r0 = nil + end + end + + node_cache[:newline_or_eoi][start_index] = r0 + + r0 + end + module Comment0 + def newline_or_eoi + elements[3] + end end def _nt_comment @@ -156,29 +240,8 @@ def _nt_comment r5 = instantiate_node(SyntaxNode,input, i5...index, s5) s1 << r5 if r5 - if (match_len = has_terminal?("\r", false, index)) - r8 = true - @index += match_len - else - terminal_parse_failure('"\\r"') - r8 = nil - end - if r8 - r7 = r8 - else - r7 = instantiate_node(SyntaxNode,input, index...index) - end + r7 = _nt_newline_or_eoi s1 << r7 - if r7 - if (match_len = has_terminal?("\n", false, index)) - r9 = true - @index += match_len - else - terminal_parse_failure('"\\n"') - r9 = nil - end - s1 << r9 - end end end end @@ -1698,7 +1761,7 @@ def _nt_hash end module Hashentries0 - def whitespace + def cs elements[0] end @@ -1732,7 +1795,7 @@ def _nt_hashentries s2, i2 = [], index loop do i3, s3 = index, [] - r4 = _nt_whitespace + r4 = _nt_cs s3 << r4 if r4 r5 = _nt_hashentry diff --git a/logstash-core/lib/logstash/config/grammar.treetop b/logstash-core/lib/logstash/config/grammar.treetop index 16dcd20b691..e0497c83244 100644 --- a/logstash-core/lib/logstash/config/grammar.treetop +++ b/logstash-core/lib/logstash/config/grammar.treetop @@ -6,8 +6,13 @@ grammar LogStashConfig cs plugin_section cs (cs plugin_section)* cs end + rule newline_or_eoi + # `!.` is a negative lookahead for 'anything', i.e. it matches at the end of input. + ("\r"? "\n") / !. + end + rule comment - (whitespace? "#" [^\r\n]* "\r"? "\n")+ + (whitespace? "#" [^\r\n]* newline_or_eoi)+ end rule cs @@ -114,7 +119,7 @@ grammar LogStashConfig end rule hashentries - hashentry (whitespace hashentry)* + hashentry (cs hashentry)* end diff --git a/logstash-core/spec/logstash/config/config_ast_spec.rb b/logstash-core/spec/logstash/config/config_ast_spec.rb index b08152e6d9e..5851b11c8e3 100644 --- a/logstash-core/spec/logstash/config/config_ast_spec.rb +++ b/logstash-core/spec/logstash/config/config_ast_spec.rb @@ -211,6 +211,51 @@ end end end + + context "commentary in odd places" do + let(:config) { %q( + # early + input # bla + { # bla + plugin # bla + # bla + { # bla + # bla + arg # bla + => # bla + bareword # bla + # bla + hasharg => { + #preentry + "firstkey" => "firstvalue" + # bla + "middle" # bla + # bla + => # bla + # bla + "midvalue" # bla + # bla + "lastkey" => "lastvalue" + # bla + } + } # bla + # bla + } # bla + # comment ending in EOF: + ).strip } + subject { LogStashConfigParser.new } + def line_to_source(*args); end + def plugin(*args); end + it "hasn't got a trailing newline in the test config" do + # just to be clear that the last comment is supposed to end with EOF not \n + expect(config).not_to(end_with "\n") + end + it "should permit the comments" do + result = subject.parse(config) + expect(result).not_to(be_nil) + expect { eval(result.compile) }.not_to(raise_error) + end + end end context "when using two plugin sections of the same type" do