Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use a better parser to parse line protocol. #176

Open
xuthus5 opened this issue Sep 11, 2024 · 0 comments
Open

Use a better parser to parse line protocol. #176

xuthus5 opened this issue Sep 11, 2024 · 0 comments

Comments

@xuthus5
Copy link
Member

xuthus5 commented Sep 11, 2024 •
edited
Loading

When I used the LineProtocolEncoder parser to parse the line protocol, I found that it was inefficient and error-prone. This is my benchmark test case:

// Creating an encoder cyclically
func BenchmarkLineProtocolEncoder_Encode(b *testing.B) {
	for i := 0; i < b.N; i++ {
		point := &Point{
			Measurement: "test",
			Precision:   PrecisionNanosecond,
			Tags: map[string]string{
				"a": "1",
				"b": "2",
				"c": "3",
				"d": "4",
				"e": "5",
				"f": "6",
				"g": "7",
				"h": "8",
				"i": "9",
				"j": "10",
				"k": "11",
				"l": "12",
				"m": "13",
				"n": "14",
			},
			Fields: map[string]interface{}{
				"f_string":  "this is a string",
				"f_int64":   int64(100),
				"f_float64": 3.14,
				"f_bool":    true,
				"json1":     "{\n	\"a\": 1,\n	\"b\": \"s\",\n	\"c\": true,\n	\"d\": 2.1\n}",
				"json2":     "{\n  \"deviceId\": 2166,\n  \"dataTime\": 1694482947700,\n  \"values\":{\n    \"temperature\": \"hhh\"\n  }\n}",
			},
			Time: time.Now(),
		}
		var buf = new(bytes.Buffer)
		encoder := NewLineProtocolEncoder(buf)
		err := encoder.Encode(point)
		if err != nil {
			b.Error(err)
		}
	}
}

output:

goos: windows
goarch: amd64
pkg: github.com/openGemini/opengemini-client-go/opengemini
cpu: AMD Ryzen 5 3600 6-Core Processor              
BenchmarkLineProtocolEncoder_Encode
BenchmarkLineProtocolEncoder_Encode-12    	  154539	      7413 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  154245	      7298 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  153619	      7524 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  154888	      7468 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  156118	      7378 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  146884	      7533 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  156384	      7484 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  151677	      7425 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  143944	      7528 ns/op
BenchmarkLineProtocolEncoder_Encode-12    	  151945	      7386 ns/op
// Define a global encoder
func BenchmarkLineProtocolEncoder_Encode1(b *testing.B) {
	buffer := new(strings.Builder)
	encoder := opengemini.NewLineProtocolEncoder(buffer)
	for i := 0; i < b.N; i++ {
		point := &opengemini.Point{
			Measurement: "test",
			Precision:   opengemini.PrecisionNanosecond,
			Tags: map[string]string{
				"a": "1",
				"b": "2",
				"c": "3",
				"d": "4",
				"e": "5",
				"f": "6",
				"g": "7",
				"h": "8",
				"i": "9",
				"j": "10",
				"k": "11",
				"l": "12",
				"m": "13",
				"n": "14",
			},
			Fields: map[string]interface{}{
				"f_string":  "this is a string",
				"f_int64":   int64(100),
				"f_float64": 3.14,
				"f_bool":    true,
				"json1":     "{\n	\"a\": 1,\n	\"b\": \"s\",\n	\"c\": true,\n	\"d\": 2.1\n}",
				"json2":     "{\n  \"deviceId\": 2166,\n  \"dataTime\": 1694482947700,\n  \"values\":{\n    \"temperature\": \"hhh\"\n  }\n}",
			},
			Time: time.Now(),
		}
		err := encoder.Encode(point)
		if err != nil {
			b.Error(err)
		}
		buffer.Reset()
	}
}

output:

goos: windows
goarch: amd64
pkg: line-protocol
cpu: Intel(R) Core(TM) i5-8600 CPU @ 3.10GHz
BenchmarkLineProtocol
BenchmarkLineProtocol-6   	  138871	      8001 ns/op
BenchmarkLineProtocol-6   	  149940	      7799 ns/op
BenchmarkLineProtocol-6   	  153727	      7664 ns/op
BenchmarkLineProtocol-6   	  131322	      7672 ns/op
BenchmarkLineProtocol-6   	  152527	      7688 ns/op
BenchmarkLineProtocol-6   	  136060	      8298 ns/op
BenchmarkLineProtocol-6   	  129442	      7898 ns/op
BenchmarkLineProtocol-6   	  151442	      7915 ns/op
BenchmarkLineProtocol-6   	  135346	      7798 ns/op
BenchmarkLineProtocol-6   	  154928	      7745 ns/op

use github.com/influxdata/line-protocol/v2

// Creating an encoder cyclically
func BenchmarkInfluxDB_LineProtocol_Encode(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var encoder lineprotocol.Encoder

		encoder.SetPrecision(lineprotocol.Nanosecond)
		encoder.StartLine("test")

		encoder.AddTag("a", "1")
		encoder.AddTag("b", "2")
		encoder.AddTag("c", "3")
		encoder.AddTag("d", "4")
		encoder.AddTag("e", "5")
		encoder.AddTag("f", "6")
		encoder.AddTag("g", "7")
		encoder.AddTag("h", "8")
		encoder.AddTag("i", "9")
		encoder.AddTag("j", "10")
		encoder.AddTag("k", "11")
		encoder.AddTag("l", "12")
		encoder.AddTag("m", "13")
		encoder.AddTag("n", "14")

		encoder.AddField("f_string", lineprotocol.MustNewValue("this is a string"))
		encoder.AddField("f_int64", lineprotocol.MustNewValue(int64(100)))
		encoder.AddField("f_float64", lineprotocol.MustNewValue(3.14))
		encoder.AddField("f_bool", lineprotocol.MustNewValue(true))
		encoder.AddField("json1", lineprotocol.MustNewValue("{\n	\"a\": 1,\n	\"b\": \"s\",\n	\"c\": true,\n	\"d\": 2.1\n}"))
		encoder.AddField("json2", lineprotocol.MustNewValue("{\n  \"deviceId\": 2166,\n  \"dataTime\": 1694482947700,\n  \"values\":{\n    \"temperature\": \"hhh\"\n  }\n}"))

		encoder.EndLine(time.Now())

		if err := encoder.Err(); err != nil {
			b.Error(err)
		}

		encoder.Bytes()
	}
}

output:

goos: windows
goarch: amd64
pkg: github.com/openGemini/opengemini-client-go/opengemini
cpu: AMD Ryzen 5 3600 6-Core Processor              
BenchmarkInfluxDB_LineProtocol_Encode
BenchmarkInfluxDB_LineProtocol_Encode-12    	  420286	      2643 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  461770	      2642 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  425738	      2689 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  411806	      2673 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  470767	      2648 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  450128	      2609 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  381250	      2665 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  418730	      2659 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  460572	      2669 ns/op
BenchmarkInfluxDB_LineProtocol_Encode-12    	  436774	      2671 ns/op
// Define a global encoder
func BenchmarkInfluxDB_LineProtocol_Encode1(b *testing.B) {
	var encoder lineprotocol.Encoder
	for i := 0; i < b.N; i++ {
		encoder.StartLine("test")
		encoder.SetPrecision(lineprotocol.Nanosecond)
		encoder.AddTag("a", "1")
		encoder.AddTag("b", "2")
		encoder.AddTag("c", "3")
		encoder.AddTag("d", "4")
		encoder.AddTag("e", "5")
		encoder.AddTag("f", "6")
		encoder.AddTag("g", "7")
		encoder.AddTag("h", "8")
		encoder.AddTag("i", "9")
		encoder.AddTag("j", "10")
		encoder.AddTag("k", "11")
		encoder.AddTag("l", "12")
		encoder.AddTag("m", "13")
		encoder.AddTag("n", "14")

		encoder.AddField("f_string", lineprotocol.MustNewValue("this is a string"))
		encoder.AddField("f_int64", lineprotocol.MustNewValue(int64(100)))
		encoder.AddField("f_float64", lineprotocol.MustNewValue(3.14))
		encoder.AddField("f_bool", lineprotocol.MustNewValue(true))
		encoder.AddField("json1", lineprotocol.MustNewValue("{\n	\"a\": 1,\n	\"b\": \"s\",\n	\"c\": true,\n	\"d\": 2.1\n}"))
		encoder.AddField("json2", lineprotocol.MustNewValue("{\n  \"deviceId\": 2166,\n  \"dataTime\": 1694482947700,\n  \"values\":{\n    \"temperature\": \"hhh\"\n  }\n}"))

		encoder.EndLine(time.Now())

		if err := encoder.Err(); err != nil {
			b.Error(err)
		}

		encoder.Reset()
	}
}

output:

goos: windows
goarch: amd64
pkg: line-protocol
cpu: Intel(R) Core(TM) i5-8600 CPU @ 3.10GHz
BenchmarkInfluxDbLineProtocol
BenchmarkInfluxDbLineProtocol-6   	  529756	      1911 ns/op
BenchmarkInfluxDbLineProtocol-6   	  626182	      1939 ns/op
BenchmarkInfluxDbLineProtocol-6   	  578482	      1924 ns/op
BenchmarkInfluxDbLineProtocol-6   	  546484	      2078 ns/op
BenchmarkInfluxDbLineProtocol-6   	  535740	      1932 ns/op
BenchmarkInfluxDbLineProtocol-6   	  571316	      1937 ns/op
BenchmarkInfluxDbLineProtocol-6   	  580036	      1924 ns/op
BenchmarkInfluxDbLineProtocol-6   	  571404	      1958 ns/op
BenchmarkInfluxDbLineProtocol-6   	  600693	      1936 ns/op
BenchmarkInfluxDbLineProtocol-6   	  556784	      1932 ns/op
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@xuthus5