From 5e14c74c46ed3a3a11c7ece037c8023ef717aae1 Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Wed, 26 Jul 2023 14:30:35 +0300 Subject: [PATCH] feat: junit xml (#4202) * feat: junit xml * feat: xml jtl format support * fix: condition tuning * fix: test fake site * fix: unit tests * Revert "fix: test fake site" This reverts commit 03a32424e7883ca71b349fa26a91506d026fc2a2. * Revert "Revert "fix: test fake site"" This reverts commit 26975688649aa6a588933ff2d7db4c6e7f6eb84b. * Revert "Revert "Revert "fix: test fake site""" This reverts commit 00fe2355364b76470bc4135b58af3e6720f0220e. --- contrib/executor/jmeter/pkg/parser/jtl.go | 15 ++-- .../executor/jmeter/pkg/parser/jtl_test.go | 18 +++-- contrib/executor/jmeter/pkg/parser/xjtl.go | 36 +++++++++ .../executor/jmeter/pkg/parser/xjtl_test.go | 76 +++++++++++++++++++ contrib/executor/jmeter/pkg/parser/xunit.go | 54 +++++++++++++ contrib/executor/jmeter/pkg/runner/runner.go | 66 ++++++++++++++-- .../executor/jmeter/pkg/runner/runner_test.go | 4 +- 7 files changed, 249 insertions(+), 20 deletions(-) create mode 100644 contrib/executor/jmeter/pkg/parser/xjtl.go create mode 100644 contrib/executor/jmeter/pkg/parser/xjtl_test.go create mode 100644 contrib/executor/jmeter/pkg/parser/xunit.go diff --git a/contrib/executor/jmeter/pkg/parser/jtl.go b/contrib/executor/jmeter/pkg/parser/jtl.go index 5cb9091b3d7..b0744782d33 100644 --- a/contrib/executor/jmeter/pkg/parser/jtl.go +++ b/contrib/executor/jmeter/pkg/parser/jtl.go @@ -4,7 +4,6 @@ import ( "encoding/csv" "errors" "io" - "log" "strconv" "time" ) @@ -23,8 +22,12 @@ type Result struct { Duration time.Duration } -func Parse(reader io.Reader) (results Results) { - res := CSVToMap(reader) +func ParseCSV(reader io.Reader) (results Results, err error) { + res, err := CSVToMap(reader) + if err != nil { + return + } + for _, r := range res { result := MapElementToResult(r) results.Results = append(results.Results, result) @@ -51,7 +54,7 @@ func MapElementToResult(in map[string]string) Result { } // CSVToMap takes a reader and returns an array of dictionaries, using the header row as the keys -func CSVToMap(reader io.Reader) []map[string]string { +func CSVToMap(reader io.Reader) ([]map[string]string, error) { r := csv.NewReader(reader) rows := []map[string]string{} var header []string @@ -61,7 +64,7 @@ func CSVToMap(reader io.Reader) []map[string]string { break } if err != nil { - log.Fatal(err) + return nil, err } if header == nil { header = record @@ -73,5 +76,5 @@ func CSVToMap(reader io.Reader) []map[string]string { rows = append(rows, dict) } } - return rows + return rows, nil } diff --git a/contrib/executor/jmeter/pkg/parser/jtl_test.go b/contrib/executor/jmeter/pkg/parser/jtl_test.go index 45fab5ae74d..5ac5d9cb217 100644 --- a/contrib/executor/jmeter/pkg/parser/jtl_test.go +++ b/contrib/executor/jmeter/pkg/parser/jtl_test.go @@ -9,36 +9,40 @@ import ( ) const ( - failedTest = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect + failedCSV = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect 1667462863619,441,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,385,0,297 1667462929945,390,HTTP Request,200,OK,Thread Group 1-1,text,false,Test failed: text expected to contain /SOME_NONExisting_String/,66428,109,1,1,https://testkube.io,339,0,249 1667462945507,344,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,294,0,207 ` - successTest = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect + successCSV = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect 1667463814102,382,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,326,0,235 1667463836936,365,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,309,0,222 1667463838447,362,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,309,0,219 ` ) -func TestParse(t *testing.T) { +func TestParseCSV(t *testing.T) { t.Parallel() - t.Run("parse failed test", func(t *testing.T) { + t.Run("parse CSV failed test", func(t *testing.T) { t.Parallel() - results := Parse(strings.NewReader(failedTest)) + results, err := ParseCSV(strings.NewReader(failedCSV)) + + assert.NoError(t, err) assert.True(t, results.HasError) assert.Equal(t, 3, len(results.Results)) assert.Equal(t, "Test failed: text expected to contain /SOME_NONExisting_String/", results.Results[1].Error) assert.Equal(t, "Test failed: text expected to contain /SOME_NONExisting_String/", results.LastErrorMessage) }) - t.Run("parse success test", func(t *testing.T) { + t.Run("parse CSV success test", func(t *testing.T) { t.Parallel() - results := Parse(strings.NewReader(successTest)) + results, err := ParseCSV(strings.NewReader(successCSV)) + + assert.NoError(t, err) assert.False(t, results.HasError) assert.Equal(t, "200", results.Results[0].ResponseCode) assert.Equal(t, time.Millisecond*382, results.Results[0].Duration) diff --git a/contrib/executor/jmeter/pkg/parser/xjtl.go b/contrib/executor/jmeter/pkg/parser/xjtl.go new file mode 100644 index 00000000000..7a39f620930 --- /dev/null +++ b/contrib/executor/jmeter/pkg/parser/xjtl.go @@ -0,0 +1,36 @@ +package parser + +import ( + "encoding/xml" +) + +// TestResults is a root element of junit xml report +type TestResults struct { + XMLName xml.Name `xml:"testResults"` + HTTPSamples []HTTPSample `xml:"httpSample,omitempty"` +} + +// HTTPSample is http sample details +type HTTPSample struct { + XMLName xml.Name `xml:"httpSample"` + Time int `xml:"t,attr"` + Success bool `xml:"s,attr"` + Label string `xml:"lb,attr"` + ResponseCode string `xml:"rc,attr"` + AssertionResult *AssertionResult `xml:"assertionResult"` +} + +// AssertionResult contains assertion +type AssertionResult struct { + XMLName xml.Name `xml:"assertionResult"` + Name string `xml:"name"` + Failure bool `xml:"failure"` + Error bool `xml:"error"` + FailureMessage string `xml:"failureMessage"` +} + +func ParseXML(data []byte) (results TestResults, err error) { + err = xml.Unmarshal(data, &results) + + return results, err +} diff --git a/contrib/executor/jmeter/pkg/parser/xjtl_test.go b/contrib/executor/jmeter/pkg/parser/xjtl_test.go new file mode 100644 index 00000000000..0fd770ac792 --- /dev/null +++ b/contrib/executor/jmeter/pkg/parser/xjtl_test.go @@ -0,0 +1,76 @@ +package parser + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + badXML = ` +12345 +` + + successXML = ` + + + + Response Assertion + false + false + + + +` + + failedXML = ` + + + + Response Assertion + true + false + Test failed: code expected to equal / ****** received : [[[Non HTTP response code: java.net.UnknownHostException]]] ****** comparison: [[[200 ]]] / + + + +` +) + +func TestParseXML(t *testing.T) { + t.Parallel() + + t.Run("parse XML success test", func(t *testing.T) { + t.Parallel() + + results, err := ParseXML([]byte(successXML)) + + assert.NoError(t, err) + assert.Equal(t, 1, len(results.HTTPSamples)) + assert.True(t, results.HTTPSamples[0].Success) + assert.Equal(t, 1259, results.HTTPSamples[0].Time) + assert.Equal(t, "Testkube - HTTP Request", results.HTTPSamples[0].Label) + assert.Equal(t, "Response Assertion", results.HTTPSamples[0].AssertionResult.Name) + }) + + t.Run("parse XML failed test", func(t *testing.T) { + t.Parallel() + + results, err := ParseXML([]byte(failedXML)) + + assert.NoError(t, err) + assert.Equal(t, 1, len(results.HTTPSamples)) + assert.False(t, results.HTTPSamples[0].Success) + assert.Equal(t, 51, results.HTTPSamples[0].Time) + assert.Equal(t, "Testkube - HTTP Request", results.HTTPSamples[0].Label) + assert.Equal(t, "Test failed: code expected to equal / ****** received : [[[Non HTTP response code: java.net.UnknownHostException]]] ****** comparison: [[[200 ]]] /", results.HTTPSamples[0].AssertionResult.FailureMessage) + }) + + t.Run("parse bad XML", func(t *testing.T) { + t.Parallel() + + _, err := ParseXML([]byte(badXML)) + + assert.EqualError(t, err, "EOF") + }) +} diff --git a/contrib/executor/jmeter/pkg/parser/xunit.go b/contrib/executor/jmeter/pkg/parser/xunit.go new file mode 100644 index 00000000000..0e8159a49fd --- /dev/null +++ b/contrib/executor/jmeter/pkg/parser/xunit.go @@ -0,0 +1,54 @@ +package parser + +import ( + "encoding/xml" +) + +// Testsuites is a root element of junit report +type Testsuites struct { + XMLName xml.Name `xml:"testsuites"` + Testsuites []Testsuite `xml:"testsuite,omitempty"` + Name string `xml:"name,attr,omitempty"` + Tests int `xml:"tests,attr,omitempty"` + Failures int `xml:"failures,attr,omitempty"` + Errors int `xml:"errors,attr,omitempty"` + Skipped int `xml:"skipped,attr,omitempty"` + Assertions int `xml:"assertions,attr,omitempty"` + Time float32 `xml:"time,attr,omitempty"` + Timestamp string `xml:"timestamp,attr,omitempty"` +} + +// Testsuite contains testsuite definition +type Testsuite struct { + XMLName xml.Name `xml:"testsuite"` + Testcases []Testcase `xml:"testcase"` + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr,omitempty"` + Assertions int `xml:"assertions,attr,omitempty"` + Time float32 `xml:"time,attr"` + Timestamp string `xml:"timestamp,attr,omitempty"` + File string `xml:"file,attr,omitempty"` +} + +// TestResult represents the result of a testcase +type TestResult struct { + Message string `xml:"message,attr"` + Type string `xml:"type,attr,omitempty"` +} + +// Testcase define a testcase +type Testcase struct { + XMLName xml.Name `xml:"testcase"` + Name string `xml:"name,attr"` + ClassName string `xml:"classname,attr"` + Assertions int `xml:"assertions,attr,omitempty"` + Time float32 `xml:"time,attr,omitempty"` + File string `xml:"file,attr,omitempty"` + Line int `xml:"line,attr,omitempty"` + Skipped *TestResult `xml:"skipped,omitempty"` + Failure *TestResult `xml:"failure,omitempty"` + Error *TestResult `xml:"error,omitempty"` +} diff --git a/contrib/executor/jmeter/pkg/runner/runner.go b/contrib/executor/jmeter/pkg/runner/runner.go index 86bf4850021..8c00a0a9c9c 100644 --- a/contrib/executor/jmeter/pkg/runner/runner.go +++ b/contrib/executor/jmeter/pkg/runner/runner.go @@ -174,8 +174,26 @@ func (r *JMeterRunner) Run(ctx context.Context, execution testkube.Execution) (r return *result.WithErrors(errors.Errorf("getting jtl report error: %v", err)), nil } - results := parser.Parse(f) - executionResult := MapResultsToExecutionResults(out, results) + results, err := parser.ParseCSV(f) + f.Close() + + var executionResult testkube.ExecutionResult + if err != nil { + data, err := os.ReadFile(jtlPath) + if err != nil { + return *result.WithErrors(errors.Errorf("getting jtl report error: %v", err)), nil + } + + testResults, err := parser.ParseXML(data) + if err != nil { + return *result.WithErrors(errors.Errorf("parsing jtl report error: %v", err)), nil + } + + executionResult = MapTestResultsToExecutionResults(out, testResults) + } else { + executionResult = MapResultsToExecutionResults(out, results) + } + output.PrintLogf("%s Mapped JMeter results to Execution Results...", ui.IconCheckMark) // scrape artifacts first even if there are errors above @@ -223,10 +241,10 @@ func MapResultsToExecutionResults(out []byte, results parser.Results) (result te testkube.ExecutionStepResult{ Name: r.Label, Duration: r.Duration.String(), - Status: MapStatus(r), + Status: MapResultStatus(r), AssertionResults: []testkube.AssertionResult{{ Name: r.Label, - Status: MapStatus(r), + Status: MapResultStatus(r), }}, }) } @@ -234,7 +252,37 @@ func MapResultsToExecutionResults(out []byte, results parser.Results) (result te return result } -func MapStatus(result parser.Result) string { +func MapTestResultsToExecutionResults(out []byte, results parser.TestResults) (result testkube.ExecutionResult) { + result.Status = testkube.ExecutionStatusPassed + + result.Output = string(out) + result.OutputType = "text/plain" + + for _, r := range results.HTTPSamples { + if !r.Success { + result.Status = testkube.ExecutionStatusFailed + if r.AssertionResult != nil { + result.ErrorMessage = r.AssertionResult.FailureMessage + } + } + + result.Steps = append( + result.Steps, + testkube.ExecutionStepResult{ + Name: r.Label, + Duration: fmt.Sprintf("%dms", r.Time), + Status: MapTestResultStatus(r.Success), + AssertionResults: []testkube.AssertionResult{{ + Name: r.Label, + Status: MapTestResultStatus(r.Success), + }}, + }) + } + + return result +} + +func MapResultStatus(result parser.Result) string { if result.Success { return string(testkube.PASSED_ExecutionStatus) } @@ -242,6 +290,14 @@ func MapStatus(result parser.Result) string { return string(testkube.FAILED_ExecutionStatus) } +func MapTestResultStatus(success bool) string { + if success { + return string(testkube.PASSED_ExecutionStatus) + } + + return string(testkube.FAILED_ExecutionStatus) +} + // GetType returns runner type func (r *JMeterRunner) GetType() runner.Type { return runner.TypeMain diff --git a/contrib/executor/jmeter/pkg/runner/runner_test.go b/contrib/executor/jmeter/pkg/runner/runner_test.go index 09897cad203..72a96a0f9d6 100644 --- a/contrib/executor/jmeter/pkg/runner/runner_test.go +++ b/contrib/executor/jmeter/pkg/runner/runner_test.go @@ -15,14 +15,14 @@ func TestMapStatus(t *testing.T) { t.Run("should map valid status", func(t *testing.T) { t.Parallel() - out := MapStatus(parser.Result{Success: false}) + out := MapResultStatus(parser.Result{Success: false}) assert.Equal(t, out, string(testkube.FAILED_ExecutionStatus)) }) t.Run("should map invalid status", func(t *testing.T) { t.Parallel() - out := MapStatus(parser.Result{Success: true}) + out := MapResultStatus(parser.Result{Success: true}) assert.Equal(t, out, string(testkube.PASSED_ExecutionStatus)) })