feat: junit xml (#4202)

* feat: junit xml * feat: xml jtl format support * fix: condition tuning * fix: test fake site * fix: unit tests * Revert "fix: test fake site" This reverts commit 03a3242. * Revert "Revert "fix: test fake site"" This reverts commit 2697568. * Revert "Revert "Revert "fix: test fake site""" This reverts commit 00fe235.
kubeshop · Jul 26, 2023 · 5e14c74 · 5e14c74
1 parent 60ab21d
commit 5e14c74
Show file tree

Hide file tree

Showing 7 changed files with 249 additions and 20 deletions.
diff --git a/contrib/executor/jmeter/pkg/parser/jtl.go b/contrib/executor/jmeter/pkg/parser/jtl.go
@@ -4,7 +4,6 @@ import (
 	"encoding/csv"
 	"errors"
 	"io"
-	"log"
 	"strconv"
 	"time"
 )
@@ -23,8 +22,12 @@ type Result struct {
 	Duration     time.Duration
 }
 
-func Parse(reader io.Reader) (results Results) {
-	res := CSVToMap(reader)
+func ParseCSV(reader io.Reader) (results Results, err error) {
+	res, err := CSVToMap(reader)
+	if err != nil {
+		return
+	}
+
 	for _, r := range res {
 		result := MapElementToResult(r)
 		results.Results = append(results.Results, result)
@@ -51,7 +54,7 @@ func MapElementToResult(in map[string]string) Result {
 }
 
 // CSVToMap takes a reader and returns an array of dictionaries, using the header row as the keys
-func CSVToMap(reader io.Reader) []map[string]string {
+func CSVToMap(reader io.Reader) ([]map[string]string, error) {
 	r := csv.NewReader(reader)
 	rows := []map[string]string{}
 	var header []string
@@ -61,7 +64,7 @@ func CSVToMap(reader io.Reader) []map[string]string {
 			break
 		}
 		if err != nil {
-			log.Fatal(err)
+			return nil, err
 		}
 		if header == nil {
 			header = record
@@ -73,5 +76,5 @@ func CSVToMap(reader io.Reader) []map[string]string {
 			rows = append(rows, dict)
 		}
 	}
-	return rows
+	return rows, nil
 }
diff --git a/contrib/executor/jmeter/pkg/parser/jtl_test.go b/contrib/executor/jmeter/pkg/parser/jtl_test.go
@@ -9,36 +9,40 @@ import (
 )
 
 const (
-	failedTest = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect
+	failedCSV = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect
 1667462863619,441,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,385,0,297
 1667462929945,390,HTTP Request,200,OK,Thread Group 1-1,text,false,Test failed: text expected to contain /SOME_NONExisting_String/,66428,109,1,1,https://testkube.io,339,0,249
 1667462945507,344,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,294,0,207
 `
 
-	successTest = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect
+	successCSV = `timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect
 1667463814102,382,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,326,0,235
 1667463836936,365,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,309,0,222
 1667463838447,362,HTTP Request,200,OK,Thread Group 1-1,text,true,,66428,109,1,1,https://testkube.io,309,0,219
 `
 )
 
-func TestParse(t *testing.T) {
+func TestParseCSV(t *testing.T) {
 	t.Parallel()
 
-	t.Run("parse failed test", func(t *testing.T) {
+	t.Run("parse CSV failed test", func(t *testing.T) {
 		t.Parallel()
 
-		results := Parse(strings.NewReader(failedTest))
+		results, err := ParseCSV(strings.NewReader(failedCSV))
+
+		assert.NoError(t, err)
 		assert.True(t, results.HasError)
 		assert.Equal(t, 3, len(results.Results))
 		assert.Equal(t, "Test failed: text expected to contain /SOME_NONExisting_String/", results.Results[1].Error)
 		assert.Equal(t, "Test failed: text expected to contain /SOME_NONExisting_String/", results.LastErrorMessage)
 	})
 
-	t.Run("parse success test", func(t *testing.T) {
+	t.Run("parse CSV success test", func(t *testing.T) {
 		t.Parallel()
 
-		results := Parse(strings.NewReader(successTest))
+		results, err := ParseCSV(strings.NewReader(successCSV))
+
+		assert.NoError(t, err)
 		assert.False(t, results.HasError)
 		assert.Equal(t, "200", results.Results[0].ResponseCode)
 		assert.Equal(t, time.Millisecond*382, results.Results[0].Duration)

diff --git a/contrib/executor/jmeter/pkg/parser/xjtl.go b/contrib/executor/jmeter/pkg/parser/xjtl.go
@@ -0,0 +1,36 @@
+package parser
+
+import (
+	"encoding/xml"
+)
+
+// TestResults is a root element of junit xml report
+type TestResults struct {
+	XMLName     xml.Name     `xml:"testResults"`
+	HTTPSamples []HTTPSample `xml:"httpSample,omitempty"`
+}
+
+// HTTPSample is http sample details
+type HTTPSample struct {
+	XMLName         xml.Name         `xml:"httpSample"`
+	Time            int              `xml:"t,attr"`
+	Success         bool             `xml:"s,attr"`
+	Label           string           `xml:"lb,attr"`
+	ResponseCode    string           `xml:"rc,attr"`
+	AssertionResult *AssertionResult `xml:"assertionResult"`
+}
+
+// AssertionResult contains assertion
+type AssertionResult struct {
+	XMLName        xml.Name `xml:"assertionResult"`
+	Name           string   `xml:"name"`
+	Failure        bool     `xml:"failure"`
+	Error          bool     `xml:"error"`
+	FailureMessage string   `xml:"failureMessage"`
+}
+
+func ParseXML(data []byte) (results TestResults, err error) {
+	err = xml.Unmarshal(data, &results)
+
+	return results, err
+}
diff --git a/contrib/executor/jmeter/pkg/parser/xjtl_test.go b/contrib/executor/jmeter/pkg/parser/xjtl_test.go
@@ -0,0 +1,76 @@
+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	badXML = `
+12345
+`
+
+	successXML = `
+<testResults version="1.2">
+	<httpSample t="1259" it="0" lt="124" ct="80" ts="1690288938130" s="true" lb="Testkube - HTTP Request" rc="200" rm="OK" tn="Thread Group 1-1" dt="text" by="65222" sby="366" ng="1" na="1">
+		<assertionResult>
+			<name>Response Assertion</name>
+			<failure>false</failure>
+			<error>false</error>
+		</assertionResult>
+	</httpSample>
+</testResults>
+`
+
+	failedXML = `
+<testResults version="1.2">
+	<httpSample t="51" it="0" lt="0" ct="51" ts="1690366471701" s="false" lb="Testkube - HTTP Request" rc="Non HTTP response code: java.net.UnknownHostException" rm="Non HTTP response message: testkube.fakeshop.io: Name does not resolve" tn="Thread Group 1-1" dt="text" by="2327" sby="0" ng="1" na="1">
+		<assertionResult>
+			<name>Response Assertion</name>
+			<failure>true</failure>
+			<error>false</error>
+			<failureMessage>Test failed: code expected to equal / ****** received : [[[Non HTTP response code: java.net.UnknownHostException]]] ****** comparison: [[[200 ]]] /</failureMessage>
+		</assertionResult>
+	</httpSample>
+</testResults>
+`
+)
+
+func TestParseXML(t *testing.T) {
+	t.Parallel()
+
+	t.Run("parse XML success test", func(t *testing.T) {
+		t.Parallel()
+
+		results, err := ParseXML([]byte(successXML))
+
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(results.HTTPSamples))
+		assert.True(t, results.HTTPSamples[0].Success)
+		assert.Equal(t, 1259, results.HTTPSamples[0].Time)
+		assert.Equal(t, "Testkube - HTTP Request", results.HTTPSamples[0].Label)
+		assert.Equal(t, "Response Assertion", results.HTTPSamples[0].AssertionResult.Name)
+	})
+
+	t.Run("parse XML failed test", func(t *testing.T) {
+		t.Parallel()
+
+		results, err := ParseXML([]byte(failedXML))
+
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(results.HTTPSamples))
+		assert.False(t, results.HTTPSamples[0].Success)
+		assert.Equal(t, 51, results.HTTPSamples[0].Time)
+		assert.Equal(t, "Testkube - HTTP Request", results.HTTPSamples[0].Label)
+		assert.Equal(t, "Test failed: code expected to equal / ****** received : [[[Non HTTP response code: java.net.UnknownHostException]]] ****** comparison: [[[200 ]]] /", results.HTTPSamples[0].AssertionResult.FailureMessage)
+	})
+
+	t.Run("parse bad XML", func(t *testing.T) {
+		t.Parallel()
+
+		_, err := ParseXML([]byte(badXML))
+
+		assert.EqualError(t, err, "EOF")
+	})
+}
diff --git a/contrib/executor/jmeter/pkg/parser/xunit.go b/contrib/executor/jmeter/pkg/parser/xunit.go
@@ -0,0 +1,54 @@
+package parser
+
+import (
+	"encoding/xml"
+)
+
+// Testsuites is a root element of junit report
+type Testsuites struct {
+	XMLName    xml.Name    `xml:"testsuites"`
+	Testsuites []Testsuite `xml:"testsuite,omitempty"`
+	Name       string      `xml:"name,attr,omitempty"`
+	Tests      int         `xml:"tests,attr,omitempty"`
+	Failures   int         `xml:"failures,attr,omitempty"`
+	Errors     int         `xml:"errors,attr,omitempty"`
+	Skipped    int         `xml:"skipped,attr,omitempty"`
+	Assertions int         `xml:"assertions,attr,omitempty"`
+	Time       float32     `xml:"time,attr,omitempty"`
+	Timestamp  string      `xml:"timestamp,attr,omitempty"`
+}
+
+// Testsuite contains testsuite definition
+type Testsuite struct {
+	XMLName    xml.Name   `xml:"testsuite"`
+	Testcases  []Testcase `xml:"testcase"`
+	Name       string     `xml:"name,attr"`
+	Tests      int        `xml:"tests,attr"`
+	Failures   int        `xml:"failures,attr"`
+	Errors     int        `xml:"errors,attr"`
+	Skipped    int        `xml:"skipped,attr,omitempty"`
+	Assertions int        `xml:"assertions,attr,omitempty"`
+	Time       float32    `xml:"time,attr"`
+	Timestamp  string     `xml:"timestamp,attr,omitempty"`
+	File       string     `xml:"file,attr,omitempty"`
+}
+
+// TestResult represents the result of a testcase
+type TestResult struct {
+	Message string `xml:"message,attr"`
+	Type    string `xml:"type,attr,omitempty"`
+}
+
+// Testcase define a testcase
+type Testcase struct {
+	XMLName    xml.Name    `xml:"testcase"`
+	Name       string      `xml:"name,attr"`
+	ClassName  string      `xml:"classname,attr"`
+	Assertions int         `xml:"assertions,attr,omitempty"`
+	Time       float32     `xml:"time,attr,omitempty"`
+	File       string      `xml:"file,attr,omitempty"`
+	Line       int         `xml:"line,attr,omitempty"`
+	Skipped    *TestResult `xml:"skipped,omitempty"`
+	Failure    *TestResult `xml:"failure,omitempty"`
+	Error      *TestResult `xml:"error,omitempty"`
+}
diff --git a/contrib/executor/jmeter/pkg/runner/runner.go b/contrib/executor/jmeter/pkg/runner/runner.go
@@ -174,8 +174,26 @@ func (r *JMeterRunner) Run(ctx context.Context, execution testkube.Execution) (r
 		return *result.WithErrors(errors.Errorf("getting jtl report error: %v", err)), nil
 	}
 
-	results := parser.Parse(f)
-	executionResult := MapResultsToExecutionResults(out, results)
+	results, err := parser.ParseCSV(f)
+	f.Close()
+
+	var executionResult testkube.ExecutionResult
+	if err != nil {
+		data, err := os.ReadFile(jtlPath)
+		if err != nil {
+			return *result.WithErrors(errors.Errorf("getting jtl report error: %v", err)), nil
+		}
+
+		testResults, err := parser.ParseXML(data)
+		if err != nil {
+			return *result.WithErrors(errors.Errorf("parsing jtl report error: %v", err)), nil
+		}
+
+		executionResult = MapTestResultsToExecutionResults(out, testResults)
+	} else {
+		executionResult = MapResultsToExecutionResults(out, results)
+	}
+
 	output.PrintLogf("%s Mapped JMeter results to Execution Results...", ui.IconCheckMark)
 
 	// scrape artifacts first even if there are errors above
@@ -223,25 +241,63 @@ func MapResultsToExecutionResults(out []byte, results parser.Results) (result te
 			testkube.ExecutionStepResult{
 				Name:     r.Label,
 				Duration: r.Duration.String(),
-				Status:   MapStatus(r),
+				Status:   MapResultStatus(r),
 				AssertionResults: []testkube.AssertionResult{{
 					Name:   r.Label,
-					Status: MapStatus(r),
+					Status: MapResultStatus(r),
 				}},
 			})
 	}
 
 	return result
 }
 
-func MapStatus(result parser.Result) string {
+func MapTestResultsToExecutionResults(out []byte, results parser.TestResults) (result testkube.ExecutionResult) {
+	result.Status = testkube.ExecutionStatusPassed
+
+	result.Output = string(out)
+	result.OutputType = "text/plain"
+
+	for _, r := range results.HTTPSamples {
+		if !r.Success {
+			result.Status = testkube.ExecutionStatusFailed
+			if r.AssertionResult != nil {
+				result.ErrorMessage = r.AssertionResult.FailureMessage
+			}
+		}
+
+		result.Steps = append(
+			result.Steps,
+			testkube.ExecutionStepResult{
+				Name:     r.Label,
+				Duration: fmt.Sprintf("%dms", r.Time),
+				Status:   MapTestResultStatus(r.Success),
+				AssertionResults: []testkube.AssertionResult{{
+					Name:   r.Label,
+					Status: MapTestResultStatus(r.Success),
+				}},
+			})
+	}
+
+	return result
+}
+
+func MapResultStatus(result parser.Result) string {
 	if result.Success {
 		return string(testkube.PASSED_ExecutionStatus)
 	}
 
 	return string(testkube.FAILED_ExecutionStatus)
 }
 
+func MapTestResultStatus(success bool) string {
+	if success {
+		return string(testkube.PASSED_ExecutionStatus)
+	}
+
+	return string(testkube.FAILED_ExecutionStatus)
+}
+
 // GetType returns runner type
 func (r *JMeterRunner) GetType() runner.Type {
 	return runner.TypeMain

diff --git a/contrib/executor/jmeter/pkg/runner/runner_test.go b/contrib/executor/jmeter/pkg/runner/runner_test.go
@@ -15,14 +15,14 @@ func TestMapStatus(t *testing.T) {
 	t.Run("should map valid status", func(t *testing.T) {
 		t.Parallel()
 
-		out := MapStatus(parser.Result{Success: false})
+		out := MapResultStatus(parser.Result{Success: false})
 		assert.Equal(t, out, string(testkube.FAILED_ExecutionStatus))
 	})
 
 	t.Run("should map invalid status", func(t *testing.T) {
 		t.Parallel()
 
-		out := MapStatus(parser.Result{Success: true})
+		out := MapResultStatus(parser.Result{Success: true})
 		assert.Equal(t, out, string(testkube.PASSED_ExecutionStatus))
 	})