forked from istio/istio
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a canonical set of performance tests. (istio#3809)
Automatic merge from submit-queue. Add a canonical set of performance tests. + Refactor setup_perf_cluster.sh to include a function for running canonical tests. + Add a new "run_canonical_perf_tests.sh" for running a standard set of perf tests. + Add a python file for converting the multiple json files into csv format for use in spreadsheets. This gets called from within "run_canonical_perf_tests.sh".
- Loading branch information
1 parent
482ab04
commit 8b7ed39
Showing
4 changed files
with
252 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| import json | ||
| import os | ||
| import sys | ||
|
|
||
| target_dir="." | ||
|
|
||
| if len(sys.argv) > 1: | ||
| target_dir = sys.argv[1] | ||
|
|
||
| # Converts Fortio result output data into a CSV line. | ||
| def csv_line(data): | ||
| rawLabels = data['Labels'].split() | ||
|
|
||
| labels = ",".join([l for l in rawLabels if l[0] != 'Q' and l[0] != 'T' and l[0] != 'C']) | ||
|
|
||
| qps = data['RequestedQPS'] | ||
| duration = data['RequestedDuration'] | ||
| clients = data['NumThreads'] | ||
| min = data['DurationHistogram']['Min'] | ||
| max = data['DurationHistogram']['Max'] | ||
| avg = data['DurationHistogram']['Avg'] | ||
|
|
||
| p50 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 50][0] | ||
| p75 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 75][0] | ||
| p90 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 90][0] | ||
| p99 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 99][0] | ||
| p99d9 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 99.9][0] | ||
|
|
||
| return ("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (labels, qps, duration, clients, min, max, avg, p50, p75, p90, p99, p99d9)) | ||
|
|
||
| # Print the header line | ||
| print "Label,Driver,Target,qps,duration,clients,min,max,avg,p50,p75,p90,p99,p99.9" | ||
|
|
||
| # For each json file in current dir, interpret it as Fortio result json file and print a csv line for it. | ||
| for fn in os.listdir(target_dir): | ||
| fullfn = os.path.join(target_dir, fn) | ||
| if os.path.isfile(fullfn) and fullfn.endswith('.json'): | ||
| with open(fullfn) as f: | ||
| data = json.load(f) | ||
| print csv_line(data) | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| #!/bin/bash | ||
|
|
||
| DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" | ||
| source "${DIR}/setup_perf_cluster.sh" | ||
|
|
||
| LABEL="${1}" | ||
| OUT_DIR="${2}" | ||
|
|
||
| if [[ -z "${OUT_DIR// }" ]]; then | ||
| OUT_DIR=$(mktemp -d -t "istio_perf.XXXXXX") | ||
| fi | ||
|
|
||
| DURATION="1m" | ||
|
|
||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 100 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 400 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1000 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1200 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1600 "${DURATION}" 16 "${OUT_DIR}" | ||
|
|
||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 100 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 400 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1000 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1200 "${DURATION}" 16 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1600 "${DURATION}" 16 "${OUT_DIR}" | ||
|
|
||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 100 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 400 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1000 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1200 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1600 "${DURATION}" 20 "${OUT_DIR}" | ||
|
|
||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 100 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 400 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1000 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1200 "${DURATION}" 20 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1600 "${DURATION}" 20 "${OUT_DIR}" | ||
|
|
||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 100 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 400 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1000 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1200 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1600 "${DURATION}" 24 "${OUT_DIR}" | ||
|
|
||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 100 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 400 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1000 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1200 "${DURATION}" 24 "${OUT_DIR}" | ||
| run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1600 "${DURATION}" 24 "${OUT_DIR}" | ||
|
|
||
| python "${DIR}/convert_perf_results.py" "${OUT_DIR}" > "${OUT_DIR}/out.csv" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters