From 8b7ed39e1d4037417e53d70426384a3c64bbf85b Mon Sep 17 00:00:00 2001 From: Ozben Evren Date: Wed, 7 Mar 2018 13:01:02 -0800 Subject: [PATCH] Add a canonical set of performance tests. (#3809) Automatic merge from submit-queue. Add a canonical set of performance tests. + Refactor setup_perf_cluster.sh to include a function for running canonical tests. + Add a new "run_canonical_perf_tests.sh" for running a standard set of perf tests. + Add a python file for converting the multiple json files into csv format for use in spreadsheets. This gets called from within "run_canonical_perf_tests.sh". --- tools/README.md | 71 ++++++++++++++++++++++++ tools/convert_perf_results.py | 41 ++++++++++++++ tools/run_canonical_perf_tests.sh | 51 ++++++++++++++++++ tools/setup_perf_cluster.sh | 89 +++++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+) create mode 100644 tools/convert_perf_results.py create mode 100755 tools/run_canonical_perf_tests.sh diff --git a/tools/README.md b/tools/README.md index 2a680c63cfca..c2323ae484c3 100644 --- a/tools/README.md +++ b/tools/README.md @@ -159,6 +159,77 @@ to one of the Fortio echo servers: Fortio provides additional load testing capabilities not covered by this document. For more information, refer to the [Fortio documentation](https://github.com/istio/fortio/blob/master/README.md) +### Canonical Tests + +There is a set of canonical tests in ```run_canonical_perf_tests.sh``` script that runs tests by changing parameters in +various dimensions: +- Number of clients +- QPS +- Cached v.s. non-cached + +If you have a change that you think might affect performance, then you can run these tests to check the affects. + +To establish a baseline, simply deploy a perf cluster using the instructions above. Then run +```run_canonical_perf_tests.sh``` to establish the baseline. You will see output that looks like this: + +``` +> run_canonical_perf_tests.sh ++++ In k8s istio ingress: http:///fortio1/fortio/ and fortio2 +Running 'canonical+fortio2+echo1+Q100+T1s+C16' and storing results in /tmp/istio_perf.cpxCcs/canonical_fortio2_echo1_Q100_T1s_C16.json ++++ In k8s istio ingress: http:///fortio1/fortio/ and fortio2 +Running 'canonical+fortio2+echo1+Q400+T1s+C16' and storing results in /tmp/istio_perf.cpxCcs/canonical_fortio2_echo1_Q400_T1s_C16.json +... +``` + +You can check the Fortio UI of the respective drivers to see the results. Also, you can checkout the raw json files +that gets stored in the temporary folder that is in the output above: + +``` +ls /tmp/istio_perf.cpxCcs/ +canonical_fortio2_echo1_Q1000_T1s_C16.json canonical_fortio2_echo1_Q100_T1s_C20.json canonical_fortio2_echo1_Q1200_T1s_C24.json canonical_fortio2_echo1_Q400_T1s_C16.json +canonical_fortio2_echo1_Q1000_T1s_C20.json canonical_fortio2_echo1_Q100_T1s_C24.json canonical_fortio2_echo1_Q1600_T1s_C16.json canonical_fortio2_echo1_Q400_T1s_C20.json +canonical_fortio2_echo1_Q1000_T1s_C24.json canonical_fortio2_echo1_Q1200_T1s_C16.json canonical_fortio2_echo1_Q1600_T1s_C20.json canonical_fortio2_echo1_Q400_T1s_C24.json +canonical_fortio2_echo1_Q100_T1s_C16.json canonical_fortio2_echo1_Q1200_T1s_C20.json canonical_fortio2_echo1_Q1600_T1s_C24.json out.csv +``` + +You can run `fortio report -data-dir /tmp/istio_perf.cpxCcs/` to see all the results and graph them/compare them by visiting `http://localhost:8080` + +Alternatively, notice the ```out.csv``` file in the folder. This file contains all the data in the individual json files, and can be +imported into a spreadsheet: + + +``` +> cat /tmp/istio_perf.cpxCcs/out.csv +Label,Driver,Target,qps,duration,clients,min,max,avg,p50,p75,p90,p99,p99.9 +canonical,fortio2,echo1,1200,1s,16,0.00243703,0.059164527,0.0134183966225,0.0108966942149,0.01594375,0.02405,0.048646875,0.0575867009348 +canonical,fortio2,echo1,1200,1s,24,0.003420898,0.086621239,0.0248239801951,0.0203296703297,0.0303731343284,0.0494375,0.080344304428,0.085993545542 +... +``` + +To test the affects of your change, simply update your cluster with your binaries by following the +[Developer Guide](https://github.com/istio/istio/blob/master/DEV-GUIDE.md) and rerun the tests again. To ensure +you're tracking the results of your changes correctly, you can explicitly specify a label: + +``` +# Notice the "mylabel" parameter below: +# +> run_canonical_perf_tests.sh mylabel ++++ In k8s istio ingress: http:///fortio1/fortio/ and fortio2 +Running 'mylabel+fortio2+echo1+Q400+T1s+C16' and storing results in /tmp/istio_perf.0XuSIH/mylabel_fortio2_echo1_Q400_T1s_C16.json ++++ In k8s istio ingress: http:///fortio1/fortio/ and fortio2 +... +``` + +After the run, you can find the new results both in Fortio UI, and also in the temporary folder: + +``` +> ls /tmp/istio_perf.0XuSIH/ +mylabel_fortio2_echo1_Q1000_T1s_C16.json mylabel_fortio2_echo1_Q100_T1s_C20.json mylabel_fortio2_echo1_Q1200_T1s_C24.json mylabel_fortio2_echo1_Q400_T1s_C16.json +mylabel_fortio2_echo1_Q1000_T1s_C20.json mylabel_fortio2_echo1_Q100_T1s_C24.json mylabel_fortio2_echo1_Q1600_T1s_C16.json mylabel_fortio2_echo1_Q400_T1s_C20.json +mylabel_fortio2_echo1_Q1000_T1s_C24.json mylabel_fortio2_echo1_Q1200_T1s_C16.json mylabel_fortio2_echo1_Q1600_T1s_C20.json mylabel_fortio2_echo1_Q400_T1s_C24.json +mylabel_fortio2_echo1_Q100_T1s_C16.json mylabel_fortio2_echo1_Q1200_T1s_C20.json mylabel_fortio2_echo1_Q1600_T1s_C24.json out.csv +``` + ### Uninstall Use the `delete_all` function to remove everything done by the `setup_all` function. The following delete functions are used by `delete_all` and may be called individually: diff --git a/tools/convert_perf_results.py b/tools/convert_perf_results.py new file mode 100644 index 000000000000..75df0624494d --- /dev/null +++ b/tools/convert_perf_results.py @@ -0,0 +1,41 @@ +import json +import os +import sys + +target_dir="." + +if len(sys.argv) > 1: + target_dir = sys.argv[1] + +# Converts Fortio result output data into a CSV line. +def csv_line(data): + rawLabels = data['Labels'].split() + + labels = ",".join([l for l in rawLabels if l[0] != 'Q' and l[0] != 'T' and l[0] != 'C']) + + qps = data['RequestedQPS'] + duration = data['RequestedDuration'] + clients = data['NumThreads'] + min = data['DurationHistogram']['Min'] + max = data['DurationHistogram']['Max'] + avg = data['DurationHistogram']['Avg'] + + p50 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 50][0] + p75 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 75][0] + p90 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 90][0] + p99 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 99][0] + p99d9 = [e['Value'] for e in data['DurationHistogram']['Percentiles'] if e['Percentile'] == 99.9][0] + + return ("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (labels, qps, duration, clients, min, max, avg, p50, p75, p90, p99, p99d9)) + +# Print the header line +print "Label,Driver,Target,qps,duration,clients,min,max,avg,p50,p75,p90,p99,p99.9" + +# For each json file in current dir, interpret it as Fortio result json file and print a csv line for it. +for fn in os.listdir(target_dir): + fullfn = os.path.join(target_dir, fn) + if os.path.isfile(fullfn) and fullfn.endswith('.json'): + with open(fullfn) as f: + data = json.load(f) + print csv_line(data) + diff --git a/tools/run_canonical_perf_tests.sh b/tools/run_canonical_perf_tests.sh new file mode 100755 index 000000000000..2804871c4b01 --- /dev/null +++ b/tools/run_canonical_perf_tests.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${DIR}/setup_perf_cluster.sh" + +LABEL="${1}" +OUT_DIR="${2}" + +if [[ -z "${OUT_DIR// }" ]]; then + OUT_DIR=$(mktemp -d -t "istio_perf.XXXXXX") +fi + +DURATION="1m" + +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 100 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 400 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1000 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1200 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1600 "${DURATION}" 16 "${OUT_DIR}" + +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 100 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 400 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1000 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1200 "${DURATION}" 16 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1600 "${DURATION}" 16 "${OUT_DIR}" + +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 100 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 400 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1000 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1200 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1600 "${DURATION}" 20 "${OUT_DIR}" + +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 100 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 400 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1000 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1200 "${DURATION}" 20 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1600 "${DURATION}" 20 "${OUT_DIR}" + +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 100 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 400 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1000 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1200 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio2" "echo1" 1600 "${DURATION}" 24 "${OUT_DIR}" + +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 100 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 400 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1000 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1200 "${DURATION}" 24 "${OUT_DIR}" +run_canonical_perf_test "${LABEL}" "fortio1" "echo2" 1600 "${DURATION}" 24 "${OUT_DIR}" + +python "${DIR}/convert_perf_results.py" "${OUT_DIR}" > "${OUT_DIR}/out.csv" \ No newline at end of file diff --git a/tools/setup_perf_cluster.sh b/tools/setup_perf_cluster.sh index d2ad6dd1d9a7..8590413c2009 100755 --- a/tools/setup_perf_cluster.sh +++ b/tools/setup_perf_cluster.sh @@ -228,6 +228,95 @@ function run_fortio_test3() { Execute curl "http://$VM_IP/fortio/?json=on&qps=-1&t=30s&c=48&load=Start&url=http://$ISTIO_INGRESS_IP/fortio1/echo" } +# Run canonical perf tests. +# The following parameters can be supplied: +# 1) Label: +# A custom label to use. This is useful when running the same suite against two target binaries/configs. +# Defaults to "canonical" +# 2) Driver: +# The load driver to use. Currently "fortio1" and "fortio2" are supported. Defaults to "fortio1". +# 3) Target: +# The target service for the load. Currently "echo1" and "echo2" are supported. +# Defaults to "echo2" +# 4) QPS: +# The QPS to apply. Defaults to 400. +# 5) Duration: +# The duration of the test. Default is 5 minutes. +# 6) Clients: +# The number of clients to use. Defaults is 16. +# 7) Outdir: +# The output dir for collecting the Json results. If not specified, a temporary dir will be created. +function run_canonical_perf_test() { + LABEL="${1}" + DRIVER="${2}" + TARGET="${3}" + QPS="${4}" + DURATION="${5}" + CLIENTS="${6}" + OUT_DIR="${7}" + + # Set defaults + LABEL="${LABEL:-canonical}" + DRIVER="${DRIVER:-fortio1}" + TARGET="${TARGET:-echo2}" + QPS="${QPS:-400}" + DURATION="${DURATION:-5m}" + CLIENTS="${CLIENTS:-16}" + + get_istio_ingress_ip + + FORTIO1_URL="http://${ISTIO_INGRESS_IP}/fortio1/fortio" + FORTIO2_URL="http://${ISTIO_INGRESS_IP}/fortio2/fortio" + case "${DRIVER}" in + "fortio1") + DRIVER_URL="${FORTIO1_URL}" + ;; + "fortio2") + DRIVER_URL="${FORTIO2_URL}" + ;; + *) + echo "unknown driver: ${DRIVER}" + exit -1 + ;; + esac + + # URL encoded URLs for echo1 and echo2. These get directly embedded as parameters into the main URL to invoke + # the test. + ECHO1_URL="echosrv1:8080/echo" + ECHO2_URL="echosrv2:8080/echo" + case "${TARGET}" in + "echo1") + TARGET_URL="${ECHO1_URL}" + ;; + "echo2") + TARGET_URL="${ECHO2_URL}" + ;; + *) + echo "unknown target: ${TARGET}" + exit -1 + ;; + esac + + PERCENTILES="50%2C+75%2C+90%2C+99%2C+99.9" + GRANULARITY="0.001" + + LABELS="${LABEL}+${DRIVER}+${TARGET}+Q${QPS}+T${DURATION}+C${CLIENTS}" + + if [[ -z "${OUT_DIR// }" ]]; then + OUT_DIR=$(mktemp -d -t "istio_perf.XXXXXX") + fi + + FILE_NAME="${LABELS//\+/_}" + OUT_FILE="${OUT_DIR}/${FILE_NAME}.json" + + echo "Running '${LABELS}' and storing results in ${OUT_FILE}" + + URL="${DRIVER_URL}/?labels=${LABELS}&url=${TARGET_URL}&qps=${QPS}&t=${DURATION}&c=${CLIENTS}&p=${PERCENTILES}&r=${GRANULARITY}&json=on&save=on&load=Start" + #echo "URL: ${URL}" + + curl -s "${URL}" -o "${OUT_FILE}" +} + function setup_vm_all() { update_gcp_opts create_vm