diff --git a/scale_test/dashboard.json b/scale_test/dashboard.json index 69ea137d..5e230dd2 100644 --- a/scale_test/dashboard.json +++ b/scale_test/dashboard.json @@ -21,6 +21,19 @@ "id": 4, "links": [], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 331, + "panels": [], + "title": "All Jobs", + "type": "row" + }, { "datasource": { "default": false, @@ -156,7 +169,7 @@ "h": 8, "w": 24, "x": 0, - "y": 0 + "y": 1 }, "id": 327, "options": { @@ -196,7 +209,7 @@ "timeField": "timestamp" } ], - "title": "All Jobs Summary", + "title": "Job Summary", "transformations": [ { "id": "organize", @@ -287,6 +300,73 @@ ], "type": "table" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 330, + "panels": [], + "title": "Selected Job UUID", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "elasticsearch", + "uid": "$Datasource" + }, + "description": "", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 329, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "#### All panels below show data for the selected job UUID only", + "mode": "markdown" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "ee0gkcjio8c8wc" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "type": "text" + }, { "datasource": { "uid": "$Datasource" @@ -416,10 +496,10 @@ ] }, "gridPos": { - "h": 8, + "h": 6, "w": 24, "x": 0, - "y": 8 + "y": 12 }, "id": 328, "options": { @@ -459,7 +539,7 @@ "timeField": "timestamp" } ], - "title": "Selected Job Summary", + "title": "Job Summary", "transformations": [ { "id": "organize", @@ -548,6 +628,641 @@ } ], "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 334, + "panels": [], + "repeat": "jobName", + "repeatDirection": "h", + "title": "Stats for Job: $jobName", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "elasticsearch", + "uid": "$Datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 350, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "labels.pod" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "elasticsearch", + "uid": "ee0gkcjio8c8wc" + }, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "uuid.keyword: $uuid AND metricName: \"namespaceCPU\" AND jobName: \"$jobName\"", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "CPU Cores by namespace", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_id": true, + "_index": true, + "_type": true, + "highlight": true, + "jobName": true, + "metricName": true, + "query": true, + "sort": true, + "timestamp": true, + "uuid": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "labels.namespace": "Namespace", + "labels.pod": "Pod", + "value": "" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Namespace": { + "aggregations": [], + "operation": "groupby" + }, + "Pod": { + "aggregations": [] + }, + "labels.namespace": { + "aggregations": [], + "operation": "groupby" + }, + "labels.pod": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Namespace" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "elasticsearch", + "uid": "$Datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "value (lastNotNull)" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 351, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "labels.pod" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "elasticsearch", + "uid": "ee0gkcjio8c8wc" + }, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "uuid.keyword: $uuid AND metricName: \"namespaceMemory\" AND jobName: \"$jobName\"", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Memory usage by namespace", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_id": true, + "_index": true, + "_type": true, + "highlight": true, + "jobName": true, + "metricName": true, + "query": true, + "sort": true, + "timestamp": true, + "uuid": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "labels.namespace": "Namespace", + "labels.pod": "Pod", + "value": "" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Namespace": { + "aggregations": [], + "operation": "groupby" + }, + "Pod": { + "aggregations": [] + }, + "labels.namespace": { + "aggregations": [], + "operation": "groupby" + }, + "labels.pod": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Namespace" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "elasticsearch", + "uid": "$Datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 341, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "labels.pod" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "elasticsearch", + "uid": "ee0gkcjio8c8wc" + }, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "uuid.keyword: $uuid AND metricName: \"namespacePodCPU\" AND jobName: \"$jobName\"", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "CPU Cores by pod", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_id": true, + "_index": true, + "_type": true, + "highlight": true, + "jobName": true, + "metricName": true, + "query": true, + "sort": true, + "timestamp": true, + "uuid": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "labels.namespace": "Namespace", + "labels.pod": "Pod", + "value": "" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Namespace": { + "aggregations": [], + "operation": "groupby" + }, + "Pod": { + "aggregations": [], + "operation": "groupby" + }, + "labels.namespace": { + "aggregations": [], + "operation": "groupby" + }, + "labels.pod": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Namespace" + } + ] + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "elasticsearch", + "uid": "$Datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "value (lastNotNull)" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 352, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "labels.pod" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "elasticsearch", + "uid": "ee0gkcjio8c8wc" + }, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "uuid.keyword: $uuid AND metricName: \"namespacePodMemory\" AND jobName: \"$jobName\"", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Memory usage by pod", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_id": true, + "_index": true, + "_type": true, + "highlight": true, + "jobName": true, + "metricName": true, + "query": true, + "sort": true, + "timestamp": true, + "uuid": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "labels.namespace": "Namespace", + "labels.pod": "Pod", + "value": "" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Namespace": { + "aggregations": [], + "operation": "groupby" + }, + "Pod": { + "aggregations": [], + "operation": "groupby" + }, + "labels.namespace": { + "aggregations": [], + "operation": "groupby" + }, + "labels.pod": { + "aggregations": [], + "operation": "groupby" + }, + "value": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Namespace" + } + ] + } + } + ], + "type": "table" } ], "refresh": false, @@ -577,8 +1292,8 @@ { "current": { "selected": false, - "text": "a6471ca4-18f3-4e12-ab0c-ad14ec84301a", - "value": "a6471ca4-18f3-4e12-ab0c-ad14ec84301a" + "text": "cab334d6-bdbc-435f-8816-c07ce45d9ac5", + "value": "cab334d6-bdbc-435f-8816-c07ce45d9ac5" }, "datasource": { "uid": "$Datasource" @@ -599,6 +1314,36 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "$Datasource" + }, + "definition": "{\"find\": \"terms\", \"field\": \"jobName.keyword\"}", + "hide": 0, + "includeAll": true, + "label": "Job Name", + "multi": true, + "name": "jobName", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"jobName.keyword\"}", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, @@ -623,6 +1368,6 @@ "timezone": "", "title": "Kuadrant Performance Testing", "uid": "fe0k12z2zvzeod", - "version": 6, + "version": 15, "weekStart": "" } \ No newline at end of file diff --git a/scale_test/metrics.yaml b/scale_test/metrics.yaml index ca272289..109188d8 100644 --- a/scale_test/metrics.yaml +++ b/scale_test/metrics.yaml @@ -1,7 +1,11 @@ -- query: sum(rate(container_cpu_usage_seconds_total{namespace=~"kuadrant-system|istio-system|gateway-system|scale-test-.*"}[5m])) by(namespace) +- query: sum(rate(container_cpu_usage_seconds_total{container="",namespace=~"kuadrant-system|istio-system|gateway-system|scale-test-.*"}[5m])) by(namespace) metricName: namespaceCPU -- query: sum(container_memory_usage_bytes{namespace=~"kuadrant-system|istio-system|gateway-system|scale-test-.*"}) by(namespace) +- query: sum(container_memory_usage_bytes{container="",namespace=~"kuadrant-system|istio-system|gateway-system|scale-test-.*"}) by(namespace) metricName: namespaceMemory +- query: sum(rate(container_cpu_usage_seconds_total{container="",namespace=~"kuadrant-system|istio-system|gateway-system|scale-test-.*"}[5m])) by(namespace, pod) + metricName: namespacePodCPU +- query: sum(container_memory_usage_bytes{container="",namespace=~"kuadrant-system|istio-system|gateway-system|scale-test-.*"}) by(namespace, pod) + metricName: namespacePodMemory # API server - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0