diff --git a/grafana/staging/dashboards/ai-gateway.json b/grafana/staging/dashboards/ai-gateway.json index ffb22038..208bda21 100644 --- a/grafana/staging/dashboards/ai-gateway.json +++ b/grafana/staging/dashboards/ai-gateway.json @@ -165,7 +165,7 @@ "disableTextWrap": false, "editorMode": "builder", "exemplar": false, - "expr": "sum by() (increase(livepeer_ticket_value_sent{job=~\"^.*$job_livepeer.*$\"}[$__range])) / 1000000000", + "expr": "sum by() (increase(livepeer_ticket_value_sent{livepeer_live_app=~\".*livepeer-ai-gateway\"}[$__range])) / 1000000000", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -260,7 +260,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "sum by(pipeline, model_name) (increase(livepeer_ai_models_requested{job=~\"^.*$job_livepeer.*$\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", + "expr": "sum by(pipeline, model_name) (increase(livepeer_ai_models_requested{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -343,7 +343,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pipeline, model_name) (\n increase(\n livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", error_code=~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", + "expr": "sum by (pipeline, model_name) (\n increase(\n livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", error_code=~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", "hide": false, "legendFormat": "{{pipeline}} | {{model_name}}", "range": true, @@ -492,7 +492,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum((rate(livepeer_ticket_value_sent{job=~\"^.*$job_livepeer.*$\"}[1d])*(60*60*24*365)/12)*1e-9 * $eth) or on() vector(0)", + "expr": "sum((rate(livepeer_ticket_value_sent{livepeer_live_app=~\".*livepeer-ai-gateway\"}[1d])*(60*60*24*365)/12)*1e-9 * $eth) or on() vector(0)", "legendFormat": "Monthly", "range": true, "refId": "A" @@ -503,7 +503,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum((rate(livepeer_ticket_value_sent{job=~\"^.*$job_livepeer.*$\"}[1d])*(60*60*24))*1e-9 * $eth) or on() vector(0)", + "expr": "sum((rate(livepeer_ticket_value_sent{livepeer_live_app=~\"prod-livepeer-ai-gateway\"}[1d])*(60*60*24))*1e-9 * $eth) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Daily", @@ -602,7 +602,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(increase(livepeer_ai_models_requested{job=~\"^.*$job_livepeer.*$\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", + "expr": "sum(increase(livepeer_ai_models_requested{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", "hide": false, "legendFormat": "Sum", "range": true, @@ -674,7 +674,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(\n increase(\n livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", error_code=~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", + "expr": "sum(\n increase(\n livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", error_code=~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", "hide": false, "legendFormat": "__auto", "range": true, @@ -746,7 +746,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(\n increase(\n livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", + "expr": "sum(\n increase(\n livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", "hide": false, "legendFormat": "__auto", "range": true, @@ -848,7 +848,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pipeline, model_name) (increase(livepeer_ai_models_requested{job=~\"^.*$job_livepeer.*$\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__interval]))", + "expr": "sum by (pipeline, model_name) (increase(livepeer_ai_models_requested{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__interval]))", "hide": false, "instant": false, "interval": "", @@ -923,7 +923,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg_over_time(livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-image\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 1e-18 * $eth", + "expr": "avg_over_time(livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-image\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 1e-18 * $eth", "hide": false, "instant": false, "legendFormat": "Cost", @@ -936,7 +936,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg_over_time(livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-image\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 1e-18 * $eth", + "expr": "avg_over_time(livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-image\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 1e-18 * $eth", "hide": false, "legendFormat": "Cost", "range": true, @@ -948,7 +948,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg_over_time(livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-video\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 25 * 1e-18 * $eth", + "expr": "avg_over_time(livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-video\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 25 * 1e-18 * $eth", "hide": false, "instant": false, "legendFormat": "Cost", @@ -961,7 +961,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg_over_time(livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-video\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 25 * 1e-18 * $eth", + "expr": "avg_over_time(livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-video\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 25 * 1e-18 * $eth", "hide": false, "instant": false, "legendFormat": "Cost", @@ -974,7 +974,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg_over_time(livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"audio-to-text\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1000 * 60 * 1e-18 * $eth", + "expr": "avg_over_time(livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"audio-to-text\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1000 * 60 * 1e-18 * $eth", "hide": false, "instant": false, "legendFormat": "Cost", @@ -987,7 +987,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg_over_time(livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"upscale\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 1e-18 * $eth", + "expr": "avg_over_time(livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"upscale\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]) * 1024 * 1024 * 1e-18 * $eth", "hide": false, "instant": false, "legendFormat": "Cost", @@ -1082,7 +1082,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (pipeline, model_name) (\n increase(\n livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", + "expr": "sum by (pipeline, model_name) (\n increase(\n livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]\n )\n)", "hide": false, "legendFormat": "{{pipeline}} | {{model_name}}", "range": true, @@ -1145,7 +1145,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null } ] }, @@ -1184,7 +1185,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "increase(livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__interval])", + "expr": "increase(livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__interval])", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1280,7 +1281,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (orchestrator_address) (increase(livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", + "expr": "sum by (orchestrator_address) (increase(livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", "hide": false, "instant": false, "legendFormat": "{{orchestrator_address}}", @@ -1327,7 +1328,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null } ] }, @@ -1402,7 +1404,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 1e-18 * $eth)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 1e-18 * $eth)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1415,7 +1417,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 1e-18 * $eth)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 1e-18 * $eth)", "hide": false, "legendFormat": "{{pipeline}} | {{model_name}}", "range": true, @@ -1427,7 +1429,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 25 * 1e-18 * $eth)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 25 * 1e-18 * $eth)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1440,7 +1442,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 25 * 1e-18 * $eth)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 25 * 1e-18 * $eth)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1453,7 +1455,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"audio-to-text\", model_name=~\"${model:pipe}\"} * 1000 * 60 * 1e-18 * $eth)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"audio-to-text\", model_name=~\"${model:pipe}\"} * 1000 * 60 * 1e-18 * $eth)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1466,7 +1468,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{job=~\"^.*$job_livepeer.*$\", pipeline=\"upscale\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 1e-18 * $eth)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_price{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"upscale\", model_name=~\"${model:pipe}\"} * 1024 * 1024 * 1e-18 * $eth)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1531,7 +1533,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null } ] }, @@ -1604,7 +1607,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1617,7 +1620,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "legendFormat": "{{pipeline}} | {{model_name}}", "range": true, @@ -1629,7 +1632,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1642,7 +1645,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1655,7 +1658,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"audio-to-text\", model_name=~\"${model:pipe}\"} * 1)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"audio-to-text\", model_name=~\"${model:pipe}\"} * 1)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1668,7 +1671,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"upscale\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (pipeline, model_name) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"upscale\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{pipeline}} | {{model_name}}", @@ -1787,7 +1790,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (error_code) (increase(livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", + "expr": "sum by (error_code) (increase(livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", "hide": false, "instant": false, "legendFormat": "{{orchestrator_address}}", @@ -1835,7 +1838,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null } ] }, @@ -1890,7 +1894,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum by (orchestrator_address, error_code) (increase(livepeer_ai_request_errors{job=~\"^.*$job_livepeer.*$\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", + "expr": "sum by (orchestrator_address, error_code) (increase(livepeer_ai_request_errors{livepeer_live_app=~\".*livepeer-ai-gateway\", error_code!~\"no orchestrators available.*\", pipeline=~\"${pipeline:pipe}\", model_name=~\"${model:pipe}\"}[$__range]))", "hide": false, "legendFormat": "{{orchestrator_address}}: {{error_code}}", "range": true, @@ -1977,7 +1981,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null } ] }, @@ -2018,7 +2023,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{orchestrator_address}}", @@ -2031,7 +2036,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-image\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "legendFormat": "{{orchestrator_address}}", "range": true, @@ -2043,7 +2048,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"text-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"text-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{orchestrator_address}}", @@ -2056,7 +2061,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"image-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"image-to-video\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{orchestrator_address}}", @@ -2069,7 +2074,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"audio-to-text\", model_name=~\"${model:pipe}\"} * 1)", + "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"audio-to-text\", model_name=~\"${model:pipe}\"} * 1)", "hide": false, "instant": false, "legendFormat": "{{orchestrator_address}}", @@ -2082,7 +2087,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{job=~\"^.*$job_livepeer.*$\", pipeline=\"upscale\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", + "expr": "avg by (orchestrator_address) (livepeer_ai_request_latency_score{livepeer_live_app=~\".*livepeer-ai-gateway\", pipeline=\"upscale\", model_name=~\"${model:pipe}\"} * 1024 * 1024)", "hide": false, "instant": false, "legendFormat": "{{orchestrator_address}}", @@ -2154,7 +2159,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2167,7 +2173,7 @@ }, "gridPos": { "h": 10, - "w": 9, + "w": 11, "x": 0, "y": 76 }, @@ -2228,8 +2234,8 @@ "description": "excludes `Polling blocks` log lines", "gridPos": { "h": 10, - "w": 15, - "x": 9, + "w": 11, + "x": 11, "y": 76 }, "id": 23763572331, @@ -2271,9 +2277,53 @@ }, "description": "", "gridPos": { - "h": 10, - "w": 15, - "x": 9, + "h": 8, + "w": 11, + "x": 0, + "y": 86 + }, + "id": 23763572333, + "links": [ + { + "title": "Explore logs", + "url": "/grafana/explore?schemaVersion=1&panes=%7B%2203k%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D~%5C%22ingress-nginx%5C%22%7D%20%7C%3D%20%60%2Fapi%2Fbeta%2Fgenerate%2F%60%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D%7D&orgId=1" + } + ], + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{app=~\"ingress-nginx\", region!~\"analytics|eu-metrics|mad|atl\"} |= `/api/beta/generate/`", + "queryType": "range", + "refId": "A" + } + ], + "title": "ingress-nginx logs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 11, + "x": 11, "y": 86 }, "id": 23763572332, @@ -2299,8 +2349,8 @@ "type": "loki", "uid": "P8E80F9AEF21F6940" }, - "editorMode": "code", - "expr": "{app=~\".*livepeer-api\"} |= `/api/beta/generate/`", + "editorMode": "builder", + "expr": "{app=~\".*livepeer-api\"} |~ `(/api/beta/generate/|ai_generate_log|generate API|ai-generate)`", "queryType": "range", "refId": "A" } @@ -2309,6 +2359,7 @@ "type": "logs" } ], + "refresh": "30s", "revision": 1, "schemaVersion": 39, "tags": [], @@ -2427,7 +2478,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-2d", "to": "now" }, "timepicker": { @@ -2439,6 +2490,6 @@ "timezone": "", "title": "AI Gateway", "uid": "Livepeer-Gateway-AI", - "version": 8, + "version": 3, "weekStart": "monday" } diff --git a/grafana/staging/dashboards/app-versions.json b/grafana/staging/dashboards/app-versions.json index efffa4be..56bf7305 100644 --- a/grafana/staging/dashboards/app-versions.json +++ b/grafana/staging/dashboards/app-versions.json @@ -744,6 +744,7 @@ }, { "datasource": { + "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, @@ -829,7 +830,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(livepeer_versions{catalyst!=\"true\", node_type=\"bctr\", livepeer_node_type!=\"prod-livepeer-recording\", }) by (livepeerversion)", + "expr": "sum(livepeer_versions{catalyst!=\"true\", node_type=\"bctr\", livepeer_node_type!=\"staging-livepeer-recording\", livepeer_node_type!=\"staging-livepeer-ai-gateway\"}) by (livepeerversion)", "legendFormat": "__auto", "range": true, "refId": "A" @@ -934,6 +935,103 @@ "title": "go-livepeer (Recording Broadcasters)", "type": "timeseries" }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 21, + "interval": "1m", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(livepeer_versions{catalyst!=\"true\", node_type=\"bctr\", livepeer_node_type=~\".+-livepeer-ai-gateway\"}) by (livepeerversion)", + "legendFormat": "{{ livepeerversion }}", + "range": true, + "refId": "A" + } + ], + "title": "go-livepeer (AI Gateway)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", diff --git a/grafana/staging/dashboards/nginx-5xx.json b/grafana/staging/dashboards/nginx-5xx.json index 55c23e08..18ed7c75 100644 --- a/grafana/staging/dashboards/nginx-5xx.json +++ b/grafana/staging/dashboards/nginx-5xx.json @@ -1686,6 +1686,6 @@ "timezone": "", "title": "nginx 5XX", "uid": "f74066bf-c68e-4303-a9c4-81423738f8a3", - "version": 2, + "version": 1, "weekStart": "" } diff --git a/grafana/staging/dashboards/orchestrator-versions.json b/grafana/staging/dashboards/orchestrator-versions.json index 1e682e42..eda2763d 100644 --- a/grafana/staging/dashboards/orchestrator-versions.json +++ b/grafana/staging/dashboards/orchestrator-versions.json @@ -362,6 +362,6 @@ "timezone": "browser", "title": "Orchestrator versions", "uid": "edyyryuj00we8b", - "version": 2, + "version": 1, "weekStart": "" }