diff --git a/config/seatunnel.yaml b/config/seatunnel.yaml index 5961c839238..aa335e5a57c 100644 --- a/config/seatunnel.yaml +++ b/config/seatunnel.yaml @@ -33,4 +33,7 @@ seatunnel: plugin-config: namespace: /tmp/seatunnel/checkpoint_snapshot storage.type: hdfs - fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission \ No newline at end of file + fs.defaultFS: file:///tmp/ # Ensure that the directory has written permission + telemetry: + metric: + enabled: true \ No newline at end of file diff --git a/docs/en/images/cluster_monitoring.png b/docs/en/images/cluster_monitoring.png new file mode 100644 index 00000000000..8f534510cca Binary files /dev/null and b/docs/en/images/cluster_monitoring.png differ diff --git a/docs/en/seatunnel-engine/grafana/dashboard.json b/docs/en/seatunnel-engine/grafana/dashboard.json new file mode 100644 index 00000000000..a0457631055 --- /dev/null +++ b/docs/en/seatunnel-engine/grafana/dashboard.json @@ -0,0 +1,1904 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-orange", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "node_count", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total Node Count", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 18, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 1 + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "sum(node_state)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "UP Node Count", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 22, + "panels": [], + "title": "Hazelcast Partition", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_partition_partitionCount", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "partitionCount", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_partition_activePartition", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "activePartition", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_partition_isClusterSafe", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "isClusterSafe", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_partition_isLocalMemberSafe", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "isLocalMemberSafe", + "type": "timeseries" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 20, + "title": "Hazelcast Executor", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_executor_executedCount", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "title": "executedCount", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_executor_isTerminated", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "title": "isTerminated", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_executor_isShutdown", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "title": "isShutdown", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_executor_poolSize", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "title": "poolSize", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_executor_maxPoolSize", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "title": "maxPoolSize", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_executor_queueRemainingCapacity", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "title": "queueRemainingCapacity", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "hazelcast_executor_queueSize", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "title": "queueSize", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 7, + "panels": [], + "title": "System", + "type": "row" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "100 * (1 - rate(process_cpu_seconds_total[15s]))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Cpu Usage", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "100 * (jvm_memory_bytes_used{area=\"heap\"} / jvm_memory_bytes_max{area=\"heap\"})", + "interval": "", + "legendFormat": "{{instance}}-{{area}}", + "refId": "A" + } + ], + "title": "Heap Memory Usage", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "round(delta(jvm_gc_collection_seconds_count[15s]))", + "interval": "", + "legendFormat": "{{instance}}-{{gc}}", + "refId": "A" + } + ], + "title": "GC Count", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "delta(jvm_gc_collection_seconds_sum[15s]) * 1000", + "interval": "", + "legendFormat": "{{instance}}-{{gc}}", + "refId": "A" + } + ], + "title": "GC Cost Time", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "jvm_threads_current", + "interval": "", + "legendFormat": "{{instance}}-current", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "jvm_threads_daemon", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-daemon", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "jvm_threads_peak", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-peak", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "jvm_threads_deadlocked", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-deadlocked", + "refId": "D" + } + ], + "title": "Jvm Threads", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 5, + "panels": [], + "title": "Job", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 76 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "sum(job_count) by (type) ", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Job Count", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "job_thread_pool_activeCount", + "interval": "", + "legendFormat": "{{instance}}-activeCount", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "job_thread_pool_corePoolSize", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-corePoolSize", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "job_thread_pool_maximumPoolSize", + "hide": true, + "interval": "", + "legendFormat": "{{instance}}-maximumPoolSize", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "job_thread_pool_poolSize", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-poolSize", + "refId": "D" + } + ], + "title": "Job Thread Pool", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "job_thread_pool_completedTask_total", + "interval": "", + "legendFormat": "{{instance}}-completedTaskTotal", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "jUi2yaj4k" + }, + "exemplar": true, + "expr": "job_thread_pool_task_total", + "hide": false, + "interval": "", + "legendFormat": "{{instance}}-taskTotal", + "refId": "B" + } + ], + "title": "Job Thread Pool Total", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 34, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Seatunnel Cluster", + "uid": "BDfcs-CVz", + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/en/seatunnel-engine/openmetrics/metrics.txt b/docs/en/seatunnel-engine/openmetrics/metrics.txt new file mode 100644 index 00000000000..ea1855fb113 --- /dev/null +++ b/docs/en/seatunnel-engine/openmetrics/metrics.txt @@ -0,0 +1,280 @@ +# TYPE job_count gauge +# HELP job_count All job counts of seatunnel cluster +job_count{cluster="seatunnel",type="canceled"} 0.0 +job_count{cluster="seatunnel",type="cancelling"} 0.0 +job_count{cluster="seatunnel",type="created"} 0.0 +job_count{cluster="seatunnel",type="failed"} 0.0 +job_count{cluster="seatunnel",type="failing"} 0.0 +job_count{cluster="seatunnel",type="finished"} 0.0 +job_count{cluster="seatunnel",type="reconciling"} 0.0 +job_count{cluster="seatunnel",type="restarting"} 0.0 +job_count{cluster="seatunnel",type="running"} 0.0 +job_count{cluster="seatunnel",type="scheduled"} 0.0 +job_count{cluster="seatunnel",type="suspended"} 0.0 +# TYPE process_cpu_seconds counter +# HELP process_cpu_seconds Total user and system CPU time spent in seconds. +process_cpu_seconds_total 15.74875 +# TYPE process_start_time_seconds gauge +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +process_start_time_seconds 1.689671779061E9 +# TYPE process_open_fds gauge +# HELP process_open_fds Number of open file descriptors. +process_open_fds 112.0 +# TYPE process_max_fds gauge +# HELP process_max_fds Maximum number of open file descriptors. +process_max_fds 10240.0 +# TYPE cluster_info gauge +# HELP cluster_info Cluster info +cluster_info{cluster="seatunnel",hazelcastVersion="5.1",master="127.0.0.1:5801"} 1.0 +# TYPE cluster_time gauge +# HELP cluster_time Cluster start time +cluster_time{cluster="seatunnel",hazelcastVersion="5.1"} 1.68967187357E12 +# TYPE node_count gauge +# HELP node_count Cluster node total count +node_count{cluster="seatunnel"} 1.0 +# TYPE jvm_threads_current gauge +# HELP jvm_threads_current Current thread count of a JVM +jvm_threads_current 105.0 +# TYPE jvm_threads_daemon gauge +# HELP jvm_threads_daemon Daemon thread count of a JVM +jvm_threads_daemon 7.0 +# TYPE jvm_threads_peak gauge +# HELP jvm_threads_peak Peak thread count of a JVM +jvm_threads_peak 118.0 +# TYPE jvm_threads_started counter +# HELP jvm_threads_started Started thread count of a JVM +jvm_threads_started_total 122.0 +# TYPE jvm_threads_deadlocked gauge +# HELP jvm_threads_deadlocked Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers +jvm_threads_deadlocked 0.0 +# TYPE jvm_threads_deadlocked_monitor gauge +# HELP jvm_threads_deadlocked_monitor Cycles of JVM-threads that are in deadlock waiting to acquire object monitors +jvm_threads_deadlocked_monitor 0.0 +# TYPE jvm_threads_state gauge +# HELP jvm_threads_state Current count of threads by state +jvm_threads_state{state="NEW"} 0.0 +jvm_threads_state{state="TERMINATED"} 0.0 +jvm_threads_state{state="RUNNABLE"} 12.0 +jvm_threads_state{state="BLOCKED"} 0.0 +jvm_threads_state{state="WAITING"} 76.0 +jvm_threads_state{state="TIMED_WAITING"} 17.0 +jvm_threads_state{state="UNKNOWN"} 0.0 +# TYPE jvm_classes_currently_loaded gauge +# HELP jvm_classes_currently_loaded The number of classes that are currently loaded in the JVM +jvm_classes_currently_loaded 8317.0 +# TYPE jvm_classes_loaded counter +# HELP jvm_classes_loaded The total number of classes that have been loaded since the JVM has started execution +jvm_classes_loaded_total 8317.0 +# TYPE jvm_classes_unloaded counter +# HELP jvm_classes_unloaded The total number of classes that have been unloaded since the JVM has started execution +jvm_classes_unloaded_total 0.0 +# TYPE node_state gauge +# HELP node_state Whether is up of seatunnel node +node_state{cluster="seatunnel",address="127.0.0.1:5801"} 1.0 +# TYPE hazelcast_executor_executedCount gauge +# HELP hazelcast_executor_executedCount The hazelcast executor executedCount of seatunnel cluster node +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="async"} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="client"} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="clientBlocking"} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="clientQuery"} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="io"} 136.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="offloadable"} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="scheduled"} 1601.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="127.0.0.1:5801",type="system"} 0.0 +# TYPE hazelcast_executor_isShutdown gauge +# HELP hazelcast_executor_isShutdown The hazelcast executor isShutdown of seatunnel cluster node +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="async"} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="client"} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="clientBlocking"} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="clientQuery"} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="io"} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="offloadable"} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="scheduled"} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="127.0.0.1:5801",type="system"} 0.0 +# TYPE hazelcast_executor_isTerminated gauge +# HELP hazelcast_executor_isTerminated The hazelcast executor isTerminated of seatunnel cluster node +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="async"} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="client"} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="clientBlocking"} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="clientQuery"} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="io"} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="offloadable"} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="scheduled"} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="127.0.0.1:5801",type="system"} 0.0 +# TYPE hazelcast_executor_maxPoolSize gauge +# HELP hazelcast_executor_maxPoolSize The hazelcast executor maxPoolSize of seatunnel cluster node +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="async"} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="client"} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="clientBlocking"} 160.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="clientQuery"} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="io"} 16.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="offloadable"} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="scheduled"} 16.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="127.0.0.1:5801",type="system"} 8.0 +# TYPE hazelcast_executor_poolSize gauge +# HELP hazelcast_executor_poolSize The hazelcast executor poolSize of seatunnel cluster node +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="async"} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="client"} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="clientBlocking"} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="clientQuery"} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="io"} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="offloadable"} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="scheduled"} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="127.0.0.1:5801",type="system"} 0.0 +# TYPE hazelcast_executor_queueRemainingCapacity gauge +# HELP hazelcast_executor_queueRemainingCapacity The hazelcast executor queueRemainingCapacity of seatunnel cluster +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="async"} 100000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="client"} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="clientBlocking"} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="clientQuery"} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="io"} 2.147483647E9 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="offloadable"} 100000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="scheduled"} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="127.0.0.1:5801",type="system"} 2.147483647E9 +# TYPE hazelcast_executor_queueSize gauge +# HELP hazelcast_executor_queueSize The hazelcast executor queueSize of seatunnel cluster node +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="async"} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="client"} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="clientBlocking"} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="clientQuery"} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="io"} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="offloadable"} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="scheduled"} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="127.0.0.1:5801",type="system"} 0.0 +# TYPE hazelcast_partition_partitionCount gauge +# HELP hazelcast_partition_partitionCount The partitionCount of seatunnel cluster node +hazelcast_partition_partitionCount{cluster="seatunnel",address="127.0.0.1:5801"} 271.0 +# TYPE hazelcast_partition_activePartition gauge +# HELP hazelcast_partition_activePartition The activePartition of seatunnel cluster node +hazelcast_partition_activePartition{cluster="seatunnel",address="127.0.0.1:5801"} 271.0 +# TYPE hazelcast_partition_isClusterSafe gauge +# HELP hazelcast_partition_isClusterSafe Whether is cluster safe of partition +hazelcast_partition_isClusterSafe{cluster="seatunnel",address="127.0.0.1:5801"} 1.0 +# TYPE hazelcast_partition_isLocalMemberSafe gauge +# HELP hazelcast_partition_isLocalMemberSafe Whether is local member safe of partition +hazelcast_partition_isLocalMemberSafe{cluster="seatunnel",address="127.0.0.1:5801"} 1.0 +# TYPE jvm_memory_pool_allocated_bytes counter +# HELP jvm_memory_pool_allocated_bytes Total bytes allocated in a given JVM memory pool. Only updated after GC, not continuously. +jvm_memory_pool_allocated_bytes_total{pool="Code Cache"} 8118976.0 +jvm_memory_pool_allocated_bytes_created{pool="Code Cache"} 1.689671786522E9 +jvm_memory_pool_allocated_bytes_total{pool="PS Eden Space"} 2.39174704E8 +jvm_memory_pool_allocated_bytes_created{pool="PS Eden Space"} 1.689671786515E9 +jvm_memory_pool_allocated_bytes_total{pool="PS Old Gen"} 1.8928464E7 +jvm_memory_pool_allocated_bytes_created{pool="PS Old Gen"} 1.689671786522E9 +jvm_memory_pool_allocated_bytes_total{pool="PS Survivor Space"} 9945064.0 +jvm_memory_pool_allocated_bytes_created{pool="PS Survivor Space"} 1.689671786522E9 +jvm_memory_pool_allocated_bytes_total{pool="Compressed Class Space"} 4591800.0 +jvm_memory_pool_allocated_bytes_created{pool="Compressed Class Space"} 1.689671786522E9 +jvm_memory_pool_allocated_bytes_total{pool="Metaspace"} 3.2616592E7 +jvm_memory_pool_allocated_bytes_created{pool="Metaspace"} 1.689671786522E9 +# TYPE jvm_buffer_pool_used_bytes gauge +# HELP jvm_buffer_pool_used_bytes Used bytes of a given JVM buffer pool. +jvm_buffer_pool_used_bytes{pool="direct"} 412746.0 +jvm_buffer_pool_used_bytes{pool="mapped"} 0.0 +# TYPE jvm_buffer_pool_capacity_bytes gauge +# HELP jvm_buffer_pool_capacity_bytes Bytes capacity of a given JVM buffer pool. +jvm_buffer_pool_capacity_bytes{pool="direct"} 412743.0 +jvm_buffer_pool_capacity_bytes{pool="mapped"} 0.0 +# TYPE jvm_buffer_pool_used_buffers gauge +# HELP jvm_buffer_pool_used_buffers Used buffers of a given JVM buffer pool. +jvm_buffer_pool_used_buffers{pool="direct"} 7.0 +jvm_buffer_pool_used_buffers{pool="mapped"} 0.0 +# TYPE job_thread_pool_activeCount gauge +# HELP job_thread_pool_activeCount The activeCount of seatunnel coordinator job's executor cached thread pool +job_thread_pool_activeCount{cluster="seatunnel",address="127.0.0.1:5801"} 0.0 +# TYPE job_thread_pool_completedTask counter +# HELP job_thread_pool_completedTask The completedTask of seatunnel coordinator job's executor cached thread pool +job_thread_pool_completedTask_total{cluster="seatunnel",address="127.0.0.1:5801"} 0.0 +# TYPE job_thread_pool_corePoolSize gauge +# HELP job_thread_pool_corePoolSize The corePoolSize of seatunnel coordinator job's executor cached thread pool +job_thread_pool_corePoolSize{cluster="seatunnel",address="127.0.0.1:5801"} 0.0 +# TYPE job_thread_pool_maximumPoolSize gauge +# HELP job_thread_pool_maximumPoolSize The maximumPoolSize of seatunnel coordinator job's executor cached thread pool +job_thread_pool_maximumPoolSize{cluster="seatunnel",address="127.0.0.1:5801"} 2.147483647E9 +# TYPE job_thread_pool_poolSize gauge +# HELP job_thread_pool_poolSize The poolSize of seatunnel coordinator job's executor cached thread pool +job_thread_pool_poolSize{cluster="seatunnel",address="127.0.0.1:5801"} 0.0 +# TYPE job_thread_pool_task counter +# HELP job_thread_pool_task The taskCount of seatunnel coordinator job's executor cached thread pool +job_thread_pool_task_total{cluster="seatunnel",address="127.0.0.1:5801"} 0.0 +# TYPE jvm_gc_collection_seconds summary +# HELP jvm_gc_collection_seconds Time spent in a given JVM garbage collector in seconds. +jvm_gc_collection_seconds_count{gc="PS Scavenge"} 6.0 +jvm_gc_collection_seconds_sum{gc="PS Scavenge"} 0.161 +jvm_gc_collection_seconds_count{gc="PS MarkSweep"} 2.0 +jvm_gc_collection_seconds_sum{gc="PS MarkSweep"} 0.068 +# TYPE jvm_memory_objects_pending_finalization gauge +# HELP jvm_memory_objects_pending_finalization The number of objects waiting in the finalizer queue. +jvm_memory_objects_pending_finalization 0.0 +# TYPE jvm_memory_bytes_used gauge +# HELP jvm_memory_bytes_used Used bytes of a given JVM memory area. +jvm_memory_bytes_used{area="heap"} 2.72221784E8 +jvm_memory_bytes_used{area="nonheap"} 5.6619408E7 +# TYPE jvm_memory_bytes_committed gauge +# HELP jvm_memory_bytes_committed Committed (bytes) of a given JVM memory area. +jvm_memory_bytes_committed{area="heap"} 6.13941248E8 +jvm_memory_bytes_committed{area="nonheap"} 6.2537728E7 +# TYPE jvm_memory_bytes_max gauge +# HELP jvm_memory_bytes_max Max (bytes) of a given JVM memory area. +jvm_memory_bytes_max{area="heap"} 3.817865216E9 +jvm_memory_bytes_max{area="nonheap"} -1.0 +# TYPE jvm_memory_bytes_init gauge +# HELP jvm_memory_bytes_init Initial bytes of a given JVM memory area. +jvm_memory_bytes_init{area="heap"} 2.68435456E8 +jvm_memory_bytes_init{area="nonheap"} 2555904.0 +# TYPE jvm_memory_pool_bytes_used gauge +# HELP jvm_memory_pool_bytes_used Used bytes of a given JVM memory pool. +jvm_memory_pool_bytes_used{pool="Code Cache"} 1.0232064E7 +jvm_memory_pool_bytes_used{pool="Metaspace"} 4.0859424E7 +jvm_memory_pool_bytes_used{pool="Compressed Class Space"} 5527920.0 +jvm_memory_pool_bytes_used{pool="PS Eden Space"} 2.5329332E8 +jvm_memory_pool_bytes_used{pool="PS Survivor Space"} 0.0 +jvm_memory_pool_bytes_used{pool="PS Old Gen"} 1.8928464E7 +# TYPE jvm_memory_pool_bytes_committed gauge +# HELP jvm_memory_pool_bytes_committed Committed bytes of a given JVM memory pool. +jvm_memory_pool_bytes_committed{pool="Code Cache"} 1.081344E7 +jvm_memory_pool_bytes_committed{pool="Metaspace"} 4.52608E7 +jvm_memory_pool_bytes_committed{pool="Compressed Class Space"} 6463488.0 +jvm_memory_pool_bytes_committed{pool="PS Eden Space"} 4.90733568E8 +jvm_memory_pool_bytes_committed{pool="PS Survivor Space"} 9961472.0 +jvm_memory_pool_bytes_committed{pool="PS Old Gen"} 1.13246208E8 +# TYPE jvm_memory_pool_bytes_max gauge +# HELP jvm_memory_pool_bytes_max Max bytes of a given JVM memory pool. +jvm_memory_pool_bytes_max{pool="Code Cache"} 2.5165824E8 +jvm_memory_pool_bytes_max{pool="Metaspace"} -1.0 +jvm_memory_pool_bytes_max{pool="Compressed Class Space"} 1.073741824E9 +jvm_memory_pool_bytes_max{pool="PS Eden Space"} 1.40771328E9 +jvm_memory_pool_bytes_max{pool="PS Survivor Space"} 9961472.0 +jvm_memory_pool_bytes_max{pool="PS Old Gen"} 2.863661056E9 +# TYPE jvm_memory_pool_bytes_init gauge +# HELP jvm_memory_pool_bytes_init Initial bytes of a given JVM memory pool. +jvm_memory_pool_bytes_init{pool="Code Cache"} 2555904.0 +jvm_memory_pool_bytes_init{pool="Metaspace"} 0.0 +jvm_memory_pool_bytes_init{pool="Compressed Class Space"} 0.0 +jvm_memory_pool_bytes_init{pool="PS Eden Space"} 6.7108864E7 +jvm_memory_pool_bytes_init{pool="PS Survivor Space"} 1.1010048E7 +jvm_memory_pool_bytes_init{pool="PS Old Gen"} 1.79306496E8 +# TYPE jvm_memory_pool_collection_used_bytes gauge +# HELP jvm_memory_pool_collection_used_bytes Used bytes after last collection of a given JVM memory pool. +jvm_memory_pool_collection_used_bytes{pool="PS Eden Space"} 0.0 +jvm_memory_pool_collection_used_bytes{pool="PS Survivor Space"} 0.0 +jvm_memory_pool_collection_used_bytes{pool="PS Old Gen"} 1.8928464E7 +# TYPE jvm_memory_pool_collection_committed_bytes gauge +# HELP jvm_memory_pool_collection_committed_bytes Committed after last collection bytes of a given JVM memory pool. +jvm_memory_pool_collection_committed_bytes{pool="PS Eden Space"} 4.90733568E8 +jvm_memory_pool_collection_committed_bytes{pool="PS Survivor Space"} 9961472.0 +jvm_memory_pool_collection_committed_bytes{pool="PS Old Gen"} 1.13246208E8 +# TYPE jvm_memory_pool_collection_max_bytes gauge +# HELP jvm_memory_pool_collection_max_bytes Max bytes after last collection of a given JVM memory pool. +jvm_memory_pool_collection_max_bytes{pool="PS Eden Space"} 1.40771328E9 +jvm_memory_pool_collection_max_bytes{pool="PS Survivor Space"} 9961472.0 +jvm_memory_pool_collection_max_bytes{pool="PS Old Gen"} 2.863661056E9 +# TYPE jvm_memory_pool_collection_init_bytes gauge +# HELP jvm_memory_pool_collection_init_bytes Initial after last collection bytes of a given JVM memory pool. +jvm_memory_pool_collection_init_bytes{pool="PS Eden Space"} 6.7108864E7 +jvm_memory_pool_collection_init_bytes{pool="PS Survivor Space"} 1.1010048E7 +jvm_memory_pool_collection_init_bytes{pool="PS Old Gen"} 1.79306496E8 +# TYPE jvm info +# HELP jvm VM version info +jvm_info{runtime="Java(TM) SE Runtime Environment",vendor="Oracle Corporation",version="1.8.0_212-b10"} 1.0 +# EOF \ No newline at end of file diff --git a/docs/en/seatunnel-engine/prometheus/metrics.txt b/docs/en/seatunnel-engine/prometheus/metrics.txt new file mode 100644 index 00000000000..e43dc6934a5 --- /dev/null +++ b/docs/en/seatunnel-engine/prometheus/metrics.txt @@ -0,0 +1,275 @@ +# HELP job_count All job counts of seatunnel cluster +# TYPE job_count gauge +job_count{cluster="seatunnel",type="canceled",} 0.0 +job_count{cluster="seatunnel",type="cancelling",} 0.0 +job_count{cluster="seatunnel",type="created",} 0.0 +job_count{cluster="seatunnel",type="failed",} 0.0 +job_count{cluster="seatunnel",type="failing",} 0.0 +job_count{cluster="seatunnel",type="finished",} 0.0 +job_count{cluster="seatunnel",type="reconciling",} 0.0 +job_count{cluster="seatunnel",type="restarting",} 0.0 +job_count{cluster="seatunnel",type="running",} 0.0 +job_count{cluster="seatunnel",type="scheduled",} 0.0 +job_count{cluster="seatunnel",type="suspended",} 0.0 +# HELP jvm_classes_currently_loaded The number of classes that are currently loaded in the JVM +# TYPE jvm_classes_currently_loaded gauge +jvm_classes_currently_loaded 8186.0 +# HELP jvm_classes_loaded_total The total number of classes that have been loaded since the JVM has started execution +# TYPE jvm_classes_loaded_total counter +jvm_classes_loaded_total 8186.0 +# HELP jvm_classes_unloaded_total The total number of classes that have been unloaded since the JVM has started execution +# TYPE jvm_classes_unloaded_total counter +jvm_classes_unloaded_total 0.0 +# HELP job_thread_pool_activeCount The activeCount of seatunnel coordinator job's executor cached thread pool +# TYPE job_thread_pool_activeCount gauge +job_thread_pool_activeCount{cluster="seatunnel",address="[localhost]:5801",} 0.0 +# HELP job_thread_pool_completedTask_total The completedTask of seatunnel coordinator job's executor cached thread pool +# TYPE job_thread_pool_completedTask_total counter +job_thread_pool_completedTask_total{cluster="seatunnel",address="[localhost]:5801",} 0.0 +# HELP job_thread_pool_corePoolSize The corePoolSize of seatunnel coordinator job's executor cached thread pool +# TYPE job_thread_pool_corePoolSize gauge +job_thread_pool_corePoolSize{cluster="seatunnel",address="[localhost]:5801",} 0.0 +# HELP job_thread_pool_maximumPoolSize The maximumPoolSize of seatunnel coordinator job's executor cached thread pool +# TYPE job_thread_pool_maximumPoolSize gauge +job_thread_pool_maximumPoolSize{cluster="seatunnel",address="[localhost]:5801",} 2.147483647E9 +# HELP job_thread_pool_poolSize The poolSize of seatunnel coordinator job's executor cached thread pool +# TYPE job_thread_pool_poolSize gauge +job_thread_pool_poolSize{cluster="seatunnel",address="[localhost]:5801",} 0.0 +# HELP job_thread_pool_task_total The taskCount of seatunnel coordinator job's executor cached thread pool +# TYPE job_thread_pool_task_total counter +job_thread_pool_task_total{cluster="seatunnel",address="[localhost]:5801",} 0.0 +# HELP jvm_info VM version info +# TYPE jvm_info gauge +jvm_info{runtime="Java(TM) SE Runtime Environment",vendor="Oracle Corporation",version="1.8.0_212-b10",} 1.0 +# HELP jvm_buffer_pool_used_bytes Used bytes of a given JVM buffer pool. +# TYPE jvm_buffer_pool_used_bytes gauge +jvm_buffer_pool_used_bytes{pool="direct",} 150449.0 +jvm_buffer_pool_used_bytes{pool="mapped",} 0.0 +# HELP jvm_buffer_pool_capacity_bytes Bytes capacity of a given JVM buffer pool. +# TYPE jvm_buffer_pool_capacity_bytes gauge +jvm_buffer_pool_capacity_bytes{pool="direct",} 150447.0 +jvm_buffer_pool_capacity_bytes{pool="mapped",} 0.0 +# HELP jvm_buffer_pool_used_buffers Used buffers of a given JVM buffer pool. +# TYPE jvm_buffer_pool_used_buffers gauge +jvm_buffer_pool_used_buffers{pool="direct",} 5.0 +jvm_buffer_pool_used_buffers{pool="mapped",} 0.0 +# HELP jvm_memory_pool_allocated_bytes_total Total bytes allocated in a given JVM memory pool. Only updated after GC, not continuously. +# TYPE jvm_memory_pool_allocated_bytes_total counter +jvm_memory_pool_allocated_bytes_total{pool="Code Cache",} 8709504.0 +jvm_memory_pool_allocated_bytes_total{pool="PS Eden Space",} 4.9551024E7 +jvm_memory_pool_allocated_bytes_total{pool="PS Old Gen",} 1.1479696E7 +jvm_memory_pool_allocated_bytes_total{pool="PS Survivor Space",} 8889712.0 +jvm_memory_pool_allocated_bytes_total{pool="Compressed Class Space",} 4552568.0 +jvm_memory_pool_allocated_bytes_total{pool="Metaspace",} 3.256092E7 +# HELP jvm_gc_collection_seconds Time spent in a given JVM garbage collector in seconds. +# TYPE jvm_gc_collection_seconds summary +jvm_gc_collection_seconds_count{gc="PS Scavenge",} 8.0 +jvm_gc_collection_seconds_sum{gc="PS Scavenge",} 0.117 +jvm_gc_collection_seconds_count{gc="PS MarkSweep",} 2.0 +jvm_gc_collection_seconds_sum{gc="PS MarkSweep",} 0.071 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 13.922742 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.689662228826E9 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 113.0 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 10240.0 +# HELP jvm_memory_objects_pending_finalization The number of objects waiting in the finalizer queue. +# TYPE jvm_memory_objects_pending_finalization gauge +jvm_memory_objects_pending_finalization 0.0 +# HELP jvm_memory_bytes_used Used bytes of a given JVM memory area. +# TYPE jvm_memory_bytes_used gauge +jvm_memory_bytes_used{area="heap",} 2.10046672E8 +jvm_memory_bytes_used{area="nonheap",} 5.7037864E7 +# HELP jvm_memory_bytes_committed Committed (bytes) of a given JVM memory area. +# TYPE jvm_memory_bytes_committed gauge +jvm_memory_bytes_committed{area="heap",} 6.01358336E8 +jvm_memory_bytes_committed{area="nonheap",} 6.221824E7 +# HELP jvm_memory_bytes_max Max (bytes) of a given JVM memory area. +# TYPE jvm_memory_bytes_max gauge +jvm_memory_bytes_max{area="heap",} 3.817865216E9 +jvm_memory_bytes_max{area="nonheap",} -1.0 +# HELP jvm_memory_bytes_init Initial bytes of a given JVM memory area. +# TYPE jvm_memory_bytes_init gauge +jvm_memory_bytes_init{area="heap",} 2.68435456E8 +jvm_memory_bytes_init{area="nonheap",} 2555904.0 +# HELP jvm_memory_pool_bytes_used Used bytes of a given JVM memory pool. +# TYPE jvm_memory_pool_bytes_used gauge +jvm_memory_pool_bytes_used{pool="Code Cache",} 1.0777728E7 +jvm_memory_pool_bytes_used{pool="Metaspace",} 4.080908E7 +jvm_memory_pool_bytes_used{pool="Compressed Class Space",} 5451056.0 +jvm_memory_pool_bytes_used{pool="PS Eden Space",} 1.98927776E8 +jvm_memory_pool_bytes_used{pool="PS Survivor Space",} 0.0 +jvm_memory_pool_bytes_used{pool="PS Old Gen",} 1.1118896E7 +# HELP jvm_memory_pool_bytes_committed Committed bytes of a given JVM memory pool. +# TYPE jvm_memory_pool_bytes_committed gauge +jvm_memory_pool_bytes_committed{pool="Code Cache",} 1.1010048E7 +jvm_memory_pool_bytes_committed{pool="Metaspace",} 4.4867584E7 +jvm_memory_pool_bytes_committed{pool="Compressed Class Space",} 6340608.0 +jvm_memory_pool_bytes_committed{pool="PS Eden Space",} 4.9283072E8 +jvm_memory_pool_bytes_committed{pool="PS Survivor Space",} 5767168.0 +jvm_memory_pool_bytes_committed{pool="PS Old Gen",} 1.02760448E8 +# HELP jvm_memory_pool_bytes_max Max bytes of a given JVM memory pool. +# TYPE jvm_memory_pool_bytes_max gauge +jvm_memory_pool_bytes_max{pool="Code Cache",} 2.5165824E8 +jvm_memory_pool_bytes_max{pool="Metaspace",} -1.0 +jvm_memory_pool_bytes_max{pool="Compressed Class Space",} 1.073741824E9 +jvm_memory_pool_bytes_max{pool="PS Eden Space",} 1.414004736E9 +jvm_memory_pool_bytes_max{pool="PS Survivor Space",} 5767168.0 +jvm_memory_pool_bytes_max{pool="PS Old Gen",} 2.863661056E9 +# HELP jvm_memory_pool_bytes_init Initial bytes of a given JVM memory pool. +# TYPE jvm_memory_pool_bytes_init gauge +jvm_memory_pool_bytes_init{pool="Code Cache",} 2555904.0 +jvm_memory_pool_bytes_init{pool="Metaspace",} 0.0 +jvm_memory_pool_bytes_init{pool="Compressed Class Space",} 0.0 +jvm_memory_pool_bytes_init{pool="PS Eden Space",} 6.7108864E7 +jvm_memory_pool_bytes_init{pool="PS Survivor Space",} 1.1010048E7 +jvm_memory_pool_bytes_init{pool="PS Old Gen",} 1.79306496E8 +# HELP jvm_memory_pool_collection_used_bytes Used bytes after last collection of a given JVM memory pool. +# TYPE jvm_memory_pool_collection_used_bytes gauge +jvm_memory_pool_collection_used_bytes{pool="PS Eden Space",} 0.0 +jvm_memory_pool_collection_used_bytes{pool="PS Survivor Space",} 0.0 +jvm_memory_pool_collection_used_bytes{pool="PS Old Gen",} 1.1118896E7 +# HELP jvm_memory_pool_collection_committed_bytes Committed after last collection bytes of a given JVM memory pool. +# TYPE jvm_memory_pool_collection_committed_bytes gauge +jvm_memory_pool_collection_committed_bytes{pool="PS Eden Space",} 4.9283072E8 +jvm_memory_pool_collection_committed_bytes{pool="PS Survivor Space",} 5767168.0 +jvm_memory_pool_collection_committed_bytes{pool="PS Old Gen",} 1.02760448E8 +# HELP jvm_memory_pool_collection_max_bytes Max bytes after last collection of a given JVM memory pool. +# TYPE jvm_memory_pool_collection_max_bytes gauge +jvm_memory_pool_collection_max_bytes{pool="PS Eden Space",} 1.414004736E9 +jvm_memory_pool_collection_max_bytes{pool="PS Survivor Space",} 5767168.0 +jvm_memory_pool_collection_max_bytes{pool="PS Old Gen",} 2.863661056E9 +# HELP jvm_memory_pool_collection_init_bytes Initial after last collection bytes of a given JVM memory pool. +# TYPE jvm_memory_pool_collection_init_bytes gauge +jvm_memory_pool_collection_init_bytes{pool="PS Eden Space",} 6.7108864E7 +jvm_memory_pool_collection_init_bytes{pool="PS Survivor Space",} 1.1010048E7 +jvm_memory_pool_collection_init_bytes{pool="PS Old Gen",} 1.79306496E8 +# HELP node_state Whether is up of seatunnel node +# TYPE node_state gauge +node_state{cluster="seatunnel",address="[localhost]:5801",} 1.0 +# HELP node_count Cluster node total count +# TYPE node_count gauge +node_count{cluster="seatunnel",} 1.0 +# HELP hazelcast_executor_executedCount The hazelcast executor executedCount of seatunnel cluster node +# TYPE hazelcast_executor_executedCount gauge +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="async",} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="client",} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="clientBlocking",} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="clientQuery",} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="io",} 128.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="offloadable",} 0.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="scheduled",} 402.0 +hazelcast_executor_executedCount{cluster="seatunnel",address="[localhost]:5801",type="system",} 0.0 +# HELP hazelcast_executor_isShutdown The hazelcast executor isShutdown of seatunnel cluster node +# TYPE hazelcast_executor_isShutdown gauge +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="async",} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="client",} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="clientBlocking",} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="clientQuery",} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="io",} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="offloadable",} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="scheduled",} 0.0 +hazelcast_executor_isShutdown{cluster="seatunnel",address="[localhost]:5801",type="system",} 0.0 +# HELP hazelcast_executor_isTerminated The hazelcast executor isTerminated of seatunnel cluster node +# TYPE hazelcast_executor_isTerminated gauge +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="async",} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="client",} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="clientBlocking",} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="clientQuery",} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="io",} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="offloadable",} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="scheduled",} 0.0 +hazelcast_executor_isTerminated{cluster="seatunnel",address="[localhost]:5801",type="system",} 0.0 +# HELP hazelcast_executor_maxPoolSize The hazelcast executor maxPoolSize of seatunnel cluster node +# TYPE hazelcast_executor_maxPoolSize gauge +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="async",} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="client",} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="clientBlocking",} 160.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="clientQuery",} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="io",} 16.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="offloadable",} 8.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="scheduled",} 16.0 +hazelcast_executor_maxPoolSize{cluster="seatunnel",address="[localhost]:5801",type="system",} 8.0 +# HELP hazelcast_executor_poolSize The hazelcast executor poolSize of seatunnel cluster node +# TYPE hazelcast_executor_poolSize gauge +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="async",} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="client",} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="clientBlocking",} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="clientQuery",} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="io",} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="offloadable",} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="scheduled",} 0.0 +hazelcast_executor_poolSize{cluster="seatunnel",address="[localhost]:5801",type="system",} 0.0 +# HELP hazelcast_executor_queueRemainingCapacity The hazelcast executor queueRemainingCapacity of seatunnel cluster +# TYPE hazelcast_executor_queueRemainingCapacity gauge +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="async",} 100000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="client",} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="clientBlocking",} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="clientQuery",} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="io",} 2.147483647E9 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="offloadable",} 100000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="scheduled",} 800000.0 +hazelcast_executor_queueRemainingCapacity{cluster="seatunnel",address="[localhost]:5801",type="system",} 2.147483647E9 +# HELP hazelcast_executor_queueSize The hazelcast executor queueSize of seatunnel cluster node +# TYPE hazelcast_executor_queueSize gauge +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="async",} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="client",} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="clientBlocking",} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="clientQuery",} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="io",} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="offloadable",} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="scheduled",} 0.0 +hazelcast_executor_queueSize{cluster="seatunnel",address="[localhost]:5801",type="system",} 0.0 +# HELP hazelcast_partition_partitionCount The partitionCount of seatunnel cluster node +# TYPE hazelcast_partition_partitionCount gauge +hazelcast_partition_partitionCount{cluster="seatunnel",address="[localhost]:5801",} 271.0 +# HELP hazelcast_partition_activePartition The activePartition of seatunnel cluster node +# TYPE hazelcast_partition_activePartition gauge +hazelcast_partition_activePartition{cluster="seatunnel",address="[localhost]:5801",} 271.0 +# HELP hazelcast_partition_isClusterSafe Whether is cluster safe of partition +# TYPE hazelcast_partition_isClusterSafe gauge +hazelcast_partition_isClusterSafe{cluster="seatunnel",address="[localhost]:5801",} 1.0 +# HELP hazelcast_partition_isLocalMemberSafe Whether is local member safe of partition +# TYPE hazelcast_partition_isLocalMemberSafe gauge +hazelcast_partition_isLocalMemberSafe{cluster="seatunnel",address="[localhost]:5801",} 1.0 +# HELP jvm_threads_current Current thread count of a JVM +# TYPE jvm_threads_current gauge +jvm_threads_current 114.0 +# HELP jvm_threads_daemon Daemon thread count of a JVM +# TYPE jvm_threads_daemon gauge +jvm_threads_daemon 5.0 +# HELP jvm_threads_peak Peak thread count of a JVM +# TYPE jvm_threads_peak gauge +jvm_threads_peak 116.0 +# HELP jvm_threads_started_total Started thread count of a JVM +# TYPE jvm_threads_started_total counter +jvm_threads_started_total 119.0 +# HELP jvm_threads_deadlocked Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers +# TYPE jvm_threads_deadlocked gauge +jvm_threads_deadlocked 0.0 +# HELP jvm_threads_deadlocked_monitor Cycles of JVM-threads that are in deadlock waiting to acquire object monitors +# TYPE jvm_threads_deadlocked_monitor gauge +jvm_threads_deadlocked_monitor 0.0 +# HELP jvm_threads_state Current count of threads by state +# TYPE jvm_threads_state gauge +jvm_threads_state{state="NEW",} 0.0 +jvm_threads_state{state="TERMINATED",} 0.0 +jvm_threads_state{state="RUNNABLE",} 10.0 +jvm_threads_state{state="BLOCKED",} 0.0 +jvm_threads_state{state="WAITING",} 75.0 +jvm_threads_state{state="TIMED_WAITING",} 29.0 +jvm_threads_state{state="UNKNOWN",} 0.0 +# HELP jvm_memory_pool_allocated_bytes_created Total bytes allocated in a given JVM memory pool. Only updated after GC, not continuously. +# TYPE jvm_memory_pool_allocated_bytes_created gauge +jvm_memory_pool_allocated_bytes_created{pool="Code Cache",} 1.689662236806E9 +jvm_memory_pool_allocated_bytes_created{pool="PS Eden Space",} 1.689662236757E9 +jvm_memory_pool_allocated_bytes_created{pool="PS Old Gen",} 1.689662236806E9 +jvm_memory_pool_allocated_bytes_created{pool="PS Survivor Space",} 1.689662236806E9 +jvm_memory_pool_allocated_bytes_created{pool="Compressed Class Space",} 1.689662236806E9 +jvm_memory_pool_allocated_bytes_created{pool="Metaspace",} 1.689662236806E9 \ No newline at end of file diff --git a/docs/en/seatunnel-engine/telemetry.md b/docs/en/seatunnel-engine/telemetry.md new file mode 100644 index 00000000000..4ac882bc394 --- /dev/null +++ b/docs/en/seatunnel-engine/telemetry.md @@ -0,0 +1,148 @@ +--- + +sidebar_position: 8 +------------------- + +# Telemetry + +Integrating `OpenTelemetry` through `Prometheus-exports` can better seamlessly connect to related monitoring platforms such as Prometheus and Grafana, improving the ability to monitor and alarm of the Seatunnel cluster. + +You can configure telemetry's configurations in the `seatunnel.yaml` file. + +The following is an example declarative configuration. + +```yaml +seatunnel: + engine: + telemetry: + metric: + enabled: true # Whether open metrics export +``` + +## Metrics + +The [metric text of prometheus](./prometheus/metrics.txt),which get from `http:{instanceHost}:5801/hazelcast/rest/instance/metrics`. + +The [metric text of openMetrics](./openmetrics/metrics.txt),which get from `http:{instanceHost}:5801/hazelcast/rest/instance/openmetrics`. + +Available metrics include the following categories. + +Note: All metrics both have the same labelName `cluster`, that's value is the config of `hazelcast.cluster-name`. + +### Node Metrics + +| MetricName | Type | Labels | DESCRIPTION | +|-------------------------------------------|-------|------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------| +| cluster_info | Gauge | **hazelcastVersion**, the version of hazelcast. **master**, seatunnel master address. | Cluster info | +| cluster_time | Gauge | **hazelcastVersion**, the version of hazelcast. | Cluster time | +| node_count | Gauge | - | Cluster node total count | +| node_state | Gauge | **address**, server instance address,for example: "127.0.0.1:5801" | Whether is up of seatunnel node | +| hazelcast_executor_executedCount | Gauge | **type**, the type of executor, including: "async" "client" "clientBlocking" "clientQuery" "io" "offloadable" "scheduled" "system" | The hazelcast executor executedCount of seatunnel cluster node | +| hazelcast_executor_isShutdown | Gauge | **type**, the type of executor, including: "async" "client" "clientBlocking" "clientQuery" "io" "offloadable" "scheduled" "system" | The hazelcast executor isShutdown of seatunnel cluster node | +| hazelcast_executor_isTerminated | Gauge | **type**, the type of executor, including: "async" "client" "clientBlocking" "clientQuery" "io" "offloadable" "scheduled" "system" | The hazelcast executor isTerminated of seatunnel cluster node | +| hazelcast_executor_maxPoolSize | Gauge | **type**, the type of executor, including: "async" "client" "clientBlocking" "clientQuery" "io" "offloadable" "scheduled" "system" | The hazelcast executor maxPoolSize of seatunnel cluster node | +| hazelcast_executor_poolSize | Gauge | **type**, the type of executor, including: "async" "client" "clientBlocking" "clientQuery" "io" "offloadable" "scheduled" "system" | The hazelcast executor poolSize of seatunnel cluster node | +| hazelcast_executor_queueRemainingCapacity | Gauge | **type**, the type of executor, including: "async" "client" "clientBlocking" "clientQuery" "io" "offloadable" "scheduled" "system" | The hazelcast executor queueRemainingCapacity of seatunnel cluster node | +| hazelcast_executor_queueSize | Gauge | **type**, the type of executor, including: "async" "client" "clientBlocking" "clientQuery" "io" "offloadable" "scheduled" "system" | The hazelcast executor queueSize of seatunnel cluster node | +| hazelcast_partition_partitionCount | Gauge | - | The partitionCount of seatunnel cluster node | +| hazelcast_partition_activePartition | Gauge | - | The activePartition of seatunnel cluster node | +| hazelcast_partition_isClusterSafe | Gauge | - | Weather is cluster safe of partition | +| hazelcast_partition_isLocalMemberSafe | Gauge | - | Weather is local member safe of partition | + +### Thread Pool Status + +| MetricName | Type | Labels | DESCRIPTION | +|-------------------------------------|---------|--------------------------------------------------------------------|--------------------------------------------------------------------------------| +| job_thread_pool_activeCount | Gauge | **address**, server instance address,for example: "127.0.0.1:5801" | The activeCount of seatunnel coordinator job's executor cached thread pool | +| job_thread_pool_corePoolSize | Gauge | **address**, server instance address,for example: "127.0.0.1:5801" | The corePoolSize of seatunnel coordinator job's executor cached thread pool | +| job_thread_pool_maximumPoolSize | Gauge | **address**, server instance address,for example: "127.0.0.1:5801" | The maximumPoolSize of seatunnel coordinator job's executor cached thread pool | +| job_thread_pool_poolSize | Gauge | **address**, server instance address,for example: "127.0.0.1:5801" | The poolSize of seatunnel coordinator job's executor cached thread pool | +| job_thread_pool_completedTask_total | Counter | **address**, server instance address,for example: "127.0.0.1:5801" | The completedTask of seatunnel coordinator job's executor cached thread pool | +| job_thread_pool_task_total | Counter | **address**, server instance address,for example: "127.0.0.1:5801" | The taskCount of seatunnel coordinator job's executor cached thread pool | + +### Job info detail + +| MetricName | Type | Labels | DESCRIPTION | +|------------|-------|-----------------------------------------------------------------------------------------------------------------------------|-------------------------------------| +| job_count | Gauge | **type**, the type of job, including: "canceled" "cancelling" "created" "failed" "failing" "finished" "running" "scheduled" | All job counts of seatunnel cluster | + +### JVM Metrics + +| MetricName | Type | Labels | DESCRIPTION | +|--------------------------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| jvm_threads_current | Gauge | - | Current thread count of a JVM | +| jvm_threads_daemon | Gauge | - | Daemon thread count of a JVM | +| jvm_threads_peak | Gauge | - | Peak thread count of a JVM | +| jvm_threads_started_total | Counter | - | Started thread count of a JVM | +| jvm_threads_deadlocked | Gauge | - | Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers | +| jvm_threads_deadlocked_monitor | Gauge | - | Cycles of JVM-threads that are in deadlock waiting to acquire object monitors | +| jvm_threads_state | Gauge | **state**, the state of jvm thread, including: "NEW" "TERMINATED" "RUNNABLE" "BLOCKED" "WAITING" "TIMED_WAITING" "UNKNOWN" | Current count of threads by state | +| jvm_classes_currently_loaded | Gauge | - | The number of classes that are currently loaded in the JVM | +| jvm_classes_loaded_total | Counter | - | The total number of classes that have been loaded since the JVM has started execution | +| jvm_classes_unloaded_total | Counter | - | The total number of classes that have been unloaded since the JVM has started execution | +| jvm_memory_pool_allocated_bytes_total | Counter | **pool**,including: "Code Cache" "PS Eden Space" "PS Old Ge" "PS Survivor Space" "Compressed Class Space" "Metaspace" | Total bytes allocated in a given JVM memory pool. Only updated after GC, not continuously | +| jvm_gc_collection_seconds_count | Summary | **gc**,including: "PS Scavenge" "PS MarkSweep" | Time spent in a given JVM garbage collector in seconds | +| jvm_gc_collection_seconds_sum | Summary | **gc**,including: "PS Scavenge" "PS MarkSweep" | Time spent in a given JVM garbage collector in seconds | +| jvm_info | Gauge | **runtime**, for example: "Java(TM) SE Runtime Environment". **vendor**, for example: "Oracle Corporation". **version** ,for example: "1.8.0_212-b10" | VM version info | +| process_cpu_seconds_total | Counter | - | Total user and system CPU time spent in seconds | +| process_start_time_seconds | Gauge | - | Start time of the process since unix epoch in seconds | +| process_open_fds | Gauge | - | Number of open file descriptors | +| process_max_fds | Gauge | - | Maximum number of open file descriptors | +| jvm_memory_objects_pending_finalization | Gauge | - | The number of objects waiting in the finalizer queue | +| jvm_memory_bytes_used | Gauge | **area**, including: "heap" "noheap" | Used bytes of a given JVM memory area | +| jvm_memory_bytes_committed | Gauge | **area**, including: "heap" "noheap" | Committed (bytes) of a given JVM memory area | +| jvm_memory_bytes_max | Gauge | **area**, including:"heap" "noheap" | Max (bytes) of a given JVM memory area | +| jvm_memory_bytes_init | Gauge | **area**, including:"heap" "noheap" | Initial bytes of a given JVM memory area | +| jvm_memory_pool_bytes_used | Gauge | **pool**, including: "Code Cache" "PS Eden Space" "PS Old Ge" "PS Survivor Space" "Compressed Class Space" "Metaspace" | Used bytes of a given JVM memory pool | +| jvm_memory_pool_bytes_committed | Gauge | **pool**, including: "Code Cache" "PS Eden Space" "PS Old Ge" "PS Survivor Space" "Compressed Class Space" "Metaspace" | Committed bytes of a given JVM memory pool | +| jvm_memory_pool_bytes_max | Gauge | **pool**, including: "Code Cache" "PS Eden Space" "PS Old Ge" "PS Survivor Space" "Compressed Class Space" "Metaspace" | Max bytes of a given JVM memory pool | +| jvm_memory_pool_bytes_init | Gauge | **pool**, including: "Code Cache" "PS Eden Space" "PS Old Ge" "PS Survivor Space" "Compressed Class Space" "Metaspace" | Initial bytes of a given JVM memory pool | +| jvm_memory_pool_allocated_bytes_created | Gauge | **pool**, including: "Code Cache" "PS Eden Space" "PS Old Ge" "PS Survivor Space" "Compressed Class Space" "Metaspace" | Total bytes allocated in a given JVM memory pool. Only updated after GC, not continuously | +| jvm_memory_pool_collection_used_bytes | Gauge | **pool**, including: "PS Eden Space" "PS Old Ge" "PS Survivor Space" | Used bytes after last collection of a given JVM memory pool | +| jvm_memory_pool_collection_committed_bytes | Gauge | **pool**, including: "PS Eden Space" "PS Old Ge" "PS Survivor Space" | Committed after last collection bytes of a given JVM memory pool | +| jvm_memory_pool_collection_max_bytes | Gauge | **pool**, including: "PS Eden Space" "PS Old Ge" "PS Survivor Space" | Max bytes after last collection of a given JVM memory pool | +| jvm_memory_pool_collection_init_bytes | Gauge | **pool**, including: "PS Eden Space" "PS Old Ge" "PS Survivor Space" | Initial after last collection bytes of a given JVM memory pool | +| jvm_buffer_pool_used_bytes | Gauge | **pool**, including: "direct" "mapped" | Used bytes of a given JVM buffer pool | +| jvm_buffer_pool_capacity_bytes | Gauge | **pool**, including: "direct" "mapped" | Bytes capacity of a given JVM buffer pool | +| jvm_buffer_pool_used_buffers | Gauge | **pool**, including: "direct" "mapped" | Used buffers of a given JVM buffer pool | + +### Cluster Monitoring By Prometheus & Grafana + +#### Install Prometheus + +For a guide on how to set up Prometheus server go to the [Installation](https://prometheus.io/docs/prometheus/latest/installation) + +#### Configuration Prometheus + +Add seatunnel instance metric exports into `/etc/prometheus/prometheus.yaml`. For example: + +```yaml +global: + # How frequently to scrape targets from this job. + scrape_interval: 15s + +scrape_configs: + # The job name assigned to scraped metrics by default. + - job_name: 'seatunnel' + scrape_interval: 5s + # Metrics export path + metrics_path: /hazelcast/rest/instance/metrics + # List of labeled statically configured targets for this job. + static_configs: + # The targets specified by the static config. + - targets: ['localhost:5801'] + # Labels assigned to all metrics scraped from the targets. + # labels: [:] +``` + +#### Install Grafana + +For a guide on how to set up Grafana server go to the [Installation](https://grafana.com/docs/grafana/latest/setup-grafana/installation) + +#### Monitoring Dashboard + +- Add Prometheus DataSource on Grafana. +- Import `Seatunnel Cluster` monitoring dashboard by [Dashboard JSON](./grafana/dashboard.json) into Grafana. + +The [effect image](../images/cluster_monitoring.png) of the dashboard + diff --git a/pom.xml b/pom.xml index f3e1dd5594c..a1c567641cf 100644 --- a/pom.xml +++ b/pom.xml @@ -144,6 +144,8 @@ true false + + 0.16.0 @@ -461,6 +463,25 @@ provided + + + io.prometheus + simpleclient + ${prometheus.simpleclient.version} + + + + io.prometheus + simpleclient_hotspot + ${prometheus.simpleclient.version} + + + + io.prometheus + simpleclient_httpserver + ${prometheus.simpleclient.version} + + diff --git a/release-note.md b/release-note.md index 1e352b80c71..b586ccd8109 100644 --- a/release-note.md +++ b/release-note.md @@ -187,6 +187,7 @@ - [Zeta] Reduce the frequency of fetching data from imap (#4851) - [Zeta] Add OSS support for Imap storage to cluster-mode type (#4683) - [Zeta] Improve local mode startup request ports (#4660) +- [Zeta] Support exposing monitoring metrics by prometheus exporter protocol (#5070) ## Docs diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/SeaTunnel.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/SeaTunnel.java index e003bf2d206..22afb8fc9dc 100644 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/SeaTunnel.java +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/SeaTunnel.java @@ -26,6 +26,8 @@ import lombok.extern.slf4j.Slf4j; +import java.io.IOException; + @Slf4j public class SeaTunnel { @@ -35,7 +37,8 @@ public class SeaTunnel { * @param command commandArgs * @param commandType */ - public static void run(Command command) throws CommandException { + public static void run(Command command) + throws CommandException, IOException { try { command.execute(); } catch (ConfigRuntimeException e) { diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/Command.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/Command.java index 1d510ab984b..e419c89b46b 100644 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/Command.java +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/command/Command.java @@ -20,6 +20,8 @@ import org.apache.seatunnel.core.starter.exception.CommandExecuteException; import org.apache.seatunnel.core.starter.exception.ConfigCheckException; +import java.io.IOException; + /** * Command interface, only has one method {@link Command#execute()}, used to execute the command * @@ -28,5 +30,5 @@ @FunctionalInterface public interface Command { - void execute() throws CommandExecuteException, ConfigCheckException; + void execute() throws CommandExecuteException, ConfigCheckException, IOException; } diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java index 8d1b434801d..460b4f2874d 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-13-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java @@ -23,8 +23,10 @@ import org.apache.seatunnel.core.starter.flink.args.FlinkCommandArgs; import org.apache.seatunnel.core.starter.utils.CommandLineUtils; +import java.io.IOException; + public class SeaTunnelFlink { - public static void main(String[] args) throws CommandException { + public static void main(String[] args) throws CommandException, IOException { FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parse( args, diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java index 1595da686a7..e7baa0e7811 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/SeaTunnelFlink.java @@ -23,8 +23,10 @@ import org.apache.seatunnel.core.starter.flink.args.FlinkCommandArgs; import org.apache.seatunnel.core.starter.utils.CommandLineUtils; +import java.io.IOException; + public class SeaTunnelFlink { - public static void main(String[] args) throws CommandException { + public static void main(String[] args) throws CommandException, IOException { FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parse( args, diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java index ca7b2ed4be1..2de9c4ba930 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-2-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java @@ -23,9 +23,11 @@ import org.apache.seatunnel.core.starter.spark.args.SparkCommandArgs; import org.apache.seatunnel.core.starter.utils.CommandLineUtils; +import java.io.IOException; + public class SeaTunnelSpark { - public static void main(String[] args) throws CommandException { + public static void main(String[] args) throws CommandException, IOException { SparkCommandArgs sparkCommandArgs = CommandLineUtils.parse( args, diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java index 9b3fde6fd10..85cf2005d51 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/SeaTunnelSpark.java @@ -23,9 +23,11 @@ import org.apache.seatunnel.core.starter.spark.args.SparkCommandArgs; import org.apache.seatunnel.core.starter.utils.CommandLineUtils; +import java.io.IOException; + public class SeaTunnelSpark { - public static void main(String[] args) throws CommandException { + public static void main(String[] args) throws CommandException, IOException { SparkCommandArgs sparkCommandArgs = CommandLineUtils.parse( args, diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClient.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClient.java index 7a37fd340c3..3a755f808ab 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClient.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClient.java @@ -23,8 +23,10 @@ import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; import org.apache.seatunnel.core.starter.utils.CommandLineUtils; +import java.io.IOException; + public class SeaTunnelClient { - public static void main(String[] args) throws CommandException { + public static void main(String[] args) throws CommandException, IOException { ClientCommandArgs clientCommandArgs = CommandLineUtils.parse( args, diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java index 96a3e32e51a..ae3284fc010 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java @@ -23,8 +23,10 @@ import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; import org.apache.seatunnel.core.starter.utils.CommandLineUtils; +import java.io.IOException; + public class SeaTunnelServer { - public static void main(String[] args) throws CommandException { + public static void main(String[] args) throws CommandException, IOException { ServerCommandArgs serverCommandArgs = CommandLineUtils.parse( args, diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java index aead1f8a964..ec3872e449f 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java @@ -21,12 +21,10 @@ import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; import org.apache.seatunnel.engine.common.config.ConfigProvider; import org.apache.seatunnel.engine.common.config.SeaTunnelConfig; -import org.apache.seatunnel.engine.server.SeaTunnelNodeContext; +import org.apache.seatunnel.engine.server.SeaTunnelServerStarter; import org.apache.commons.lang3.StringUtils; -import com.hazelcast.instance.impl.HazelcastInstanceFactory; - /** This command is used to execute the SeaTunnel engine job by SeaTunnel API. */ public class ServerExecuteCommand implements Command { @@ -42,9 +40,6 @@ public void execute() { if (StringUtils.isNotEmpty(serverCommandArgs.getClusterName())) { seaTunnelConfig.getHazelcastConfig().setClusterName(serverCommandArgs.getClusterName()); } - HazelcastInstanceFactory.newHazelcastInstance( - seaTunnelConfig.getHazelcastConfig(), - Thread.currentThread().getName(), - new SeaTunnelNodeContext(seaTunnelConfig)); + SeaTunnelServerStarter.createHazelcastInstance(); } } diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE index 310d0618335..44ad2d2cd61 100644 --- a/seatunnel-dist/release-docs/LICENSE +++ b/seatunnel-dist/release-docs/LICENSE @@ -277,6 +277,13 @@ The text of each license is the standard Apache 2.0 license. (Apache-2.0) accessors-smart (net.minidev:accessors-smart:2.4.7 - https://mvnrepository.com/artifact/net.minidev/accessors-smart) (Apache-2.0) json-smart (net.minidev:json-smart:2.4.7 - https://mvnrepository.com/artifact/net.minidev/json-smart) (Apache-2.0) json-path (com.jayway.jsonpath:json-path:2.7.0 - https://mvnrepository.com/artifact/com.jayway.jsonpath/json-path) + (The Apache Software License, Version 2.0) Prometheus Java Simpleclient (io.prometheus:simpleclient:0.16.0 - https://mvnrepository.com/artifact/io.prometheus/simpleclient/0.16.0) + (The Apache Software License, Version 2.0) Prometheus Java Simpleclient Common (io.prometheus:simpleclient_common:0.16.0 - https://mvnrepository.com/artifact/io.prometheus/simpleclient_common/0.16.0) + (The Apache Software License, Version 2.0) Prometheus Java Simpleclient Hotspot (io.prometheus:simpleclient_hotspot:0.16.0 - https://mvnrepository.com/artifact/io.prometheus/simpleclient_hotspot/0.16.0) + (The Apache Software License, Version 2.0) Prometheus Java Simpleclient Httpserver (io.prometheus:simpleclient_httpserver:0.16.0 - https://mvnrepository.com/artifact/io.prometheus/simpleclient_httpserver/0.16.0) + (The Apache Software License, Version 2.0) Prometheus Java Span Context Supplier - Common (io.prometheus:simpleclient_tracer_common:0.16.0 - https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_common/0.16.0) + (The Apache Software License, Version 2.0) Prometheus Java Span Context Supplier - OpenTelemetry (io.prometheus:simpleclient_tracer_otel:0.16.0 - https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel/0.16.0) + (The Apache Software License, Version 2.0) Prometheus Java Span Context Supplier - OpenTelemetry Agent (io.prometheus:simpleclient_tracer_otel_agent:0.16.0 - https://mvnrepository.com/artifact/io.prometheus/simpleclient_tracer_otel_agent/0.16.0) ======================================================================== MOZILLA PUBLIC LICENSE License diff --git a/seatunnel-dist/release-docs/NOTICE b/seatunnel-dist/release-docs/NOTICE index 4bed2d4be19..cbbf8cc6549 100644 --- a/seatunnel-dist/release-docs/NOTICE +++ b/seatunnel-dist/release-docs/NOTICE @@ -706,4 +706,20 @@ and updated within the WildFly project (https://github.com/wildfly/wildfly). The class org.apache.calcite.linq4j.tree.ConstantExpression contains code originating from the Calcite project (https://github.com/apache/calcite). +========================================================================= + +Prometheus NOTICE + +========================================================================= +Prometheus instrumentation library for JVM applications +Copyright 2012-2015 The Prometheus Authors + +This product includes software developed at +Boxever Ltd. (http://www.boxever.com/). + +This product includes software developed at +SoundCloud Ltd. (http://soundcloud.com/). + +This product includes software developed as part of the +Ocelli project by Netflix Inc. (https://github.com/Netflix/ocelli/). ========================================================================= \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TelemetryApiIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TelemetryApiIT.java new file mode 100644 index 00000000000..b134473467e --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TelemetryApiIT.java @@ -0,0 +1,522 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.e2e; + +import org.apache.seatunnel.common.config.Common; +import org.apache.seatunnel.common.config.DeployMode; +import org.apache.seatunnel.engine.client.SeaTunnelClient; +import org.apache.seatunnel.engine.client.job.ClientJobExecutionEnvironment; +import org.apache.seatunnel.engine.client.job.ClientJobProxy; +import org.apache.seatunnel.engine.common.config.ConfigProvider; +import org.apache.seatunnel.engine.common.config.JobConfig; +import org.apache.seatunnel.engine.common.config.SeaTunnelConfig; +import org.apache.seatunnel.engine.core.job.JobStatus; +import org.apache.seatunnel.engine.server.SeaTunnelServerStarter; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import com.hazelcast.client.config.ClientConfig; +import com.hazelcast.instance.impl.HazelcastInstanceImpl; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.TimeUnit; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.matchesRegex; + +@Slf4j +public class TelemetryApiIT { + + private static final String HOST = "http://localhost:"; + + private static ClientJobProxy clientJobProxy; + + private static HazelcastInstanceImpl hazelcastInstance; + + private static String testClusterName; + + @BeforeAll + static void beforeClass() throws Exception { + testClusterName = TestUtils.getClusterName("TelemetryApiIT"); + SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); + seaTunnelConfig.getHazelcastConfig().setClusterName(testClusterName); + hazelcastInstance = SeaTunnelServerStarter.createHazelcastInstance(seaTunnelConfig); + // createTelemetryInstance + SeaTunnelServerStarter.initTelemetryInstance(hazelcastInstance.node); + Common.setDeployMode(DeployMode.CLIENT); + String filePath = TestUtils.getResource("stream_fakesource_to_file.conf"); + JobConfig jobConfig = new JobConfig(); + jobConfig.setName("fake_to_file"); + + ClientConfig clientConfig = ConfigProvider.locateAndGetClientConfig(); + clientConfig.setClusterName(testClusterName); + SeaTunnelClient engineClient = new SeaTunnelClient(clientConfig); + ClientJobExecutionEnvironment jobExecutionEnv = + engineClient.createExecutionContext(filePath, jobConfig, seaTunnelConfig); + + clientJobProxy = jobExecutionEnv.execute(); + + Awaitility.await() + .atMost(2, TimeUnit.MINUTES) + .untilAsserted( + () -> + Assertions.assertEquals( + JobStatus.RUNNING, clientJobProxy.getJobStatus())); + } + + @Test + public void testGetMetrics() throws InterruptedException { + given().get( + HOST + + hazelcastInstance + .getCluster() + .getLocalMember() + .getAddress() + .getPort() + + "/hazelcast/rest/instance/metrics") + .then() + .statusCode(200) + // Use regular expressions to verify whether the response body is the indicator data + // of Prometheus + // Metric data is usually multi-line, use newlines for validation + .body(matchesRegex("(?s)^.*# HELP.*# TYPE.*$")) + // Verify that the response body contains a specific metric + // JVM metrics + .body(containsString("jvm_threads")) + .body(containsString("jvm_memory_pool")) + .body(containsString("jvm_gc")) + .body(containsString("jvm_info")) + .body(containsString("jvm_memory_bytes")) + .body(containsString("jvm_classes")) + .body(containsString("jvm_buffer_pool")) + .body(containsString("process_start")) + // cluster_info + .body(containsString("cluster_info{cluster=\"" + testClusterName)) + // cluster_time + .body(containsString("cluster_time{cluster=\"" + testClusterName)) + // Job thread pool metrics + .body( + matchesRegex( + "(?s)^.*job_thread_pool_activeCount\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + .body( + matchesRegex( + "(?s)^.*job_thread_pool_completedTask_total\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + .body( + matchesRegex( + "(?s)^.*job_thread_pool_corePoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + .body( + matchesRegex( + "(?s)^.*job_thread_pool_maximumPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + .body( + matchesRegex( + "(?s)^.*job_thread_pool_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + .body( + matchesRegex( + "(?s)^.*job_thread_pool_task_total\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + // Job count metrics + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"canceled\",} 0.0")) + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"cancelling\",} 0.0")) + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"created\",} 0.0")) + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"failed\",} 0.0")) + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"failing\",} 0.0")) + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"finished\",} 0.0")) + // Running job count is 1 + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"running\",} 1.0")) + .body( + containsString( + "job_count{cluster=\"" + + testClusterName + + "\",type=\"scheduled\",} 0.0")) + // Node + .body( + matchesRegex( + "(?s)^.*node_state\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + // hazelcast_executor_executedCount + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"async\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"client\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientBlocking\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientQuery\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"io\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"offloadable\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"scheduled\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_executedCount\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"system\".*$")) + // hazelcast_executor_isShutdown + + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"async\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"client\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientBlocking\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientQuery\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"io\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"offloadable\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"scheduled\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isShutdown\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"system\".*$")) + + // hazelcast_executor_isTerminated + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"async\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"client\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientBlocking\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientQuery\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"io\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"offloadable\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"scheduled\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_isTerminated\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"system\".*$")) + + // hazelcast_executor_maxPoolSize + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"async\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"client\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientBlocking\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientQuery\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"io\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"offloadable\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"scheduled\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_maxPoolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"system\".*$")) + + // hazelcast_executor_poolSize + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"async\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"client\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientBlocking\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientQuery\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"io\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"offloadable\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"scheduled\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_poolSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"system\".*$")) + + // hazelcast_executor_queueRemainingCapacity + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"async\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"client\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientBlocking\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientQuery\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"io\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"offloadable\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"scheduled\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueRemainingCapacity\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"system\".*$")) + + // hazelcast_executor_queueSize + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"async\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"client\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientBlocking\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"clientQuery\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"io\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"offloadable\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"scheduled\".*$")) + .body( + matchesRegex( + "(?s)^.*hazelcast_executor_queueSize\\{cluster=\"" + + testClusterName + + "\",address=.*,type=\"system\".*$")) + + // hazelcast_partition_partitionCount + .body( + matchesRegex( + "(?s)^.*hazelcast_partition_partitionCount\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + // hazelcast_partition_activePartition + .body( + matchesRegex( + "(?s)^.*hazelcast_partition_activePartition\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + // hazelcast_partition_isClusterSafe + .body( + matchesRegex( + "(?s)^.*hazelcast_partition_isClusterSafe\\{cluster=\"" + + testClusterName + + "\",address=.*$")) + // hazelcast_partition_isLocalMemberSafe + .body( + matchesRegex( + "(?s)^.*hazelcast_partition_isLocalMemberSafe\\{cluster=\"" + + testClusterName + + "\",address=.*$")); + } + + @AfterAll + static void afterClass() { + if (hazelcastInstance != null) { + hazelcastInstance.shutdown(); + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/EngineConfig.java b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/EngineConfig.java index 67521d645d8..5f7193d9fbe 100644 --- a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/EngineConfig.java +++ b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/EngineConfig.java @@ -22,6 +22,7 @@ import org.apache.seatunnel.engine.common.config.server.QueueType; import org.apache.seatunnel.engine.common.config.server.ServerConfigOptions; import org.apache.seatunnel.engine.common.config.server.SlotServiceConfig; +import org.apache.seatunnel.engine.common.config.server.TelemetryConfig; import org.apache.seatunnel.engine.common.config.server.ThreadShareMode; import lombok.Data; @@ -53,6 +54,8 @@ public class EngineConfig { private ConnectorJarStorageConfig connectorJarStorageConfig = ServerConfigOptions.CONNECTOR_JAR_STORAGE_CONFIG.defaultValue(); + private TelemetryConfig telemetryConfig = ServerConfigOptions.TELEMETRY.defaultValue(); + private QueueType queueType = ServerConfigOptions.QUEUE_TYPE.defaultValue(); private int historyJobExpireMinutes = ServerConfigOptions.HISTORY_JOB_EXPIRE_MINUTES.defaultValue(); diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelDomConfigProcessor.java b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelDomConfigProcessor.java index ad842512ca9..2f1fb87a7be 100644 --- a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelDomConfigProcessor.java +++ b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/YamlSeaTunnelDomConfigProcessor.java @@ -25,6 +25,8 @@ import org.apache.seatunnel.engine.common.config.server.QueueType; import org.apache.seatunnel.engine.common.config.server.ServerConfigOptions; import org.apache.seatunnel.engine.common.config.server.SlotServiceConfig; +import org.apache.seatunnel.engine.common.config.server.TelemetryConfig; +import org.apache.seatunnel.engine.common.config.server.TelemetryMetricConfig; import org.apache.seatunnel.engine.common.config.server.ThreadShareMode; import org.apache.commons.lang3.StringUtils; @@ -143,6 +145,8 @@ private void parseEngineConfig(Node engineNode, SeaTunnelConfig config) { getTextContent(node))); } else if (ServerConfigOptions.CONNECTOR_JAR_STORAGE_CONFIG.key().equals(name)) { engineConfig.setConnectorJarStorageConfig(parseConnectorJarStorageConfig(node)); + } else if (ServerConfigOptions.TELEMETRY.key().equals(name)) { + engineConfig.setTelemetryConfig(parseTelemetryConfig(node)); } else { LOGGER.warning("Unrecognized element: " + name); } @@ -214,6 +218,34 @@ private Map parseCheckpointPluginConfig(Node checkpointPluginCon return checkpointPluginConfig; } + private TelemetryConfig parseTelemetryConfig(Node telemetryNode) { + TelemetryConfig metricConfig = new TelemetryConfig(); + for (Node node : childElements(telemetryNode)) { + String name = cleanNodeName(node); + if (ServerConfigOptions.TELEMETRY_METRIC.key().equals(name)) { + metricConfig.setMetric(parseTelemetryMetricConfig(node)); + } else { + LOGGER.warning("Unrecognized element: " + name); + } + } + + return metricConfig; + } + + private TelemetryMetricConfig parseTelemetryMetricConfig(Node metricNode) { + TelemetryMetricConfig metricConfig = new TelemetryMetricConfig(); + for (Node node : childElements(metricNode)) { + String name = cleanNodeName(node); + if (ServerConfigOptions.TELEMETRY_METRIC_ENABLED.key().equals(name)) { + metricConfig.setEnabled(getBooleanValue(getTextContent(node))); + } else { + LOGGER.warning("Unrecognized element: " + name); + } + } + + return metricConfig; + } + private ConnectorJarStorageConfig parseConnectorJarStorageConfig( Node connectorJarStorageConfigNode) { ConnectorJarStorageConfig connectorJarStorageConfig = new ConnectorJarStorageConfig(); diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/ServerConfigOptions.java b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/ServerConfigOptions.java index 5c104c93315..05676e6d970 100644 --- a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/ServerConfigOptions.java +++ b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/ServerConfigOptions.java @@ -196,4 +196,22 @@ public class ServerConfigOptions { .type(new TypeReference() {}) .defaultValue(new ConnectorJarStorageConfig()) .withDescription("The connector jar storage configuration."); + + public static final Option TELEMETRY_METRIC_ENABLED = + Options.key("enabled") + .booleanType() + .defaultValue(true) + .withDescription("Whether open metrics export."); + + public static final Option TELEMETRY_METRIC = + Options.key("metric") + .type(new TypeReference() {}) + .defaultValue(new TelemetryMetricConfig()) + .withDescription("The telemetry metric configuration."); + + public static final Option TELEMETRY = + Options.key("telemetry") + .type(new TypeReference() {}) + .defaultValue(new TelemetryConfig()) + .withDescription("The telemetry configuration."); } diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/TelemetryConfig.java b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/TelemetryConfig.java new file mode 100644 index 00000000000..c3e603eea4c --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/TelemetryConfig.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.common.config.server; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class TelemetryConfig implements Serializable { + + private TelemetryMetricConfig metric = ServerConfigOptions.TELEMETRY_METRIC.defaultValue(); +} diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/TelemetryMetricConfig.java b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/TelemetryMetricConfig.java new file mode 100644 index 00000000000..a211514e353 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/config/server/TelemetryMetricConfig.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.common.config.server; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class TelemetryMetricConfig implements Serializable { + + private boolean enabled = ServerConfigOptions.TELEMETRY_METRIC_ENABLED.defaultValue(); +} diff --git a/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml b/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml index db8e4ae3c81..56137a606d0 100644 --- a/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml +++ b/seatunnel-engine/seatunnel-engine-common/src/main/resources/seatunnel.yaml @@ -38,3 +38,6 @@ seatunnel: connector-jar-storage-path: "" connector-jar-cleanup-task-interval: 3600 connector-jar-expiry-time: 600 + telemetry: + metric: + enabled: true diff --git a/seatunnel-engine/seatunnel-engine-server/pom.xml b/seatunnel-engine/seatunnel-engine-server/pom.xml index b31756674bf..608bc173880 100644 --- a/seatunnel-engine/seatunnel-engine-server/pom.xml +++ b/seatunnel-engine/seatunnel-engine-server/pom.xml @@ -82,6 +82,23 @@ optional provided + + + + io.prometheus + simpleclient + + + + io.prometheus + simpleclient_hotspot + + + + io.prometheus + simpleclient_httpserver + + diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java index d92f40ce390..37149940f07 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/CoordinatorService.java @@ -50,6 +50,8 @@ import org.apache.seatunnel.engine.server.resourcemanager.resource.SlotProfile; import org.apache.seatunnel.engine.server.service.jar.ConnectorPackageService; import org.apache.seatunnel.engine.server.task.operation.GetMetricsOperation; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.JobCounter; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.ThreadPoolStatus; import org.apache.seatunnel.engine.server.utils.NodeEngineUtil; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -671,31 +673,48 @@ public void memberRemoved(MembershipServiceEvent event) { } public void printExecutionInfo() { - ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executorService; - int activeCount = threadPoolExecutor.getActiveCount(); - int corePoolSize = threadPoolExecutor.getCorePoolSize(); - int maximumPoolSize = threadPoolExecutor.getMaximumPoolSize(); - int poolSize = threadPoolExecutor.getPoolSize(); - long completedTaskCount = threadPoolExecutor.getCompletedTaskCount(); - long taskCount = threadPoolExecutor.getTaskCount(); + ThreadPoolStatus threadPoolStatus = getThreadPoolStatusMetrics(); logger.info( StringFormatUtils.formatTable( "CoordinatorService Thread Pool Status", "activeCount", - activeCount, + threadPoolStatus.getActiveCount(), "corePoolSize", - corePoolSize, + threadPoolStatus.getCorePoolSize(), "maximumPoolSize", - maximumPoolSize, + threadPoolStatus.getMaximumPoolSize(), "poolSize", - poolSize, + threadPoolStatus.getPoolSize(), "completedTaskCount", - completedTaskCount, + threadPoolStatus.getCompletedTaskCount(), "taskCount", - taskCount)); + threadPoolStatus.getTaskCount())); } public void printJobDetailInfo() { + JobCounter jobCounter = getJobCountMetrics(); + logger.info( + StringFormatUtils.formatTable( + "Job info detail", + "createdJobCount", + jobCounter.getCreatedJobCount(), + "scheduledJobCount", + jobCounter.getScheduledJobCount(), + "runningJobCount", + jobCounter.getRunningJobCount(), + "failingJobCount", + jobCounter.getFailingJobCount(), + "failedJobCount", + jobCounter.getFailedJobCount(), + "cancellingJobCount", + jobCounter.getCancellingJobCount(), + "canceledJobCount", + jobCounter.getCanceledJobCount(), + "finishedJobCount", + jobCounter.getFinishedJobCount())); + } + + public JobCounter getJobCountMetrics() { AtomicLong createdJobCount = new AtomicLong(); AtomicLong scheduledJobCount = new AtomicLong(); AtomicLong runningJobCount = new AtomicLong(); @@ -743,26 +762,26 @@ public void printJobDetailInfo() { } }); } + return new JobCounter( + createdJobCount.longValue(), + scheduledJobCount.longValue(), + runningJobCount.longValue(), + failingJobCount.longValue(), + failedJobCount.longValue(), + cancellingJobCount.longValue(), + canceledJobCount.longValue(), + finishedJobCount.longValue()); + } - logger.info( - StringFormatUtils.formatTable( - "Job info detail", - "createdJobCount", - createdJobCount, - "scheduledJobCount", - scheduledJobCount, - "runningJobCount", - runningJobCount, - "failingJobCount", - failingJobCount, - "failedJobCount", - failedJobCount, - "cancellingJobCount", - cancellingJobCount, - "canceledJobCount", - canceledJobCount, - "finishedJobCount", - finishedJobCount)); + public ThreadPoolStatus getThreadPoolStatusMetrics() { + ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executorService; + return new ThreadPoolStatus( + threadPoolExecutor.getActiveCount(), + threadPoolExecutor.getCorePoolSize(), + threadPoolExecutor.getMaximumPoolSize(), + threadPoolExecutor.getPoolSize(), + threadPoolExecutor.getCompletedTaskCount(), + threadPoolExecutor.getTaskCount()); } public ConnectorPackageService getConnectorPackageService() { diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/NodeExtension.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/NodeExtension.java index 37e00cffab2..6690ae02f5d 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/NodeExtension.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/NodeExtension.java @@ -28,6 +28,8 @@ import com.hazelcast.instance.impl.Node; import com.hazelcast.internal.ascii.TextCommandService; import com.hazelcast.internal.ascii.TextCommandServiceImpl; +import io.prometheus.client.CollectorRegistry; +import lombok.Getter; import lombok.NonNull; import java.util.Map; @@ -37,10 +39,12 @@ public class NodeExtension extends DefaultNodeExtension { private final NodeExtensionCommon extCommon; + @Getter private CollectorRegistry collectorRegistry; public NodeExtension(@NonNull Node node, @NonNull SeaTunnelConfig seaTunnelConfig) { super(node); extCommon = new NodeExtensionCommon(node, new SeaTunnelServer(seaTunnelConfig)); + collectorRegistry = CollectorRegistry.defaultRegistry; } @Override diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java index ebb0edea850..a97a42a34c8 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java @@ -26,6 +26,7 @@ import org.apache.seatunnel.engine.server.service.jar.ConnectorPackageService; import org.apache.seatunnel.engine.server.service.slot.DefaultSlotService; import org.apache.seatunnel.engine.server.service.slot.SlotService; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.ThreadPoolStatus; import com.hazelcast.internal.services.ManagedService; import com.hazelcast.internal.services.MembershipAwareService; @@ -266,4 +267,8 @@ public NodeEngineImpl getNodeEngine() { public ConnectorPackageService getConnectorPackageService() { return getCoordinatorService().getConnectorPackageService(); } + + public ThreadPoolStatus getThreadPoolStatusMetrics() { + return coordinatorService.getThreadPoolStatusMetrics(); + } } diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServerStarter.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServerStarter.java index 4102ace4f25..10463cd63bd 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServerStarter.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServerStarter.java @@ -19,10 +19,12 @@ import org.apache.seatunnel.engine.common.config.ConfigProvider; import org.apache.seatunnel.engine.common.config.SeaTunnelConfig; +import org.apache.seatunnel.engine.server.telemetry.metrics.ExportsInstanceInitializer; import com.hazelcast.instance.impl.HazelcastInstanceFactory; import com.hazelcast.instance.impl.HazelcastInstanceImpl; import com.hazelcast.instance.impl.HazelcastInstanceProxy; +import com.hazelcast.instance.impl.Node; import lombok.NonNull; public class SeaTunnelServerStarter { @@ -34,11 +36,14 @@ public static void main(String[] args) { public static HazelcastInstanceImpl createHazelcastInstance(String clusterName) { SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); seaTunnelConfig.getHazelcastConfig().setClusterName(clusterName); - return createHazelcastInstance(seaTunnelConfig); + HazelcastInstanceImpl hazelcastInstance = createHazelcastInstance(seaTunnelConfig); + initTelemetryInstance(hazelcastInstance.node); + return hazelcastInstance; } public static HazelcastInstanceImpl createHazelcastInstance( @NonNull SeaTunnelConfig seaTunnelConfig) { + checkTelemetryConfig(seaTunnelConfig); return ((HazelcastInstanceProxy) HazelcastInstanceFactory.newHazelcastInstance( seaTunnelConfig.getHazelcastConfig(), @@ -50,6 +55,22 @@ public static HazelcastInstanceImpl createHazelcastInstance( public static HazelcastInstanceImpl createHazelcastInstance() { SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); - return createHazelcastInstance(seaTunnelConfig); + HazelcastInstanceImpl hazelcastInstance = createHazelcastInstance(seaTunnelConfig); + initTelemetryInstance(hazelcastInstance.node); + return hazelcastInstance; + } + + public static void initTelemetryInstance(@NonNull Node node) { + ExportsInstanceInitializer.init(node); + } + + private static void checkTelemetryConfig(SeaTunnelConfig seaTunnelConfig) { + // "hazelcast.jmx" need to set "true", for hazelcast metrics + if (seaTunnelConfig.getEngineConfig().getTelemetryConfig().getMetric().isEnabled()) { + seaTunnelConfig + .getHazelcastConfig() + .getProperties() + .setProperty("hazelcast.jmx", "true"); + } } } diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java index 0b9d0dc15d8..db7028aea82 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestConstant.java @@ -49,4 +49,7 @@ public class RestConstant { "/hazelcast/rest/maps/system-monitoring-information"; public static final String STOP_JOB_URL = "/hazelcast/rest/maps/stop-job"; + + public static final String TELEMETRY_METRICS_URL = "/hazelcast/rest/instance/metrics"; + public static final String TELEMETRY_OPEN_METRICS_URL = "/hazelcast/rest/instance/openmetrics"; } diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpGetCommandProcessor.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpGetCommandProcessor.java index 9addb8d8ec8..2cfb490c827 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpGetCommandProcessor.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/rest/RestHttpGetCommandProcessor.java @@ -46,7 +46,10 @@ import com.hazelcast.jet.impl.execution.init.CustomClassLoadedObject; import com.hazelcast.map.IMap; import com.hazelcast.spi.impl.NodeEngine; +import io.prometheus.client.exporter.common.TextFormat; +import java.io.IOException; +import java.io.StringWriter; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; @@ -59,6 +62,8 @@ import static org.apache.seatunnel.engine.server.rest.RestConstant.RUNNING_JOBS_URL; import static org.apache.seatunnel.engine.server.rest.RestConstant.RUNNING_JOB_URL; import static org.apache.seatunnel.engine.server.rest.RestConstant.SYSTEM_MONITORING_INFORMATION; +import static org.apache.seatunnel.engine.server.rest.RestConstant.TELEMETRY_METRICS_URL; +import static org.apache.seatunnel.engine.server.rest.RestConstant.TELEMETRY_OPEN_METRICS_URL; public class RestHttpGetCommandProcessor extends HttpCommandProcessor { @@ -93,6 +98,10 @@ public void handle(HttpGetCommand httpGetCommand) { handleJobInfoById(httpGetCommand, uri); } else if (uri.startsWith(SYSTEM_MONITORING_INFORMATION)) { getSystemMonitoringInformation(httpGetCommand); + } else if (uri.equals(TELEMETRY_METRICS_URL)) { + handleMetrics(httpGetCommand, TextFormat.CONTENT_TYPE_004); + } else if (uri.equals(TELEMETRY_OPEN_METRICS_URL)) { + handleMetrics(httpGetCommand, TextFormat.CONTENT_TYPE_OPENMETRICS_100); } else { original.handle(httpGetCommand); } @@ -209,6 +218,31 @@ private Map getJobMetrics(String jobMetrics) { return metricsMap; } + private void handleMetrics(HttpGetCommand httpGetCommand, String contentType) { + StringWriter stringWriter = new StringWriter(); + org.apache.seatunnel.engine.server.NodeExtension nodeExtension = + (org.apache.seatunnel.engine.server.NodeExtension) + textCommandService.getNode().getNodeExtension(); + try { + TextFormat.writeFormat( + contentType, + stringWriter, + nodeExtension.getCollectorRegistry().metricFamilySamples()); + this.prepareResponse(httpGetCommand, stringWriter.toString()); + } catch (IOException e) { + httpGetCommand.send400(); + } finally { + try { + if (stringWriter != null) { + stringWriter.close(); + } + } catch (IOException e) { + logger.warning("An error occurred while handling request " + httpGetCommand, e); + prepareResponse(SC_500, httpGetCommand, exceptionResponse(e)); + } + } + } + private SeaTunnelServer getSeaTunnelServer() { Map extensionServices = this.textCommandService.getNode().getNodeExtension().createExtensionServices(); diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/AbstractCollector.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/AbstractCollector.java new file mode 100644 index 00000000000..0439fc1d77e --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/AbstractCollector.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics; + +import org.apache.seatunnel.engine.server.CoordinatorService; +import org.apache.seatunnel.engine.server.SeaTunnelServer; + +import com.google.common.collect.Lists; +import com.hazelcast.cluster.impl.MemberImpl; +import com.hazelcast.instance.impl.Node; +import com.hazelcast.internal.cluster.ClusterService; +import com.hazelcast.internal.jmx.ManagementService; +import com.hazelcast.logging.ILogger; +import io.prometheus.client.Collector; +import io.prometheus.client.GaugeMetricFamily; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractCollector extends Collector { + + protected static String CLUSTER = "cluster"; + protected static String ADDRESS = "address"; + + protected Node node; + + public AbstractCollector(final Node node) { + this.node = node; + } + + protected Node getNode() { + return node; + } + + protected ILogger getLogger(Class clazz) { + return getNode().getLogger(clazz); + } + + protected boolean isMaster() { + return getNode().isMaster(); + } + + protected MemberImpl getLocalMember() { + return getNode().nodeEngine.getLocalMember(); + } + + protected SeaTunnelServer getServer() { + return getNode().getNodeEngine().getService(SeaTunnelServer.SERVICE_NAME); + } + + protected CoordinatorService getCoordinatorService() { + return getServer().getCoordinatorService(); + } + + protected ManagementService getManagementService() { + return getNode().hazelcastInstance.getManagementService(); + } + + protected ClusterService getClusterService() { + return getNode().getClusterService(); + } + + protected String localAddress() { + return getLocalMember().getInetAddress().getHostAddress() + + ":" + + getLocalMember().getPort(); + } + + protected String masterAddress() throws UnknownHostException { + return getClusterService().getMasterAddress().getInetAddress().getHostAddress() + + ":" + + getClusterService().getMasterAddress().getPort(); + } + + protected String getClusterName() { + return getNode().getConfig().getClusterName(); + } + + protected List labelValues(String... values) { + List labelValues = new ArrayList<>(); + labelValues.add(getClusterName()); + if (values != null) { + labelValues.addAll(Lists.newArrayList(values)); + } + return labelValues; + } + + protected List clusterLabelNames(String... labels) { + List labelNames = new ArrayList<>(); + labelNames.add(CLUSTER); + if (labels != null) { + labelNames.addAll(Lists.newArrayList(labels)); + } + return labelNames; + } + + protected void longMetric( + GaugeMetricFamily metricFamily, long count, List labelValues) { + metricFamily.addMetric(labelValues, count); + } + + protected void intMetric(GaugeMetricFamily metricFamily, int count, List labelValues) { + metricFamily.addMetric(labelValues, count); + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/ExportsInstanceInitializer.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/ExportsInstanceInitializer.java new file mode 100644 index 00000000000..f737fbf6c24 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/ExportsInstanceInitializer.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics; + +import org.apache.seatunnel.engine.server.telemetry.metrics.exports.ClusterMetricExports; +import org.apache.seatunnel.engine.server.telemetry.metrics.exports.JobMetricExports; +import org.apache.seatunnel.engine.server.telemetry.metrics.exports.JobThreadPoolStatusExports; +import org.apache.seatunnel.engine.server.telemetry.metrics.exports.NodeMetricExports; + +import com.hazelcast.instance.impl.Node; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.hotspot.DefaultExports; + +public final class ExportsInstanceInitializer { + + private static boolean initialized = false; + + private ExportsInstanceInitializer() {} + + public static synchronized void init(Node node) { + if (!initialized) { + // initialize jvm collector + DefaultExports.initialize(); + + // register collectors + CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry; + // Job info detail + new JobMetricExports(node).register(collectorRegistry); + // Thread pool status + new JobThreadPoolStatusExports(node).register(collectorRegistry); + // Node metrics + new NodeMetricExports(node).register(collectorRegistry); + // Cluster metrics + new ClusterMetricExports(node).register(collectorRegistry); + initialized = true; + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/JobCounter.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/JobCounter.java new file mode 100644 index 00000000000..a965bed7c66 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/JobCounter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class JobCounter { + private long createdJobCount; + private long scheduledJobCount; + private long runningJobCount; + private long failingJobCount; + private long failedJobCount; + private long cancellingJobCount; + private long canceledJobCount; + private long finishedJobCount; +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/ThreadPoolStatus.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/ThreadPoolStatus.java new file mode 100644 index 00000000000..ecbf341145d --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/entity/ThreadPoolStatus.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class ThreadPoolStatus { + private int activeCount; + private int corePoolSize; + private int maximumPoolSize; + private int poolSize; + private long completedTaskCount; + private long taskCount; +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/ClusterMetricExports.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/ClusterMetricExports.java new file mode 100644 index 00000000000..51b4cb448c7 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/ClusterMetricExports.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.exports; + +import org.apache.seatunnel.engine.server.telemetry.metrics.AbstractCollector; + +import com.hazelcast.cluster.impl.MemberImpl; +import com.hazelcast.instance.impl.Node; +import io.prometheus.client.GaugeMetricFamily; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class ClusterMetricExports extends AbstractCollector { + + public ClusterMetricExports(Node node) { + super(node); + } + + @Override + public List collect() { + List mfs = new ArrayList(); + + // cluster_info + clusterInfo(mfs); + // cluster_time + clusterTime(mfs); + // instance count + nodeCount(mfs); + + return mfs; + } + + private void clusterTime(final List mfs) { + GaugeMetricFamily metricFamily = + new GaugeMetricFamily( + "cluster_time", + "Cluster start time", + clusterLabelNames("hazelcastVersion")); + List labelValues = labelValues(getClusterService().getClusterVersion().toString()); + + metricFamily.addMetric(labelValues, getClusterService().getClusterTime()); + mfs.add(metricFamily); + } + + private void clusterInfo(final List mfs) { + GaugeMetricFamily metricFamily = + new GaugeMetricFamily( + "cluster_info", + "Cluster info", + clusterLabelNames("hazelcastVersion", "master")); + List labelValues = null; + try { + labelValues = + labelValues( + getClusterService().getClusterVersion().toString(), masterAddress()); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + + metricFamily.addMetric(labelValues, 1); + mfs.add(metricFamily); + } + + private void nodeCount(final List mfs) { + Collection memberImpls = getClusterService().getMemberImpls(); + + GaugeMetricFamily metricFamily = + new GaugeMetricFamily( + "node_count", "Cluster node total count ", clusterLabelNames()); + List labelValues = labelValues(); + + metricFamily.addMetric(labelValues, memberImpls == null ? 0 : memberImpls.size()); + mfs.add(metricFamily); + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/JobMetricExports.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/JobMetricExports.java new file mode 100644 index 00000000000..7a7bf7f9003 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/JobMetricExports.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.exports; + +import org.apache.seatunnel.engine.server.CoordinatorService; +import org.apache.seatunnel.engine.server.telemetry.metrics.AbstractCollector; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.JobCounter; + +import com.hazelcast.instance.impl.Node; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.List; + +public class JobMetricExports extends AbstractCollector { + + public JobMetricExports(Node node) { + super(node); + } + + @Override + public List collect() { + List mfs = new ArrayList(); + // Only the master can get job metrics + if (isMaster()) { + CoordinatorService coordinatorService = getCoordinatorService(); + JobCounter jobCountMetrics = coordinatorService.getJobCountMetrics(); + + GaugeMetricFamily metricFamily = + new GaugeMetricFamily( + "job_count", + "All job counts of seatunnel cluster ", + clusterLabelNames("type")); + + metricFamily.addMetric(labelValues("canceled"), jobCountMetrics.getCanceledJobCount()); + metricFamily.addMetric( + labelValues("cancelling"), jobCountMetrics.getCancellingJobCount()); + metricFamily.addMetric(labelValues("created"), jobCountMetrics.getCreatedJobCount()); + metricFamily.addMetric(labelValues("failed"), jobCountMetrics.getFailedJobCount()); + metricFamily.addMetric(labelValues("failing"), jobCountMetrics.getFailingJobCount()); + metricFamily.addMetric(labelValues("finished"), jobCountMetrics.getFinishedJobCount()); + metricFamily.addMetric(labelValues("running"), jobCountMetrics.getRunningJobCount()); + metricFamily.addMetric( + labelValues("scheduled"), jobCountMetrics.getScheduledJobCount()); + + mfs.add(metricFamily); + } + return mfs; + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/JobThreadPoolStatusExports.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/JobThreadPoolStatusExports.java new file mode 100644 index 00000000000..80ee89343e0 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/JobThreadPoolStatusExports.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.exports; + +import org.apache.seatunnel.engine.server.telemetry.metrics.AbstractCollector; +import org.apache.seatunnel.engine.server.telemetry.metrics.entity.ThreadPoolStatus; + +import com.hazelcast.instance.impl.Node; +import io.prometheus.client.CounterMetricFamily; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.List; + +public class JobThreadPoolStatusExports extends AbstractCollector { + + private static String HELP = + "The %s of seatunnel coordinator job's executor cached thread pool"; + + public JobThreadPoolStatusExports(Node node) { + super(node); + } + + @Override + public List collect() { + List mfs = new ArrayList(); + + List labelValues = labelValues(localAddress()); + + ThreadPoolStatus threadPoolStatusMetrics = getServer().getThreadPoolStatusMetrics(); + List labelNames = clusterLabelNames(ADDRESS); + + GaugeMetricFamily activeCount = + new GaugeMetricFamily( + "job_thread_pool_activeCount", + String.format(HELP, "activeCount"), + labelNames); + activeCount.addMetric(labelValues, threadPoolStatusMetrics.getActiveCount()); + mfs.add(activeCount); + + CounterMetricFamily completedTask = + new CounterMetricFamily( + "job_thread_pool_completedTask", + String.format(HELP, "completedTask"), + labelNames); + completedTask.addMetric(labelValues, threadPoolStatusMetrics.getCompletedTaskCount()); + mfs.add(completedTask); + + GaugeMetricFamily corePoolSize = + new GaugeMetricFamily( + "job_thread_pool_corePoolSize", + String.format(HELP, "corePoolSize"), + labelNames); + corePoolSize.addMetric(labelValues, threadPoolStatusMetrics.getCorePoolSize()); + mfs.add(corePoolSize); + + GaugeMetricFamily maximumPoolSize = + new GaugeMetricFamily( + "job_thread_pool_maximumPoolSize", + String.format(HELP, "maximumPoolSize"), + labelNames); + maximumPoolSize.addMetric(labelValues, threadPoolStatusMetrics.getMaximumPoolSize()); + mfs.add(maximumPoolSize); + + GaugeMetricFamily poolSize = + new GaugeMetricFamily( + "job_thread_pool_poolSize", String.format(HELP, "poolSize"), labelNames); + poolSize.addMetric(labelValues, threadPoolStatusMetrics.getPoolSize()); + mfs.add(poolSize); + + CounterMetricFamily taskCount = + new CounterMetricFamily( + "job_thread_pool_task", String.format(HELP, "taskCount"), labelNames); + taskCount.addMetric(labelValues, threadPoolStatusMetrics.getTaskCount()); + mfs.add(taskCount); + + return mfs; + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/NodeMetricExports.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/NodeMetricExports.java new file mode 100644 index 00000000000..c092ffd51b9 --- /dev/null +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/telemetry/metrics/exports/NodeMetricExports.java @@ -0,0 +1,407 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.server.telemetry.metrics.exports; + +import org.apache.seatunnel.engine.server.telemetry.metrics.AbstractCollector; + +import com.hazelcast.instance.impl.Node; +import com.hazelcast.internal.jmx.InstanceMBean; +import com.hazelcast.internal.jmx.PartitionServiceMBean; +import io.prometheus.client.GaugeMetricFamily; + +import java.util.ArrayList; +import java.util.List; + +public class NodeMetricExports extends AbstractCollector { + + public NodeMetricExports(Node node) { + super(node); + } + + @Override + public List collect() { + List mfs = new ArrayList(); + // instance state + nodeState(mfs); + + InstanceMBean instanceMBean = getManagementService().getInstanceMBean(); + if (instanceMBean == null) { + return mfs; + } + + // node hazelcast executor + String address = localAddress(); + List labelNames = clusterLabelNames(ADDRESS, "type"); + GaugeMetricFamily isShutdownMetricFamily = + new GaugeMetricFamily( + "hazelcast_executor_isShutdown", + "The hazelcast executor isShutdown of seatunnel cluster node", + labelNames); + GaugeMetricFamily isTerminatedMetricFamily = + new GaugeMetricFamily( + "hazelcast_executor_isTerminated", + "The hazelcast executor isTerminated of seatunnel cluster node", + labelNames); + + GaugeMetricFamily maxPoolSizeMetricFamily = + new GaugeMetricFamily( + "hazelcast_executor_maxPoolSize", + "The hazelcast executor maxPoolSize of seatunnel cluster node", + labelNames); + + GaugeMetricFamily poolSizeMetricFamily = + new GaugeMetricFamily( + "hazelcast_executor_poolSize", + "The hazelcast executor poolSize of seatunnel cluster node", + labelNames); + + GaugeMetricFamily queueRemainingCapacityMetricFamily = + new GaugeMetricFamily( + "hazelcast_executor_queueRemainingCapacity", + "The hazelcast executor queueRemainingCapacity of seatunnel cluster ", + labelNames); + + GaugeMetricFamily queueSizeMetricFamily = + new GaugeMetricFamily( + "hazelcast_executor_queueSize", + "The hazelcast executor queueSize of seatunnel cluster node", + labelNames); + + GaugeMetricFamily executedCountMetricFamily = + new GaugeMetricFamily( + "hazelcast_executor_executedCount", + "The hazelcast executor executedCount of seatunnel cluster node", + labelNames); + + List asyncValues = labelValues(address, "async"); + List clientBlockingValues = labelValues(address, "clientBlocking"); + List clientExecutorValues = labelValues(address, "client"); + List clientQueryValues = labelValues(address, "clientQuery"); + List ioValues = labelValues(address, "io"); + List offloadableValues = labelValues(address, "offloadable"); + List scheduledValues = labelValues(address, "scheduled"); + List systemValues = labelValues(address, "system"); + + // Executor executedCount + longMetric( + executedCountMetricFamily, + instanceMBean.getAsyncExecutorMBean().getExecutedCount(), + asyncValues); + longMetric( + executedCountMetricFamily, + instanceMBean.getClientExecutorMBean().getExecutedCount(), + clientExecutorValues); + longMetric( + executedCountMetricFamily, + instanceMBean.getClientBlockingExecutorMBean().getExecutedCount(), + clientBlockingValues); + longMetric( + executedCountMetricFamily, + instanceMBean.getClientQueryExecutorMBean().getExecutedCount(), + clientQueryValues); + longMetric( + executedCountMetricFamily, + instanceMBean.getIoExecutorMBean().getExecutedCount(), + ioValues); + longMetric( + executedCountMetricFamily, + instanceMBean.getOffloadableExecutorMBean().getExecutedCount(), + offloadableValues); + longMetric( + executedCountMetricFamily, + instanceMBean.getScheduledExecutorMBean().getExecutedCount(), + scheduledValues); + longMetric( + executedCountMetricFamily, + instanceMBean.getSystemExecutorMBean().getExecutedCount(), + systemValues); + mfs.add(executedCountMetricFamily); + + // Executor isShutdown + intMetric( + isShutdownMetricFamily, + instanceMBean.getAsyncExecutorMBean().isShutdown() ? 1 : 0, + asyncValues); + intMetric( + isShutdownMetricFamily, + instanceMBean.getClientExecutorMBean().isShutdown() ? 1 : 0, + clientExecutorValues); + intMetric( + isShutdownMetricFamily, + instanceMBean.getClientBlockingExecutorMBean().isShutdown() ? 1 : 0, + clientBlockingValues); + intMetric( + isShutdownMetricFamily, + instanceMBean.getClientQueryExecutorMBean().isShutdown() ? 1 : 0, + clientQueryValues); + intMetric( + isShutdownMetricFamily, + instanceMBean.getIoExecutorMBean().isShutdown() ? 1 : 0, + ioValues); + intMetric( + isShutdownMetricFamily, + instanceMBean.getOffloadableExecutorMBean().isShutdown() ? 1 : 0, + offloadableValues); + intMetric( + isShutdownMetricFamily, + instanceMBean.getScheduledExecutorMBean().isShutdown() ? 1 : 0, + scheduledValues); + intMetric( + isShutdownMetricFamily, + instanceMBean.getSystemExecutorMBean().isShutdown() ? 1 : 0, + systemValues); + mfs.add(isShutdownMetricFamily); + + // Executor isTerminated + intMetric( + isTerminatedMetricFamily, + instanceMBean.getAsyncExecutorMBean().isTerminated() ? 1 : 0, + asyncValues); + intMetric( + isTerminatedMetricFamily, + instanceMBean.getClientExecutorMBean().isTerminated() ? 1 : 0, + clientExecutorValues); + intMetric( + isTerminatedMetricFamily, + instanceMBean.getClientBlockingExecutorMBean().isTerminated() ? 1 : 0, + clientBlockingValues); + intMetric( + isTerminatedMetricFamily, + instanceMBean.getClientQueryExecutorMBean().isTerminated() ? 1 : 0, + clientQueryValues); + intMetric( + isTerminatedMetricFamily, + instanceMBean.getIoExecutorMBean().isTerminated() ? 1 : 0, + ioValues); + intMetric( + isTerminatedMetricFamily, + instanceMBean.getOffloadableExecutorMBean().isTerminated() ? 1 : 0, + offloadableValues); + intMetric( + isTerminatedMetricFamily, + instanceMBean.getScheduledExecutorMBean().isTerminated() ? 1 : 0, + scheduledValues); + intMetric( + isTerminatedMetricFamily, + instanceMBean.getSystemExecutorMBean().isTerminated() ? 1 : 0, + systemValues); + mfs.add(isTerminatedMetricFamily); + + // Executor maxPoolSize + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getAsyncExecutorMBean().maxPoolSize(), + asyncValues); + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getClientExecutorMBean().maxPoolSize(), + clientExecutorValues); + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getClientBlockingExecutorMBean().maxPoolSize(), + clientBlockingValues); + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getClientQueryExecutorMBean().maxPoolSize(), + clientQueryValues); + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getIoExecutorMBean().maxPoolSize(), + ioValues); + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getOffloadableExecutorMBean().maxPoolSize(), + offloadableValues); + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getScheduledExecutorMBean().maxPoolSize(), + scheduledValues); + intMetric( + maxPoolSizeMetricFamily, + instanceMBean.getSystemExecutorMBean().maxPoolSize(), + systemValues); + mfs.add(maxPoolSizeMetricFamily); + + // Executor poolSize + intMetric( + poolSizeMetricFamily, + instanceMBean.getAsyncExecutorMBean().poolSize(), + asyncValues); + intMetric( + poolSizeMetricFamily, + instanceMBean.getClientExecutorMBean().poolSize(), + clientExecutorValues); + intMetric( + poolSizeMetricFamily, + instanceMBean.getClientBlockingExecutorMBean().poolSize(), + clientBlockingValues); + intMetric( + poolSizeMetricFamily, + instanceMBean.getClientQueryExecutorMBean().poolSize(), + clientQueryValues); + intMetric(poolSizeMetricFamily, instanceMBean.getIoExecutorMBean().poolSize(), ioValues); + intMetric( + poolSizeMetricFamily, + instanceMBean.getOffloadableExecutorMBean().poolSize(), + offloadableValues); + intMetric( + poolSizeMetricFamily, + instanceMBean.getScheduledExecutorMBean().poolSize(), + scheduledValues); + intMetric( + poolSizeMetricFamily, + instanceMBean.getSystemExecutorMBean().poolSize(), + systemValues); + mfs.add(poolSizeMetricFamily); + + // Executor queueRemainingCapacity + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getAsyncExecutorMBean().queueRemainingCapacity(), + asyncValues); + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getClientExecutorMBean().queueRemainingCapacity(), + clientExecutorValues); + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getClientBlockingExecutorMBean().queueRemainingCapacity(), + clientBlockingValues); + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getClientQueryExecutorMBean().queueRemainingCapacity(), + clientQueryValues); + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getIoExecutorMBean().queueRemainingCapacity(), + ioValues); + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getOffloadableExecutorMBean().queueRemainingCapacity(), + offloadableValues); + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getScheduledExecutorMBean().queueRemainingCapacity(), + scheduledValues); + intMetric( + queueRemainingCapacityMetricFamily, + instanceMBean.getSystemExecutorMBean().queueRemainingCapacity(), + systemValues); + mfs.add(queueRemainingCapacityMetricFamily); + + // Executor queueSize + intMetric( + queueSizeMetricFamily, + instanceMBean.getAsyncExecutorMBean().queueSize(), + asyncValues); + intMetric( + queueSizeMetricFamily, + instanceMBean.getClientExecutorMBean().queueSize(), + clientExecutorValues); + intMetric( + queueSizeMetricFamily, + instanceMBean.getClientBlockingExecutorMBean().queueSize(), + clientBlockingValues); + intMetric( + queueSizeMetricFamily, + instanceMBean.getClientQueryExecutorMBean().queueSize(), + clientQueryValues); + intMetric(queueSizeMetricFamily, instanceMBean.getIoExecutorMBean().queueSize(), ioValues); + intMetric( + queueSizeMetricFamily, + instanceMBean.getOffloadableExecutorMBean().queueSize(), + offloadableValues); + intMetric( + queueSizeMetricFamily, + instanceMBean.getScheduledExecutorMBean().queueSize(), + scheduledValues); + intMetric( + queueSizeMetricFamily, + instanceMBean.getSystemExecutorMBean().queueSize(), + systemValues); + mfs.add(queueSizeMetricFamily); + + // partition metric + partitionMetric(instanceMBean.getPartitionServiceMBean(), mfs, address); + + return mfs; + } + + private void partitionMetric( + PartitionServiceMBean partitionServiceMBean, + List mfs, + String address) { + List labelNames = clusterLabelNames(ADDRESS); + + GaugeMetricFamily partitionPartitionCount = + new GaugeMetricFamily( + "hazelcast_partition_partitionCount", + "The partitionCount of seatunnel cluster node", + labelNames); + intMetric( + partitionPartitionCount, + partitionServiceMBean.getPartitionCount(), + labelValues(address)); + mfs.add(partitionPartitionCount); + + GaugeMetricFamily partitionActivePartition = + new GaugeMetricFamily( + "hazelcast_partition_activePartition", + "The activePartition of seatunnel cluster node", + labelNames); + intMetric( + partitionActivePartition, + partitionServiceMBean.getActivePartitionCount(), + labelValues(address)); + mfs.add(partitionActivePartition); + + GaugeMetricFamily partitionIsClusterSafe = + new GaugeMetricFamily( + "hazelcast_partition_isClusterSafe", + "Whether is cluster safe of partition", + labelNames); + intMetric( + partitionIsClusterSafe, + partitionServiceMBean.isClusterSafe() ? 1 : 0, + labelValues(address)); + mfs.add(partitionIsClusterSafe); + + GaugeMetricFamily partitionIsLocalMemberSafe = + new GaugeMetricFamily( + "hazelcast_partition_isLocalMemberSafe", + "Whether is local member safe of partition", + labelNames); + intMetric( + partitionIsLocalMemberSafe, + partitionServiceMBean.isLocalMemberSafe() ? 1 : 0, + labelValues(address)); + mfs.add(partitionIsLocalMemberSafe); + } + + private void nodeState(List mfs) { + GaugeMetricFamily metricFamily = + new GaugeMetricFamily( + "node_state", + "Whether is up of seatunnel node ", + clusterLabelNames(ADDRESS)); + String address = localAddress(); + List labelValues = labelValues(address); + metricFamily.addMetric(labelValues, 1); + mfs.add(metricFamily); + } +} diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java index 2a7c25e0830..2cb74ac48d6 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java @@ -23,6 +23,7 @@ import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Paths; @@ -30,7 +31,7 @@ public class SeaTunnelEngineExample { public static void main(String[] args) - throws FileNotFoundException, URISyntaxException, CommandException { + throws IOException, URISyntaxException, CommandException { String configurePath = args.length > 0 ? args[0] : "/examples/fake_to_console.conf"; String configFile = getTestConfigFile(configurePath); ClientCommandArgs clientCommandArgs = new ClientCommandArgs(); diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java index df539ea6e05..2bb1ed69c4b 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java @@ -21,8 +21,10 @@ import org.apache.seatunnel.core.starter.exception.CommandException; import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; +import java.io.IOException; + public class SeaTunnelEngineServerExample { - public static void main(String[] args) throws CommandException { + public static void main(String[] args) throws CommandException, IOException { ServerCommandArgs serverCommandArgs = new ServerCommandArgs(); SeaTunnel.run(serverCommandArgs.buildCommand()); } diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/java/org/apache/seatunnel/example/flink/v2/SeaTunnelApiExample.java b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/java/org/apache/seatunnel/example/flink/v2/SeaTunnelApiExample.java index 553c1963622..f6b1c6a7941 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/java/org/apache/seatunnel/example/flink/v2/SeaTunnelApiExample.java +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/java/org/apache/seatunnel/example/flink/v2/SeaTunnelApiExample.java @@ -22,6 +22,7 @@ import org.apache.seatunnel.core.starter.flink.args.FlinkCommandArgs; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Paths; @@ -29,7 +30,7 @@ public class SeaTunnelApiExample { public static void main(String[] args) - throws FileNotFoundException, URISyntaxException, CommandException { + throws IOException, URISyntaxException, CommandException { String configurePath = args.length > 0 ? args[0] : "/examples/fake_to_console.conf"; String configFile = getTestConfigFile(configurePath); FlinkCommandArgs flinkCommandArgs = new FlinkCommandArgs(); diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/ExampleUtils.java b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/ExampleUtils.java index 2c7ddc5de69..0bbfff04356 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/ExampleUtils.java +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/ExampleUtils.java @@ -23,6 +23,7 @@ import org.apache.seatunnel.core.starter.spark.args.SparkCommandArgs; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Paths; @@ -30,7 +31,7 @@ public class ExampleUtils { public static void builder(String configurePath) - throws FileNotFoundException, URISyntaxException, CommandException { + throws IOException, URISyntaxException, CommandException { String configFile = getTestConfigFile(configurePath); SparkCommandArgs sparkCommandArgs = new SparkCommandArgs(); sparkCommandArgs.setConfigFile(configFile); diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java index 870ec64895a..b0a12dc3fc4 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/java/org/apache/seatunnel/example/spark/v2/SeaTunnelApiExample.java @@ -19,13 +19,13 @@ import org.apache.seatunnel.core.starter.exception.CommandException; -import java.io.FileNotFoundException; +import java.io.IOException; import java.net.URISyntaxException; public class SeaTunnelApiExample { public static void main(String[] args) - throws FileNotFoundException, URISyntaxException, CommandException { + throws IOException, URISyntaxException, CommandException { String configurePath = args.length > 0 ? args[0] : "/examples/spark.batch.conf"; ExampleUtils.builder(configurePath); } diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 52205f52e46..6f97a7c66d4 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -39,4 +39,11 @@ listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar json-path-2.7.0.jar json-smart-2.4.7.jar accessors-smart-2.4.7.jar -asm-9.1.jar \ No newline at end of file +asm-9.1.jar +simpleclient-0.16.0.jar +simpleclient_common-0.16.0.jar +simpleclient_hotspot-0.16.0.jar +simpleclient_httpserver-0.16.0.jar +simpleclient_tracer_common-0.16.0.jar +simpleclient_tracer_otel-0.16.0.jar +simpleclient_tracer_otel_agent-0.16.0.jar \ No newline at end of file