{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Customer Direct Remote Write version of the AutoMQ Kafka Connect cluster dashboard. Uses the original Opsbox layout and styling; customer is removed, while environment_id and automq_instance_id remain available.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 1, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 250, "panels": [], "title": "Cluster Info", "type": "row" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 1 }, "id": 249, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "name", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(automq_instance_id) ({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\"})", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{automq_instance_id}}", "range": true, "refId": "A", "useBackend": false } ], "title": "instanceId", "type": "stat" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 4, "y": 1 }, "id": 248, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "name", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(environment_id) ({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\"})", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{environment_id}}", "range": true, "refId": "A", "useBackend": false } ], "title": "envId", "type": "stat" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 8, "y": 1 }, "id": 251, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "name", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(connect_cluster_id) ({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\"})", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{connect_cluster_id}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Connect Cluster ID", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 199, "panels": [], "title": "Connector", "type": "row" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "1": { "index": 0, "text": "Source" } }, "type": "value" } ], "noValue": "Sink", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 12, "x": 0, "y": 5 }, "id": 244, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "count by (connect_cluster_id) (\n {\n __name__=~\"kafka_source_task_metrics_source_record_poll_rate(_ratio)?_per_second\",\n environment_id=~\"${envId:regex}\",\n automq_instance_id=~\"${instance:regex}\",\n connect_cluster_id=~\"${connect_instance_id:regex}\",\n connector=~\"${connector:regex}\"\n }\n)", "format": "time_series", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "useBackend": false }, { "datasource": { "name": "Expression", "type": "__expr__", "uid": "__expr__" }, "expression": "A", "hide": true, "reducer": "last", "refId": "C", "settings": { "mode": "replaceNN", "replaceWithValue": 0 }, "type": "reduce" }, { "conditions": [ { "evaluator": { "params": [ 0, 0 ], "type": "gt" }, "query": { "params": [] }, "reducer": { "params": [], "type": "last" }, "type": "query" } ], "datasource": { "name": "Expression", "type": "__expr__", "uid": "__expr__" }, "expression": "C", "hide": false, "refId": "E", "type": "threshold" } ], "title": "Type", "type": "stat" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 5 }, "id": 245, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "count by(connect_cluster_id) (count by (connect_cluster_id, host_name) ({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\"}))", "instant": false, "legendFormat": "{{connect_cluster_id}}", "range": true, "refId": "A" } ], "title": "worker count", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "decimals": 0, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 8 }, "id": 212, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Tasks Total", "type": "stat" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "decimals": 0, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 4, "y": 8 }, "id": 213, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_running_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Tasks Running", "type": "stat" }, { "fieldConfig": { "defaults": { "decimals": 0, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 8, "y": 8 }, "id": 215, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_paused_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Tasks Paused", "type": "stat" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "decimals": 0, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 11 }, "id": 214, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_failed_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Tasks Failed", "type": "stat" }, { "fieldConfig": { "defaults": { "decimals": 0, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 4, "y": 11 }, "id": 216, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_unassigned_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Tasks Unassigned", "type": "stat" }, { "fieldConfig": { "defaults": { "decimals": 0, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "purple", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 8, "y": 11 }, "id": 217, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_destroyed_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"})", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Tasks Destroyed", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Bytes per second read off all sockets", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "id": 238, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by(connect_cluster_id)(({__name__=~\"kafka_consumer_node_metrics_incoming_byte_rate(_bytes_per_second)?\",connect_cluster_id=~\"$connect_instance_id\"} or on(connect_cluster_id, client_id, host_name) {__name__=~\"kafka_consumer_node_metrics_incoming_byte_rate_bytes_per_second\",connect_cluster_id=~\"$connect_instance_id\"}))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{connect_cluster_id}}", "range": true, "refId": "A" } ], "title": "Incoming Byte Rate", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Status of tasks over time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "destroyed" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "paused" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "running" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "unassigned" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "id": 226, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_running_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", host_name=~\"${worker:regex}\"})", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "running", "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_failed_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\", host_name=~\"${worker:regex}\"})", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "failed", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_paused_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\", host_name=~\"${worker:regex}\"})", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "paused", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_destroyed_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\", host_name=~\"${worker:regex}\"})", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "destroyed", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum({__name__=~\"kafka_connect_worker_metrics_connector_unassigned_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\", host_name=~\"${worker:regex}\"})", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "unassigned", "refId": "E" } ], "title": "Status of tasks", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of errors seen by task", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, "id": 241, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum by(connect_cluster_id) (rate({__name__=~\"kafka_task_error_metrics_total_record_errors(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\"}[$__rate_interval]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connect_cluster_id}}", "range": true, "refId": "A", "useBackend": false } ], "title": "record errors per second", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Average number of outgoing bytes sent per second to all servers", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, "id": 239, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(connect_cluster_id) (({__name__=~\"kafka_producer_metrics_outgoing_byte_rate(_bytes_per_second)?\",connect_cluster_id=~\"$connect_instance_id\"} or on(connect_cluster_id, client_id, host_name) {__name__=~\"kafka_producer_metrics_outgoing_byte_rate_bytes_per_second\",connect_cluster_id=~\"$connect_instance_id\"}))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{connect_cluster_id}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Outgoing Byte Rate", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "record read per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }, "id": 240, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(connect_cluster_id) ({__name__=~\"kafka_sink_task_metrics_sink_record_read_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\"})", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connect_cluster_id}}", "refId": "A", "useBackend": false } ], "title": "record read per second", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The average per-second number of records output from the transformations and written to Kafka for this task belonging to the named source connector in this worker. This is after transformations are applied and excludes any records filtered out by the transformations.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 30 }, "id": 242, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(connect_cluster_id) ({__name__=~\"kafka_source_task_metrics_source_record_write_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connect_cluster_id}}", "refId": "A", "useBackend": false } ], "title": "Source Record Write per second", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Average percentage of the task’s offset commit attempts that succeeded", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "id": 243, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "avg by(connect_cluster_id) ({__name__=~\"kafka_connector_task_metrics_offset_commit_success_percentage(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{connect_cluster_id}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Offset commit success percentage", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, "id": 97, "panels": [], "title": "Connect Worker", "type": "row" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "memory", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "displayName", "value": "${__field.labels.host_name}" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 47 }, "id": 224, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by(host_name) (jvm_memory_limit_bytes{connect_cluster_id=~\"$connect_instance_id\", jvm_memory_type=\"heap\", host_name=~\"${worker:regex}\"})", "hide": true, "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "sum by(host_name) (jvm_memory_used_after_last_gc_bytes{connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "hide": true, "interval": "", "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "name": "Expression", "type": "__expr__", "uid": "__expr__" }, "expression": "$B / $A", "hide": false, "refId": "C", "type": "math" } ], "title": "JVM Heap Memory Utilization", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Cores", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "displayName", "value": "${__field.labels.host_name}" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 47 }, "id": 223, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "exemplar": true, "expr": "rate(jvm_cpu_time_seconds_total{connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}[$__rate_interval])", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, "interval": "", "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "jvm_cpu_count{connect_cluster_id=~\"$connect_instance_id\",host_name=~\"${worker:regex}\"}", "hide": true, "instant": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "name": "Expression", "type": "__expr__", "uid": "__expr__" }, "expression": "$A / $B", "hide": false, "refId": "C", "type": "math" } ], "title": "CPU Usage", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Average number of network operations (reads or writes) on all connections per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, "id": 95, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_consumer_metrics_network_io_rate(_ratio)?_per_second\",host_name=~\"${worker:regex}\", connect_cluster_id=~\"$connect_instance_id\"})", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "consumer-{{host_name}}", "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_connect_metrics_network_io_rate(_ratio)?_per_second\",host_name=~\"${worker:regex}\", connect_cluster_id=~\"$connect_instance_id\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{host_name}}", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_producer_metrics_network_io_rate(_ratio)?_per_second\",host_name=~\"${worker:regex}\", connect_cluster_id=~\"$connect_instance_id\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "producer-{{host_name}}", "range": true, "refId": "C", "useBackend": false } ], "title": "Network IO Rate", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Current number of active connections", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "connections" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "E" }, "properties": [ { "id": "displayName" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, "id": 169, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_connect_metrics_connection_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_consumer_metrics_connection_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "consumer-{{host_name}}", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_producer_metrics_connection_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "producer-{{host_name}}", "range": true, "refId": "C", "useBackend": false } ], "title": "Current number of active connections", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Bytes per second read off all sockets", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 63 }, "id": 91, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by(host_name)(({__name__=~\"kafka_consumer_node_metrics_incoming_byte_rate(_bytes_per_second)?\",connect_cluster_id=~\"$connect_instance_id\",client_id!=\"\",host_name=~\"${worker:regex}\"} or on(connect_cluster_id, client_id, host_name) {__name__=~\"kafka_consumer_node_metrics_incoming_byte_rate_bytes_per_second\",connect_cluster_id=~\"$connect_instance_id\",client_id!=\"\",host_name=~\"${worker:regex}\"}))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A" } ], "title": "Incoming Byte Rate", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Average number of outgoing bytes sent per second to all servers", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 63 }, "id": 171, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) (({__name__=~\"kafka_producer_metrics_outgoing_byte_rate(_bytes_per_second)?\",connect_cluster_id=~\"$connect_instance_id\", client_id!=\"\", host_name=~\"${worker:regex}\"} or on(connect_cluster_id, client_id, host_name) {__name__=~\"kafka_producer_metrics_outgoing_byte_rate_bytes_per_second\",connect_cluster_id=~\"$connect_instance_id\", client_id!=\"\", host_name=~\"${worker:regex}\"}))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Outgoing Byte Rate", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Average number of requests sent per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 71 }, "id": 172, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_consumer_metrics_request_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "format": "time_series", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "consumer:{{host_name}}", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_producer_metrics_request_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "fullMetaSearch": false, "hide": true, "includeNullMetadata": true, "instant": false, "legendFormat": "producer:{{host_name}}", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "name": "Expression", "type": "__expr__", "uid": "__expr__" }, "expression": "$A + $B", "hide": false, "reducer": "mean", "refId": "C", "type": "math" } ], "title": "Average number of requests", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Responses received and sent per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 71 }, "id": 173, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_connect_metrics_response_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_producer_metrics_response_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "producer-{{host_name}}", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) ({__name__=~\"kafka_consumer_metrics_response_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "consumer-{{host_name}}", "range": true, "refId": "C", "useBackend": false } ], "title": "Responses received and sent", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Connections that failed authentication", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "connections" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 79 }, "id": 170, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (connect_cluster_id, host_name)({__name__=~\"kafka_connect_metrics_failed_authentication(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\",client_id!=\"\"}\nor {__name__=~\"kafka_consumer_metrics_failed_authentication(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\",client_id!=\"\"}\nor {__name__=~\"kafka_producer_metrics_failed_authentication(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\",client_id!=\"\"}\nor {__name__=~\"kafka_admin_client_metrics_failed_authentication(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\",client_id!=\"\"})", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "refId": "A" } ], "title": "Failed authentication connections", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Rebalances average time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 30, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 79 }, "id": 209, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "({__name__=~\"kafka_connect_worker_rebalance_metrics_rebalance_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\"} or on(connect_cluster_id, host_name) {__name__=~\"kafka_connect_worker_rebalance_metrics_rebalance_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\"})", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connect_cluster_id}}-{{host_name}}", "range": true, "refId": "A" } ], "title": "Rebalances average time", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 87 }, "id": 112, "panels": [], "title": "Task metrics", "type": "row" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Average size of the batches processed by the connector", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 88 }, "id": 113, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "expr": "{__name__=~\"kafka_connector_task_metrics_batch_size_avg(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}-{{task}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Batch Size Average", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Maximum size of the batches processed by the connector", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 88 }, "id": 114, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_connector_task_metrics_batch_size_max(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Batch Size Max", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Average percentage of the task’s offset commit attempts that succeeded", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 95 }, "id": 115, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_connector_task_metrics_offset_commit_success_percentage(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", task=~\"${task:regex}\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Offset commit success percentage", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "The average time in milliseconds taken by this task to commit offsets", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 95 }, "id": 116, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "({__name__=~\"kafka_connector_task_metrics_offset_commit_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"} or on(connect_cluster_id, connector, task, host_name) {__name__=~\"kafka_connector_task_metrics_offset_commit_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Offset commit Average Time", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 102 }, "id": 201, "panels": [], "title": "Task Errors metrics", "type": "row" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of failures seen by task", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 103 }, "id": 203, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) (rate({__name__=~\"kafka_task_error_metrics_total_record_failures(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}[$__rate_interval]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Total record failures rate", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of errors seen by task", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 103 }, "id": 205, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum by(host_name) (rate({__name__=~\"kafka_task_error_metrics_total_record_errors(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}[$__rate_interval]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Total record errors rate", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of records skipped by task", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 103 }, "id": 206, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) (rate({__name__=~\"kafka_task_error_metrics_total_records_skipped(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}[$__rate_interval]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Total record skipped rate", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "The number of messages that was logged into either the dead letter queue or with Log4j", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 110 }, "id": 208, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) (rate({__name__=~\"kafka_task_error_metrics_total_errors_logged(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}[$__rate_interval]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Total errors logged rate", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Total number of retries made by task", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 110 }, "id": 207, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) (rate({__name__=~\"kafka_task_error_metrics_total_retries(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}[$__rate_interval]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Total retries rate", "type": "timeseries" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "Number of produce requests to the dead letter queue", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 110 }, "id": 202, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(host_name) (rate({__name__=~\"kafka_task_error_metrics_deadletterqueue_produce_requests(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\", host_name=~\"${worker:regex}\"}[$__rate_interval]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{host_name}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Dead letter queue Produce requests rate", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 117 }, "id": 139, "panels": [], "title": "Source metrics", "type": "row" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The average time in milliseconds taken by this task to poll for a batch of source records", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 118 }, "id": 140, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "({__name__=~\"kafka_source_task_metrics_poll_batch_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"} or on(connect_cluster_id, connector, task, host_name) {__name__=~\"kafka_source_task_metrics_poll_batch_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Poll Batch Average time", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The maximum time in milliseconds taken by this task to poll for a batch of source records", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 118 }, "id": 141, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "({__name__=~\"kafka_source_task_metrics_poll_batch_max_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"} or on(connect_cluster_id, connector, task, host_name) {__name__=~\"kafka_source_task_metrics_poll_batch_max_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"})", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Poll Batch Max time", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The maximum number of records that have been produced by this task but not yet completely written to Kafka.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 118 }, "id": 145, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_source_task_metrics_source_record_active_count_max(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Source Record Active Count max", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The average per-second number of records produced/polled (before transformation) by this task belonging to the named source connector in this worker.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 125 }, "id": 144, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_source_task_metrics_source_record_poll_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Source Record Poll rate", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The average per-second number of records output from the transformations and written to Kafka for this task belonging to the named source connector in this worker. This is after transformations are applied and excludes any records filtered out by the transformations.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 125 }, "id": 143, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_source_task_metrics_source_record_write_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Source Record Write rate", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The average number of records that have been produced by this task but not yet completely written to Kafka.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 125 }, "id": 142, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_source_task_metrics_source_record_active_count_avg(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Source Record Active Count average", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 132 }, "id": 134, "panels": [], "title": "Sink metrics", "type": "row" }, { "datasource": { "default": true, "type": "prometheus", "uid": "${datasource}" }, "description": "The average time in milliseconds taken by this task to put a batch of sinks records", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 133 }, "id": 136, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "({__name__=~\"kafka_sink_task_metrics_put_batch_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\", host_name=~\"${worker:regex}\"} or on(connect_cluster_id, connector, task, host_name) {__name__=~\"kafka_sink_task_metrics_put_batch_avg_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\", host_name=~\"${worker:regex}\"})", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Put Batch Average time", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "kafka_sink_task_metrics_partition_count", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 133 }, "id": 237, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_sink_task_metrics_partition_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\", host_name=~\"${worker:regex}\"}", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "partition count assigned", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "The maximum time in milliseconds taken by this task to put a batch of sinks records", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 142 }, "id": 137, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "({__name__=~\"kafka_sink_task_metrics_put_batch_max_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\", host_name=~\"${worker:regex}\"} or on(connect_cluster_id, connector, task, host_name) {__name__=~\"kafka_sink_task_metrics_put_batch_max_time_ms(_milliseconds)?\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\", host_name=~\"${worker:regex}\"})", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "Put Batch Max time", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "record read per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "recps" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 142 }, "id": 236, "options": { "legend": { "calcs": [ "min", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "desc" } }, "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "{__name__=~\"kafka_sink_task_metrics_sink_record_read_rate(_ratio)?_per_second\",connect_cluster_id=~\"$connect_instance_id\", connector=~\"${connector:regex}\", task=~\"${task:regex}\", host_name=~\"${worker:regex}\"}", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{connector}}-{{task}}", "refId": "A", "useBackend": false } ], "title": "record read per second", "type": "timeseries" } ], "refresh": "1m", "schemaVersion": 39, "tags": [ "automq", "kafka-connect", "remote-write" ], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\"},environment_id)", "hide": 0, "includeAll": true, "label": "envId", "multi": true, "name": "envId", "options": [], "query": { "qryType": 1, "query": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\"},environment_id)", "refId": "PrometheusVariableQueryEditor-envId" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\"},automq_instance_id)", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": { "qryType": 1, "query": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\"},automq_instance_id)", "refId": "PrometheusVariableQueryEditor-instance" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",automq_instance_id=~\"${instance:regex}\",environment_id=~\"${envId:regex}\"},connect_cluster_id)", "hide": 0, "includeAll": true, "label": "Connect Cluster ID", "multi": true, "name": "connect_instance_id", "options": [], "query": { "qryType": 1, "query": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",automq_instance_id=~\"${instance:regex}\",environment_id=~\"${envId:regex}\"},connect_cluster_id)", "refId": "PrometheusVariableQueryEditor-connect_instance_id" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_running_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\"},connector)", "hide": 0, "includeAll": true, "label": "Connector name", "multi": false, "name": "connector", "options": [], "query": { "qryType": 1, "query": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_running_task_count(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\"},connector)", "refId": "PrometheusVariableQueryEditor-connector" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"},host_name)", "hide": 0, "includeAll": true, "label": "worker", "multi": true, "name": "worker", "options": [], "query": { "qryType": 1, "query": "label_values({__name__=~\"kafka_connect_worker_metrics_connector_total_task_count(_ratio)?_total\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\"},host_name)", "refId": "PrometheusVariableQueryEditor-worker" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values({__name__=~\"kafka_connector_task_metrics_batch_size_avg(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\",host_name=~\"${worker:regex}\"},task)", "hide": 0, "includeAll": true, "label": "task", "multi": true, "name": "task", "options": [], "query": { "qryType": 1, "query": "label_values({__name__=~\"kafka_connector_task_metrics_batch_size_avg(_ratio)?\",connect_cluster_id=~\"$connect_instance_id\",connector=~\"${connector:regex}\",host_name=~\"${worker:regex}\"},task)", "refId": "PrometheusVariableQueryEditor-task" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": { "refresh_intervals": [ "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "Kafka Connect cluster", "uid": "automq-connect-customer", "version": 2, "weekStart": "" }