diff --git a/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-dashboard.yaml b/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-dashboard.yaml index 5ab4e31..139bc79 100644 --- a/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-dashboard.yaml +++ b/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-dashboard.yaml @@ -9,4 +9,4 @@ metadata: grafana_folder: Kafka data: kafka-exporter-yc.json: |- - {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"links":[],"panels":[{"id":1,"type":"row","title":"YC Kafka exporter","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[]},{"id":2,"type":"stat","title":"Exporter up","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"max(up{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\"})","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"red"},{"color":"green","value":1}]}},"overrides":[]},"options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":3,"type":"stat","title":"Topics without new messages for 12h","gridPos":{"h":4,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"count((sum by (topic) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h])) == 0) and on(topic) (sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}) > 0))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1},{"color":"red","value":5}]}},"overrides":[]},"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":4,"type":"stat","title":"Known topics","gridPos":{"h":4,"w":9,"x":15,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"count(count by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":5,"type":"table","title":"Topics with no offset growth for 12h","gridPos":{"h":8,"w":24,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"(sum by (topic, kafka_instance, cluster) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h])) == 0) and on(topic) (sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}) > 0)","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"custom":{"align":"auto","cellOptions":{"type":"auto"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"showHeader":true,"cellHeight":"sm"},"transformations":[{"id":"labelsToFields","options":{"mode":"columns"}}]},{"id":10,"type":"row","title":"Topic traffic","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[]},{"id":11,"type":"timeseries","title":"Topic offset delta by $__rate_interval","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (clamp_min(delta(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"}[$__rate_interval]), 0))","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":12,"type":"timeseries","title":"Topic offset delta over 12h","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"}[12h]))","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":13,"type":"timeseries","title":"Current topic offset","gridPos":{"h":8,"w":12,"x":0,"y":22},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"})","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":14,"type":"bargauge","title":"Partitions by topic","gridPos":{"h":8,"w":12,"x":12,"y":22},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"})","refId":"A","range":false,"legendFormat":"{{topic}}","instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"displayMode":"gradient","legend":{"displayMode":"list","placement":"bottom","showLegend":false},"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true}},{"id":20,"type":"row","title":"Consumer groups","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":30},"panels":[]},{"id":21,"type":"timeseries","title":"Consumer lag by group/topic","gridPos":{"h":8,"w":24,"x":0,"y":31},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (consumergroup, topic) (kafka_consumergroup_lag_sum{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"})","refId":"A","range":true,"legendFormat":"{{consumergroup}} / {{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}}],"preload":false,"refresh":"30s","schemaVersion":41,"tags":["kafka","kafka-exporter","brusnika-stage"],"templating":{"list":[{"current":{"text":"VictoriaMetrics","value":"vm"},"includeAll":false,"label":"Data Source","name":"ds_datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"definition":"label_values(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\"},topic)","allValue":".*","includeAll":true,"label":"Topic","name":"topic","options":[],"query":{"qryType":1,"query":"label_values(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\"},topic)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"}]},"time":{"from":"now-12h","to":"now"},"timepicker":{},"timezone":"","title":"Kafka Exporter / YC Kafka","uid":"kafka-exporter-yc","version":1} + {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"links":[],"panels":[{"id":1,"type":"row","title":"YC Kafka exporter","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[]},{"id":2,"type":"stat","title":"Exporter up","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"max(up{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\"})","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"red"},{"color":"green","value":1}]}},"overrides":[]},"options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":3,"type":"stat","title":"Topics without new messages for 12h","gridPos":{"h":4,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"count(((sum by (topic) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h])) == 0) and on(topic) (min by (topic) (count_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h])) >= 1400) and on(topic) (sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}) > 0)))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1},{"color":"red","value":5}]}},"overrides":[]},"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":4,"type":"stat","title":"Known topics","gridPos":{"h":4,"w":9,"x":15,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"count(count by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":5,"type":"table","title":"Topics with no offset growth for 12h","gridPos":{"h":8,"w":24,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"((sum by (topic, kafka_instance, cluster) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h])) == 0) and on(topic) (min by (topic) (count_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}[12h])) >= 1400) and on(topic) (sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic!~\"^__.*\"}) > 0))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"custom":{"align":"auto","cellOptions":{"type":"auto"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"showHeader":true,"cellHeight":"sm"},"transformations":[{"id":"labelsToFields","options":{"mode":"columns"}}]},{"id":10,"type":"row","title":"Topic traffic","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[]},{"id":11,"type":"timeseries","title":"Topic offset delta by $__rate_interval","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (clamp_min(delta(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"}[$__rate_interval]), 0))","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":12,"type":"timeseries","title":"Topic offset delta over 12h","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"}[12h]))","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":13,"type":"timeseries","title":"Current topic offset","gridPos":{"h":8,"w":12,"x":0,"y":22},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"})","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":14,"type":"bargauge","title":"Partitions by topic","gridPos":{"h":8,"w":12,"x":12,"y":22},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"})","refId":"A","range":false,"legendFormat":"{{topic}}","instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"displayMode":"gradient","legend":{"displayMode":"list","placement":"bottom","showLegend":false},"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true}},{"id":20,"type":"row","title":"Consumer groups","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":30},"panels":[]},{"id":21,"type":"timeseries","title":"Consumer lag by group/topic","gridPos":{"h":8,"w":24,"x":0,"y":31},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (consumergroup, topic) (kafka_consumergroup_lag_sum{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\",topic=~\"$topic\"})","refId":"A","range":true,"legendFormat":"{{consumergroup}} / {{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}}],"preload":false,"refresh":"30s","schemaVersion":41,"tags":["kafka","kafka-exporter","brusnika-stage"],"templating":{"list":[{"current":{"text":"VictoriaMetrics","value":"vm"},"includeAll":false,"label":"Data Source","name":"ds_datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"definition":"label_values(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\"},topic)","allValue":".*","includeAll":true,"label":"Topic","name":"topic","options":[],"query":{"qryType":1,"query":"label_values(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-stage\"},topic)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"}]},"time":{"from":"now-12h","to":"now"},"timepicker":{},"timezone":"","title":"Kafka Exporter / YC Kafka","uid":"kafka-exporter-yc","version":1} diff --git a/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-rules.yaml b/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-rules.yaml index 451f469..59517e2 100644 --- a/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-rules.yaml +++ b/clusters/brusnika-stage/infrastructure/kafka-exporter-yc-rules.yaml @@ -30,6 +30,12 @@ spec: ) == 0 ) and on (topic, kafka_instance, source_cluster, cluster) + ( + min by (topic, kafka_instance, source_cluster, cluster) ( + count_over_time(kafka_topic_partition_current_offset{kafka_instance="yc-kafka", cluster="brusnika-stage", topic!~"^__.*"}[12h]) + ) >= 1400 + ) + and on (topic, kafka_instance, source_cluster, cluster) ( sum by (topic, kafka_instance, source_cluster, cluster) ( kafka_topic_partitions{kafka_instance="yc-kafka", cluster="brusnika-stage", topic!~"^__.*"}