apiVersion: v1
kind: ConfigMap
metadata:
  name: kafka-exporter-yc-dashboard
  namespace: prometheus-stack
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: Kafka
data:
  kafka-exporter-yc.json: |-
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"links":[],"panels":[{"id":1,"type":"row","title":"YC Kafka exporter","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[]},{"id":2,"type":"stat","title":"Exporter up","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"max(up{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\"})","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"red"},{"color":"green","value":1}]}},"overrides":[]},"options":{"colorMode":"background","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":3,"type":"stat","title":"Topics without new messages for 12h","gridPos":{"h":4,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"count(((sum by (topic) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}[12h])) == 0) and on(topic) (min by (topic) (count_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}[12h])) >= 1400) and on(topic) (sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}) > 0)))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1},{"color":"red","value":5}]}},"overrides":[]},"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":4,"type":"stat","title":"Known topics","gridPos":{"h":4,"w":9,"x":15,"y":1},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"count(count by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"}},{"id":5,"type":"table","title":"Topics with no offset growth for 12h","gridPos":{"h":8,"w":24,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"((sum by (topic, kafka_instance, cluster) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}[12h])) == 0) and on(topic) (min by (topic) (count_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}[12h])) >= 1400) and on(topic) (sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"}) > 0))","refId":"A","range":false,"instant":true}],"fieldConfig":{"defaults":{"custom":{"align":"auto","cellOptions":{"type":"auto"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"showHeader":true,"cellHeight":"sm"},"transformations":[{"id":"labelsToFields","options":{"mode":"columns"}}]},{"id":10,"type":"row","title":"Topic traffic","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[]},{"id":11,"type":"timeseries","title":"Topic offset delta by $__rate_interval","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (clamp_min(delta(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"$topic\"}[$__rate_interval]), 0))","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":12,"type":"timeseries","title":"Topic offset delta over 12h","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (max_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"$topic\"}[12h]) - min_over_time(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"$topic\"}[12h]))","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":13,"type":"timeseries","title":"Current topic offset","gridPos":{"h":8,"w":12,"x":0,"y":22},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"$topic\"})","refId":"A","range":true,"legendFormat":"{{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}},{"id":14,"type":"bargauge","title":"Partitions by topic","gridPos":{"h":8,"w":12,"x":12,"y":22},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (topic) (kafka_topic_partitions{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"$topic\"})","refId":"A","range":false,"legendFormat":"{{topic}}","instant":true}],"fieldConfig":{"defaults":{"unit":"short","mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"displayMode":"gradient","legend":{"displayMode":"list","placement":"bottom","showLegend":false},"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true}},{"id":20,"type":"row","title":"Consumer groups","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":30},"panels":[]},{"id":21,"type":"timeseries","title":"Consumer lag by group/topic","gridPos":{"h":8,"w":24,"x":0,"y":31},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"targets":[{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"editorMode":"code","expr":"sum by (consumergroup, topic) (kafka_consumergroup_lag_sum{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"$topic\"})","refId":"A","range":true,"legendFormat":"{{consumergroup}} / {{topic}}"}],"fieldConfig":{"defaults":{"unit":"short","custom":{"drawStyle":"line","lineInterpolation":"linear","lineWidth":1,"fillOpacity":10,"spanNulls":false,"showPoints":"auto","axisPlacement":"auto","scaleDistribution":{"type":"linear"},"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}}}],"preload":false,"refresh":"30s","schemaVersion":41,"tags":["kafka","kafka-exporter","brusnika-prod"],"templating":{"list":[{"current":{"text":"VictoriaMetrics","value":"vm"},"includeAll":false,"label":"Data Source","name":"ds_datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"current":{"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"definition":"label_values(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"},topic)","allValue":".*","includeAll":true,"label":"Topic","name":"topic","options":[],"query":{"qryType":1,"query":"label_values(kafka_topic_partition_current_offset{kafka_instance=\"yc-kafka\",cluster=\"brusnika-prod\",topic=~\"^(prod|system_log)\"},topic)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"regex":"","type":"query"}]},"time":{"from":"now-12h","to":"now"},"timepicker":{},"timezone":"","title":"Kafka Exporter / YC Kafka","uid":"kafka-exporter-yc","version":1}