diff --git a/clusters/brusnika-stage/infrastructure/istio-gateway-podmonitor.yaml b/clusters/brusnika-stage/infrastructure/istio-gateway-podmonitor.yaml deleted file mode 100644 index 7db47a2..0000000 --- a/clusters/brusnika-stage/infrastructure/istio-gateway-podmonitor.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: istio-ingressgateway-stats - namespace: prometheus-stack - labels: - release: prometheus -spec: - namespaceSelector: - matchNames: - - ingress-nginx - selector: - matchLabels: - istio: ingressgateway - podMetricsEndpoints: - - path: /stats/prometheus - targetPort: 15020 - interval: 15s diff --git a/clusters/brusnika-stage/infrastructure/istio-gateway-stats-scrape.yaml b/clusters/brusnika-stage/infrastructure/istio-gateway-stats-scrape.yaml new file mode 100644 index 0000000..a0f3295 --- /dev/null +++ b/clusters/brusnika-stage/infrastructure/istio-gateway-stats-scrape.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway-stats + namespace: ingress-nginx + labels: + istio: ingressgateway-stats +spec: + selector: + istio: ingressgateway + ports: + - name: stats + port: 15020 + targetPort: 15020 +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + name: istio-ingressgateway-stats + namespace: prometheus-stack +spec: + namespaceSelector: + matchNames: + - ingress-nginx + selector: + matchLabels: + istio: ingressgateway-stats + endpoints: + - port: stats + path: /stats/prometheus + interval: 15s diff --git a/clusters/brusnika-stage/infrastructure/kustomization.yaml b/clusters/brusnika-stage/infrastructure/kustomization.yaml index 0db9281..e153780 100644 --- a/clusters/brusnika-stage/infrastructure/kustomization.yaml +++ b/clusters/brusnika-stage/infrastructure/kustomization.yaml @@ -18,7 +18,8 @@ resources: - ./lb-service-override.yaml - ./vault-ingress.yaml - ./clusterissuer-letsencrypt.yaml - - ./istio-gateway-podmonitor.yaml + - ./node-exporter-vmnodescrape.yaml + - ./istio-gateway-stats-scrape.yaml - ./istio-dashboard-compat-vmrule.yaml - ../../../infrastructure/failed-pod-cleanup patches: diff --git a/clusters/brusnika-stage/infrastructure/node-exporter-vmnodescrape.yaml b/clusters/brusnika-stage/infrastructure/node-exporter-vmnodescrape.yaml new file mode 100644 index 0000000..b090585 --- /dev/null +++ b/clusters/brusnika-stage/infrastructure/node-exporter-vmnodescrape.yaml @@ -0,0 +1,20 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMNodeScrape +metadata: + name: vm-prod-node-exporter + namespace: vmstack +spec: + path: /metrics + port: "9100" + scheme: http + selector: {} + relabelConfigs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - action: replace + sourceLabels: + - __meta_kubernetes_node_name + targetLabel: node + - action: replace + replacement: vm-stack/vm-prod-node-exporter + targetLabel: job diff --git a/clusters/brusnika-stage/infrastructure/patches/prometheus-stack.yaml b/clusters/brusnika-stage/infrastructure/patches/prometheus-stack.yaml index 2764b54..f3c4611 100644 --- a/clusters/brusnika-stage/infrastructure/patches/prometheus-stack.yaml +++ b/clusters/brusnika-stage/infrastructure/patches/prometheus-stack.yaml @@ -6,6 +6,19 @@ metadata: spec: interval: 5m timeout: 20m + postRenderers: + - kustomize: + patches: + - target: + version: v1 + kind: ConfigMap + name: prometheus-stack-k8s-cluster-health + namespace: prometheus-stack + patch: |- + - op: replace + path: /data/k8s-cluster-health.json + value: |- + {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":4400,"links":[],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":1,"panels":[],"title":"Nodes — состояние и ресурсы","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":1},"id":2,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"expr":"sum(max by (node) (kube_node_status_condition{condition=\"Ready\",status=\"true\"}))","instant":true,"refId":"A"}],"title":"Nodes Ready (по узлам)","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":4,"w":18,"x":6,"y":1},"id":3,"options":{"displayMode":"gradient","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"sum by (condition) (max by (node,condition) (kube_node_status_condition{status=\"true\"}))","instant":true,"legendFormat":"{{label_name}}","refId":"A"}],"title":"Node conditions (true) по типу","type":"bargauge"},{"datasource":{"uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[{"options":{"0":{"color":"#808080","index":0},"1":{"color":"red","index":1}},"type":"value"}]},"overrides":[]},"gridPos":{"h":13,"w":24,"x":0,"y":5},"id":4,"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"pieType":"pie","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"max by (node, key, value, effect) (kube_node_spec_taint{effect=\"NoSchedule\"})","instant":true,"legendFormat":"{{ node }} | {{ key }}={{ value }}","range":false,"refId":"A"}],"title":"Node taints (NoSchedule)","type":"piechart"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":6,"w":12,"x":0,"y":18},"id":7,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"max by (node) (kube_node_status_allocatable{resource=\"memory\"})","instant":false,"legendFormat":"{{ node }}","range":true,"refId":"A"}],"title":"Allocatable Memory per node","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"cores"},"overrides":[]},"gridPos":{"h":6,"w":12,"x":12,"y":18},"id":6,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"max by (node) (kube_node_status_allocatable{resource=\"cpu\"})","instant":false,"legendFormat":"{{ node }}","range":true,"refId":"A"}],"title":"Allocatable CPU (cores) per node","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":24},"id":10,"panels":[],"title":"Pods — статусы и рестарты","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"gridPos":{"h":6,"w":12,"x":0,"y":25},"id":11,"options":{"displayMode":"gradient","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"sum by (phase) (kube_pod_status_phase{namespace=~\"$namespace\"})","instant":true,"legendFormat":"{{label_name}}","refId":"A"}],"title":"Pod phase (по фазам, namespace=$namespace)","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"Error":{"color":"red","index":0}},"type":"value"}],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":6,"w":12,"x":12,"y":25},"id":14,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":" sum by (reason) (\n kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\"}\n )","instant":true,"legendFormat":"{{ reason }}","refId":"A"}],"title":"Last terminated reason (containers) [namespace=$namespace]","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":24,"x":0,"y":31},"id":13,"options":{"displayMode":"gradient","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"topk(10, sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[$__rate_interval])))","instant":true,"legendFormat":"{{ namespace }} / {{ pod }}","refId":"A"}],"title":"Top pod restarts (Δ за $__rate_interval) [namespace=$namespace]","type":"bargauge"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":20,"panels":[],"title":"Jobs / CronJobs","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":7,"x":0,"y":39},"id":21,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"expr":"sum(kube_job_status_failed)","instant":true,"refId":"A"}],"title":"Jobs failed","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":8,"x":7,"y":39},"id":22,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"expr":"sum(kube_job_status_active)","instant":true,"refId":"A"}],"title":"Jobs active","type":"stat"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":3,"w":9,"x":15,"y":39},"id":23,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.6.1","targets":[{"expr":"sum(kube_job_status_succeeded)","instant":true,"refId":"A"}],"title":"Jobs succeeded","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":42},"id":30,"panels":[],"title":"Storage — PV/PVC и объём","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[]},"overrides":[]},"gridPos":{"h":8,"w":6,"x":0,"y":43},"id":31,"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"pieType":"donut","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"expr":"sum by (phase) (kube_persistentvolumeclaim_status_phase)","instant":true,"refId":"A"}],"title":"PVC status (по фазам)","type":"piechart"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[]},"overrides":[]},"gridPos":{"h":8,"w":6,"x":6,"y":43},"id":32,"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"pieType":"donut","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"expr":"sum by (phase) (kube_persistentvolume_status_phase)","instant":true,"refId":"A"}],"title":"PV status (по фазам)","type":"piechart"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":43},"id":33,"options":{"displayMode":"gradient","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"topk(\n 10,\n (1 - (\n sum by (namespace, persistentvolumeclaim) (kubelet_volume_stats_available_bytes{persistentvolumeclaim!=\"\"})\n /\n sum by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{persistentvolumeclaim!=\"\"})\n )) * 100\n)","instant":true,"legendFormat":"{{ persistentvolumeclaim }}","refId":"A"}],"title":"Top PVC usage (util%)","type":"bargauge"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":51},"id":40,"panels":[],"title":"Workloads — StatefulSets и HPA","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":52},"id":41,"options":{"displayMode":"lcd","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"kube_statefulset_replicas{namespace=~\"$namespace\"}","format":"time_series","instant":true,"legendFormat":"{{namespace}}/{{statefulset}} - Desired","range":false,"refId":"A"}],"title":"StatefulSet replicas (desired)","transformations":[{"id":"labelsToFields","options":{"mode":"seriesToRows","valueLabel":"statefulset"}}],"type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":52},"id":55,"options":{"displayMode":"lcd","legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","exemplar":false,"expr":"kube_statefulset_status_replicas_ready{namespace=~\"$namespace\"}","format":"time_series","instant":true,"legendFormat":"{{namespace}}/{{statefulset}} - Desired","range":false,"refId":"A"}],"title":"StatefulSet replicas (desired)","transformations":[{"id":"labelsToFields","options":{"mode":"seriesToRows","valueLabel":"statefulset"}}],"type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","fillOpacity":80,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineWidth":1,"scaleDistribution":{"type":"linear"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":60},"id":42,"options":{"barRadius":0,"barWidth":0.97,"fullHighlight":false,"groupWidth":0.7,"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"orientation":"auto","showValue":"auto","stacking":"none","tooltip":{"hideZeros":false,"mode":"single","sort":"none"},"xTickLabelRotation":0,"xTickLabelSpacing":0},"pluginVersion":"11.6.1","targets":[{"expr":"kube_horizontalpodautoscaler_spec_min_replicas","instant":true,"refId":"A"}],"title":"HPA min","transformations":[{"id":"labelsToFields","options":{"mode":"seriesToRows","valueLabel":"horizontalpodautoscaler"}}],"type":"barchart"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","fillOpacity":80,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineWidth":1,"scaleDistribution":{"type":"linear"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":60},"id":56,"options":{"barRadius":0,"barWidth":0.97,"fullHighlight":false,"groupWidth":0.7,"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"orientation":"auto","showValue":"auto","stacking":"none","tooltip":{"hideZeros":false,"mode":"single","sort":"none"},"xTickLabelRotation":0,"xTickLabelSpacing":0},"pluginVersion":"11.6.1","targets":[{"expr":"kube_horizontalpodautoscaler_spec_max_replicas","instant":true,"refId":"B"}],"title":"HPA max","transformations":[{"id":"labelsToFields","options":{"mode":"seriesToRows","valueLabel":"horizontalpodautoscaler"}}],"type":"barchart"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","fillOpacity":80,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineWidth":1,"scaleDistribution":{"type":"linear"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":60},"id":57,"options":{"barRadius":0,"barWidth":0.97,"fullHighlight":false,"groupWidth":0.7,"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"orientation":"auto","showValue":"auto","stacking":"none","tooltip":{"hideZeros":false,"mode":"single","sort":"none"},"xTickLabelRotation":0,"xTickLabelSpacing":0},"pluginVersion":"11.6.1","targets":[{"expr":"kube_horizontalpodautoscaler_status_desired_replicas","instant":true,"refId":"C"}],"title":"HPA min/max/desired","transformations":[{"id":"labelsToFields","options":{"mode":"seriesToRows","valueLabel":"horizontalpodautoscaler"}}],"type":"barchart"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"hideFrom":{"legend":false,"tooltip":false,"viz":false}},"mappings":[]},"overrides":[]},"gridPos":{"h":9,"w":24,"x":0,"y":68},"id":43,"options":{"legend":{"displayMode":"list","placement":"bottom","showLegend":true},"pieType":"pie","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"editorMode":"code","expr":"sum by (condition) (max by (horizontalpodautoscaler,condition) (kube_horizontalpodautoscaler_status_condition{status=\"true\"}))","instant":true,"legendFormat":"{{label_name}}","refId":"A"}],"title":"HPA conditions (true) по типу","type":"piechart"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":77},"id":50,"panels":[],"title":"API Server — запросы и задержки","type":"row"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":78},"id":51,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"expr":"sum by (code) (rate(apiserver_request_total[$__rate_interval]))","range":true,"refId":"A"}],"title":"API requests rate by code (apiserver_request_total)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":78},"id":52,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"expr":"sum by (code) (rate(rest_client_requests_total{code=~\"4..|5..\"}[$__rate_interval]))","range":true,"refId":"A"}],"title":"Client REST 4xx/5xx (rest_client_requests_total)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":6,"w":12,"x":0,"y":86},"id":54,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"11.6.1","targets":[{"expr":"histogram_quantile(0.99, sum by (verb,le) (rate(apiserver_request_duration_seconds_bucket[$__rate_interval])))","range":true,"refId":"A"}],"title":"API latency p99 by verb","type":"timeseries"}],"preload":false,"refresh":"30s","schemaVersion":41,"tags":["kubernetes","kube-state-metrics","apiserver"],"templating":{"list":[{"current":{"text":"VictoriaMetrics","value":"vm"},"label":"Data Source","name":"ds_datasource","options":[],"query":"prometheus","refresh":1,"type":"datasource"},{"current":{"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"includeAll":true,"label":"Namespace","name":"namespace","options":[],"query":"label_values(kube_pod_info, namespace)","refresh":1,"type":"query"},{"current":{"text":"All","value":"$__all"},"datasource":{"type":"prometheus","uid":"${ds_datasource}"},"definition":"label_values({job=\"vm-stack/vm-prod-node-exporter\"},node)","includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"qryType":1,"query":"label_values({job=\"vm-stack/vm-prod-node-exporter\"},node)","refId":"PrometheusVariableQueryEditor-VariableQuery"},"refresh":1,"type":"query","allValue":".*"}]},"time":{"from":"now-3h","to":"now"},"timepicker":{},"timezone":"","title":"Kubernetes / Cluster Health","uid":"k8s-cluster-health","version":7} install: disableWait: true upgrade: diff --git a/clusters/brusnika-stage/infrastructure/patches/vmstack.yaml b/clusters/brusnika-stage/infrastructure/patches/vmstack.yaml index e2a6c40..aff411e 100644 --- a/clusters/brusnika-stage/infrastructure/patches/vmstack.yaml +++ b/clusters/brusnika-stage/infrastructure/patches/vmstack.yaml @@ -86,7 +86,7 @@ spec: - __meta_kubernetes_pod_node_name targetLabel: node - targetLabel: job - replacement: brusnika-stage/vm-node-exporter + replacement: vm-stack/vm-prod-node-exporter metricRelabelConfigs: - action: drop