Fix brusnika observability alerting and grafana secrets

clusters/brusnika-prod/infrastructure/patches/prometheus-stack.yaml

@@ -51,6 +51,14 @@ spec:
               - op: replace
                 path: /spec/groups/0/rules/4/labels/cluster
                 value: brusnika-prod
+              - op: replace
+                path: /spec/groups/0/rules/4/expr
+                value: >-
+                  (
+                    sum(rate(container_cpu_cfs_throttled_periods_total{container!="",namespace!="kube-system"}[15m])) by (container, pod, namespace, scope)
+                    /
+                    sum(rate(container_cpu_cfs_periods_total{container!="",namespace!="kube-system"}[15m])) by (container, pod, namespace, scope)
+                  ) > 0.80
               - op: replace
                 path: /spec/groups/0/rules/5/labels/cluster
                 value: brusnika-prod
@@ -245,26 +253,17 @@ spec:
     grafana:
       enabled: true
       adminUser: grafana-admin
-      admin:
+      podAnnotations:
-        existingSecret: grafana-admin
+        vault.hashicorp.com/agent-init-first: "true"
-        userKey: admin-user
+        vault.hashicorp.com/agent-inject: "true"
-        passwordKey: admin-password
+        vault.hashicorp.com/agent-pre-populate-only: "true"
-      extraObjects:
+        vault.hashicorp.com/auth-path: auth/kubernetes
-        - |
+        vault.hashicorp.com/role: grafana-admin
-          {{- $secret := lookup "v1" "Secret" .Release.Namespace "grafana-admin" }}
+        vault.hashicorp.com/agent-inject-secret-grafana-admin-password: secrets/data/vault/apps/grafana-admin
-          apiVersion: v1
+        vault.hashicorp.com/agent-inject-template-grafana-admin-password: |-
-          kind: Secret
+          {{- with secret "secrets/data/vault/apps/grafana-admin" -}}
-          metadata:
+          {{ index .Data.data "admin-password" }}
-            name: grafana-admin
+          {{- end -}}
-            namespace: {{ .Release.Namespace }}
-            labels:
-              app.kubernetes.io/name: grafana
-              app.kubernetes.io/instance: {{ .Release.Name }}
-              app.kubernetes.io/managed-by: Helm
-          type: Opaque
-          data:
-            admin-user: {{ .Values.adminUser | b64enc | quote }}
-            admin-password: {{ if $secret }}{{ index $secret.data "admin-password" | quote }}{{ else }}{{ randAlphaNum 40 | b64enc | quote }}{{ end }}
       persistence:
         enabled: true
         type: sts
@@ -277,6 +276,8 @@ spec:
       env:
         GF_SERVER_DOMAIN: grafana.brusnika.onprem.sarex.io
         GF_SERVER_ROOT_URL: https://grafana.brusnika.onprem.sarex.io/
+        GF_SECURITY_ADMIN_USER: grafana-admin
+        GF_SECURITY_ADMIN_PASSWORD__FILE: /vault/secrets/grafana-admin-password
       sidecar:
         dashboards:
           enabled: true

clusters/brusnika-stage/infrastructure/patches/prometheus-stack.yaml

@@ -245,26 +245,17 @@ spec:
     grafana:
       enabled: true
       adminUser: grafana-admin
-      admin:
+      podAnnotations:
-        existingSecret: grafana-admin
+        vault.hashicorp.com/agent-init-first: "true"
-        userKey: admin-user
+        vault.hashicorp.com/agent-inject: "true"
-        passwordKey: admin-password
+        vault.hashicorp.com/agent-pre-populate-only: "true"
-      extraObjects:
+        vault.hashicorp.com/auth-path: auth/kubernetes
-        - |
+        vault.hashicorp.com/role: grafana-admin
-          {{- $secret := lookup "v1" "Secret" .Release.Namespace "grafana-admin" }}
+        vault.hashicorp.com/agent-inject-secret-grafana-admin-password: secrets/data/vault/apps/grafana-admin
-          apiVersion: v1
+        vault.hashicorp.com/agent-inject-template-grafana-admin-password: |-
-          kind: Secret
+          {{- with secret "secrets/data/vault/apps/grafana-admin" -}}
-          metadata:
+          {{ index .Data.data "admin-password" }}
-            name: grafana-admin
+          {{- end -}}
-            namespace: {{ .Release.Namespace }}
-            labels:
-              app.kubernetes.io/name: grafana
-              app.kubernetes.io/instance: {{ .Release.Name }}
-              app.kubernetes.io/managed-by: Helm
-          type: Opaque
-          data:
-            admin-user: {{ .Values.adminUser | b64enc | quote }}
-            admin-password: {{ if $secret }}{{ index $secret.data "admin-password" | quote }}{{ else }}{{ randAlphaNum 40 | b64enc | quote }}{{ end }}
       persistence:
         enabled: true
         type: sts
@@ -277,6 +268,8 @@ spec:
       env:
         GF_SERVER_DOMAIN: grafana.test.sarex.brusnika.tech
         GF_SERVER_ROOT_URL: https://grafana.test.sarex.brusnika.tech/
+        GF_SECURITY_ADMIN_USER: grafana-admin
+        GF_SECURITY_ADMIN_PASSWORD__FILE: /vault/secrets/grafana-admin-password
       sidecar:
         dashboards:
           enabled: true