feat: prometheus alerting

deploy/dev/kustomization.yaml

@@ -22,6 +22,17 @@ configMapGenerator:
   behavior: merge
   files:
   - prometheus.yml=prometheus.yaml
+  - alerting_rules.yml=prometheus-alerting-rules.yaml
+  options:
+    labels:
+      app: prometheus
+- name: prometheus-alertmanager
+  behavior: merge
+  files:
+  - alertmanager.yml=prometheus-alertmanager.yaml
+  options:
+    labels:
+      app: prometheus
 - name: grafana
   behavior: merge
   files:

deploy/dev/prometheus-alerting-rules.yaml

@@ -0,0 +1 @@
+{}

deploy/dev/prometheus-alertmanager.yaml

@@ -0,0 +1,17 @@
+global: {}
+receivers:
+- name: mailgun-receiver
+  email_configs:
+  - to: alerts@netflux.io
+    from: alerts@netflux.io
+    smarthost: smtp.eu.mailgun.org:587
+    auth_username: test@example.com
+    auth_password: foobar
+    send_resolved: true
+route:
+  group_interval: 5m
+  group_wait: 10s
+  receiver: mailgun-receiver
+  repeat_interval: 3h
+templates:
+- /etc/alertmanager/*.tmpl

deploy/dev/prometheus.yaml

@@ -296,15 +296,12 @@ alerting:
     - source_labels: [__meta_kubernetes_namespace]
       regex: default
       action: keep
-    - source_labels: [__meta_kubernetes_pod_label_app]
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
       regex: prometheus
       action: keep
-    - source_labels: [__meta_kubernetes_pod_label_component]
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
       regex: alertmanager
       action: keep
-    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_probe]
-      regex: .*
-      action: keep
     - source_labels: [__meta_kubernetes_pod_container_port_number]
       regex: "9093"
       action: keep

deploy/prod/kustomization.yaml

@@ -11,6 +11,17 @@ configMapGenerator:
   behavior: merge
   files:
   - prometheus.yml=prometheus.yaml
+  - alerting_rules.yml=prometheus-alerting-rules.yaml
+  options:
+    labels:
+      app: prometheus
+- name: prometheus-alertmanager
+  behavior: merge
+  files:
+  - alertmanager.yml=secrets/prometheus-alertmanager.yaml
+  options:
+    labels:
+      app: prometheus
 - name: grafana
   behavior: merge
   files:

deploy/prod/prometheus-alerting-rules.yaml

@@ -0,0 +1,10 @@
+groups:
+- name: default-group
+  rules:
+  - alert: DBRootFSUsed
+    expr: 100 - ((node_filesystem_avail_bytes{instance="prod-db:9100",job="node",mountpoint="/",fstype!="rootfs"} * 100) / node_filesystem_size_bytes{instance="prod-db:9100",job="node",mountpoint="/",fstype!="rootfs"}) > 85
+    for: 10m
+    labels:
+      severity: alert
+    annotations:
+      summary: database disk space

deploy/prod/prometheus.yaml

@@ -305,15 +305,12 @@ alerting:
     - source_labels: [__meta_kubernetes_namespace]
       regex: default
       action: keep
-    - source_labels: [__meta_kubernetes_pod_label_app]
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
       regex: prometheus
       action: keep
-    - source_labels: [__meta_kubernetes_pod_label_component]
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
       regex: alertmanager
       action: keep
-    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_probe]
-      regex: .*
-      action: keep
     - source_labels: [__meta_kubernetes_pod_container_port_number]
       regex: "9093"
       action: keep