IT4Change/prometheus
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: IT4Change:725c370dd1d908cdf1a6a2486f98614eddffc824
Branches Tags
IT4Change:master
...
compare: IT4Change:61ed6185af410c09ce6446bdf79bea7627ea8921
Branches Tags
IT4Change:master

2 Commits
725c370dd1 ... 61ed6185af

Author SHA1 Message Date
Ulf Gebhardt
61ed6185af
alert manager, secrets per environment 2025-07-16 20:59:42 +02:00
Ulf Gebhardt
eef1df8596
sops file 2025-07-16 20:58:55 +02:00
8 changed files with 236 additions and 3 deletions
Show Stats Expand all files Collapse all files

17
.sops.yaml Normal file
View File

@ -0,0 +1,17 @@
creation_rules:
- age: >-
age1al36hkk8can83zpxq8qyy07gpv83hdw9vchfly5f264kanz405as283a00,
age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw,
age1zycwtk6dkxj6vuqhj9jw7932ythky9p3att6df4z9qasyw8v5dxquejcmp,
age15arcg8x6ltnsacwalvny0h2d4d4wkdmax328mw3v5vda9zm97uqshtavmr,
age1khw2eps099audp3uu5s9rk07qznllh5c8a43gv5dtpnq2a7lue6qrehn5s,
age1f6mzqe0cejajzt0c7nwdjz4xvs4hjct9d8hrgj60e7unzyfd7prsn0npe5,
age1t0ufylv5xfwhmcamu4gpwtay4wcuyqgzlkht4t04s9qjl8xjks9skxrt02
# age1al36hkk8can83zpxq8qyy07gpv83hdw9vchfly5f264kanz405as283a00 SOPS_KEY github secret
# age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw @roschaefer
# age1zycwtk6dkxj6vuqhj9jw7932ythky9p3att6df4z9qasyw8v5dxquejcmp @mahula
# age15arcg8x6ltnsacwalvny0h2d4d4wkdmax328mw3v5vda9zm97uqshtavmr @Elweyn
# age1khw2eps099audp3uu5s9rk07qznllh5c8a43gv5dtpnq2a7lue6qrehn5s @ulfgebhardt
# age1f6mzqe0cejajzt0c7nwdjz4xvs4hjct9d8hrgj60e7unzyfd7prsn0npe5 @Tirokk
# age1t0ufylv5xfwhmcamu4gpwtay4wcuyqgzlkht4t04s9qjl8xjks9skxrt02 @Bettelstab

74
environments/default.secrets.yaml Normal file
View File

@ -0,0 +1,74 @@
discord:
WEBHOOK_URL: ENC[AES256_GCM,data:A0qxSbqbLKdYt1WM0xquidn8yKrkFBWF+ktsuTyyWy3HLcxxB9UEUIeRHrRS5NdJY0fjk9UoB/XIuxwAzwDK0Z5TSOefxNYhZSz2OH24Lqg8Wzk2q2gE6fqwsWlF0RGwJl3IhjgOdltP2Co7mHMaamnB6gQvPHcW1Q==,iv:DOvbvEgjJnw5KUdW1+7J4EKsdWzcWz/Vf1RVMG3T9rg=,tag:SwO1402li3RaHJrZt/gy4w==,type:str]
USERNAME: ENC[AES256_GCM,data:8OplmPnNPIk1pKzzRHlCte997YXDOxq+EqM=,iv:WEtU82UE3ur+uZt1YfAd0zCRiEvDoR6vxOY1VT2emng=,tag:ZoPZvUV7ZocjbfqzSiQs0g==,type:str]
grafana:
ADMIN_PASSWORD: ENC[AES256_GCM,data:oBA4kA==,iv:u3pSJCfnG8DwuXOeY5y0x2JGR6gyZnXf0/tqgWT0EQI=,tag:BtJMEr3jfaXVKkDf3AHqvg==,type:str]
sops:
age:
- recipient: age1al36hkk8can83zpxq8qyy07gpv83hdw9vchfly5f264kanz405as283a00
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLd1BvUUVRbFZQemNtcFZ6
dUliNmpIUDcvL2F1cENvWldsUE9FWVFxZ21rCm9GWkxKZ05qVjhMNy9ueW43d1Mz
TTI2RzFsR1B3RlFWVitwcUpqRTdEQjQKLS0tIENZeEJCSlJMcHVMaXB1dFB3YmhL
enVVbGVWcmJoM1hJNTlzSlhpaS8rUWsK9Y1sjUnFjB3s2wHVvMU3bVC1LIYvrz8t
n/QaIHUIEf0NB/ZPj6r6hplCnf+EJVKuVl5pu4xw2ED9PvXQ6UUZvQ==
-----END AGE ENCRYPTED FILE-----
- recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAydThhbUlBTGFIOElBUStr
WHdMNzBSbnlyYlFyVHhMbGJUSmozUjRINUhFCkNFbVBzTTl1cmVSRlRFL29VUFF0
Qy9sQk8yc0Q1aGljMk1Ob1NFVkZQd2sKLS0tIGpidFhscFAwc2pVRWxtVFY1OFo3
bzljNTc1MDQ4ckNQNzFjNDFGeVV5TzQKdIqZMcxhtjmPD8nsIHi8XbcZHcefo32l
AXXquc/+5+OBocUvAMZ9UWOdx8QCQAmaZ5YtXEePp+FFZKBcnPCRMQ==
-----END AGE ENCRYPTED FILE-----
- recipient: age1zycwtk6dkxj6vuqhj9jw7932ythky9p3att6df4z9qasyw8v5dxquejcmp
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3clQ3NVM4eEpJTzgxVTR1
cm9vMm1qTGkvWElpckxvOXBRMzMrUlNLaVhZCjJvRElJa1ptU2szZXZjUEZ0RXd5
cndZWXI2RHhuYzRnOFBLV0lZelQzKzAKLS0tIGpnVzdqWEV5RlV0UVdLUTVneklT
SEw3RkdrN0xOWndLb01nd1ovR01JZ1EKCvlakyb1WQeDaeDHHdrQEzO9fIynZsjk
ci8ccnOuZYjCHOc6U4enjlD559IZdniOPA72qdEFgquCtMwDi72buA==
-----END AGE ENCRYPTED FILE-----
- recipient: age15arcg8x6ltnsacwalvny0h2d4d4wkdmax328mw3v5vda9zm97uqshtavmr
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhcys4T2J1MkRHNHN2b2hB
akt4OEdYclBHaC9WNVdUdVhhalFaRzdDL1JVCkZDcElHclowaXFIRHJhaHluVW9j
d0VoVUZMcWlQclBrUXlRb3R3UzdpVzQKLS0tIEdyZ0dTc0lKOGJDTlNBUnZlcnp6
Z1dZeWRsUkVpMzF4RWtMd0pqV3g5RHcKdmPPkfoMaHwmdfVm+vnaWpuzgEK4NREx
NSt4JDmqxDV0j4iQMzMyULgHdeyvxnXpHiyNh4FnKzZljh8J1O8/yw==
-----END AGE ENCRYPTED FILE-----
- recipient: age1khw2eps099audp3uu5s9rk07qznllh5c8a43gv5dtpnq2a7lue6qrehn5s
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYL3lnR2dZMmVpS3lMa1kz
b0lIeVVsUzUwSWszNzBVdWpCak5Rb0lKcFY4CnN0ckFjcDZtRDZsMkcxRWMvOHo4
d01ySkJRemEzQ3dGK2NBU3pIZ0ROU0EKLS0tIFIwaVlhc2h0ZThwclBBMWNTc2dF
emdXSnhBV1VMbXp6ai9MaTBSZkNzYUUKkvZSOuYITTnDdm8RLk6h4inF3AqpfjX6
TByKxFuoRWQNu0mB1RNniwwYegfY/hIoXQ8hFEBaYLqapqadz+X+Kg==
-----END AGE ENCRYPTED FILE-----
- recipient: age1f6mzqe0cejajzt0c7nwdjz4xvs4hjct9d8hrgj60e7unzyfd7prsn0npe5
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWK1lPTE9ac01kazdEVHd1
c25FWFVZVDhkeUYyeXdqeGFabEZtY0haeGhBCnpRQ2wwTG96cmlTZXl3WHc2UytL
YzVYdEZ1U2EzVXltZ2FibERnRWM3Yk0KLS0tIHVpaDVIM1N5M2hMNHY0anNmK0c0
cnp5ZU1lMzJrRlNFQ2VLSmxGUElOMjQKrbR6dL1UwkRTwdHFrq6HAvt4R8SsAbqE
V3tS9utgx5PEDQkVC/7ueuXFyeQyJFya7lvZREvJOLRTRDl6PbC/Ew==
-----END AGE ENCRYPTED FILE-----
- recipient: age1t0ufylv5xfwhmcamu4gpwtay4wcuyqgzlkht4t04s9qjl8xjks9skxrt02
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWdDFhVU16KzhwMmdpUHRo
TWNTaWdlN1FWYzhFb00zWGpON29JTEhuRDE0CmxmdkQ4ZkYrWnJIblBDK3dIVUN5
K2pKNmRkWnB4OVNreVJOV3JCUjNPY0UKLS0tIGVBaUN3VTZWOUkrcFZNTVV4S0RH
TTVLamdEaEZOYk55cldCVzBuWm1UTEEKjrVRYcy6P3JyPlgSrAxm127TqQzfi7mj
McQxS+qNleBjIvfWDhb8I7dsVt/3CSfZ+HHVZ3APhHLAT+av+pyi3w==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2025-07-16T18:39:47Z"
mac: ENC[AES256_GCM,data:WbsGhvKwgIYzYhKM/sPkLESi0c79oYDzfgDzxt3jYHjhCIBhrX/gmwBXIwCOopiUrCQcecFi15Fj+F2w7v4wq6KMrHpqga+7DMfMzz2dWO0ZKxcomgvGkRREfcm6y2lFUZHOjMAF9msOiUkkW5dRtxkcTZlNk3+JPGwivI90RZA=,iv:J0IehP59T5P75lENjIq8fiFsnuGAZgAtyrQXw+ybmjA=,tag:WdiouOZNJqSmIVMl8CPUcA==,type:str]
unencrypted_suffix: _unencrypted
version: 3.10.2

0
environments/default.yaml.gotmpl Normal file
View File

74
environments/production.secrets.yaml Normal file
View File

@ -0,0 +1,74 @@
discord:
WEBHOOK_URL: ENC[AES256_GCM,data:A0qxSbqbLKdYt1WM0xquidn8yKrkFBWF+ktsuTyyWy3HLcxxB9UEUIeRHrRS5NdJY0fjk9UoB/XIuxwAzwDK0Z5TSOefxNYhZSz2OH24Lqg8Wzk2q2gE6fqwsWlF0RGwJl3IhjgOdltP2Co7mHMaamnB6gQvPHcW1Q==,iv:DOvbvEgjJnw5KUdW1+7J4EKsdWzcWz/Vf1RVMG3T9rg=,tag:SwO1402li3RaHJrZt/gy4w==,type:str]
USERNAME: ENC[AES256_GCM,data:izME9gTtPKLWvrG2E4SpzQpbd1UlKnNdiNELjpE=,iv:QhHD/UsCl+E97qJPcKUIXR4SLCuplJ42P9WW2flc3+8=,tag:g9g0So9eGV6wOHNmoW1X7g==,type:str]
grafana:
ADMIN_PASSWORD: ENC[AES256_GCM,data:oBA4kA==,iv:u3pSJCfnG8DwuXOeY5y0x2JGR6gyZnXf0/tqgWT0EQI=,tag:BtJMEr3jfaXVKkDf3AHqvg==,type:str]
sops:
age:
- recipient: age1al36hkk8can83zpxq8qyy07gpv83hdw9vchfly5f264kanz405as283a00
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLd1BvUUVRbFZQemNtcFZ6
dUliNmpIUDcvL2F1cENvWldsUE9FWVFxZ21rCm9GWkxKZ05qVjhMNy9ueW43d1Mz
TTI2RzFsR1B3RlFWVitwcUpqRTdEQjQKLS0tIENZeEJCSlJMcHVMaXB1dFB3YmhL
enVVbGVWcmJoM1hJNTlzSlhpaS8rUWsK9Y1sjUnFjB3s2wHVvMU3bVC1LIYvrz8t
n/QaIHUIEf0NB/ZPj6r6hplCnf+EJVKuVl5pu4xw2ED9PvXQ6UUZvQ==
-----END AGE ENCRYPTED FILE-----
- recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAydThhbUlBTGFIOElBUStr
WHdMNzBSbnlyYlFyVHhMbGJUSmozUjRINUhFCkNFbVBzTTl1cmVSRlRFL29VUFF0
Qy9sQk8yc0Q1aGljMk1Ob1NFVkZQd2sKLS0tIGpidFhscFAwc2pVRWxtVFY1OFo3
bzljNTc1MDQ4ckNQNzFjNDFGeVV5TzQKdIqZMcxhtjmPD8nsIHi8XbcZHcefo32l
AXXquc/+5+OBocUvAMZ9UWOdx8QCQAmaZ5YtXEePp+FFZKBcnPCRMQ==
-----END AGE ENCRYPTED FILE-----
- recipient: age1zycwtk6dkxj6vuqhj9jw7932ythky9p3att6df4z9qasyw8v5dxquejcmp
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3clQ3NVM4eEpJTzgxVTR1
cm9vMm1qTGkvWElpckxvOXBRMzMrUlNLaVhZCjJvRElJa1ptU2szZXZjUEZ0RXd5
cndZWXI2RHhuYzRnOFBLV0lZelQzKzAKLS0tIGpnVzdqWEV5RlV0UVdLUTVneklT
SEw3RkdrN0xOWndLb01nd1ovR01JZ1EKCvlakyb1WQeDaeDHHdrQEzO9fIynZsjk
ci8ccnOuZYjCHOc6U4enjlD559IZdniOPA72qdEFgquCtMwDi72buA==
-----END AGE ENCRYPTED FILE-----
- recipient: age15arcg8x6ltnsacwalvny0h2d4d4wkdmax328mw3v5vda9zm97uqshtavmr
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhcys4T2J1MkRHNHN2b2hB
akt4OEdYclBHaC9WNVdUdVhhalFaRzdDL1JVCkZDcElHclowaXFIRHJhaHluVW9j
d0VoVUZMcWlQclBrUXlRb3R3UzdpVzQKLS0tIEdyZ0dTc0lKOGJDTlNBUnZlcnp6
Z1dZeWRsUkVpMzF4RWtMd0pqV3g5RHcKdmPPkfoMaHwmdfVm+vnaWpuzgEK4NREx
NSt4JDmqxDV0j4iQMzMyULgHdeyvxnXpHiyNh4FnKzZljh8J1O8/yw==
-----END AGE ENCRYPTED FILE-----
- recipient: age1khw2eps099audp3uu5s9rk07qznllh5c8a43gv5dtpnq2a7lue6qrehn5s
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYL3lnR2dZMmVpS3lMa1kz
b0lIeVVsUzUwSWszNzBVdWpCak5Rb0lKcFY4CnN0ckFjcDZtRDZsMkcxRWMvOHo4
d01ySkJRemEzQ3dGK2NBU3pIZ0ROU0EKLS0tIFIwaVlhc2h0ZThwclBBMWNTc2dF
emdXSnhBV1VMbXp6ai9MaTBSZkNzYUUKkvZSOuYITTnDdm8RLk6h4inF3AqpfjX6
TByKxFuoRWQNu0mB1RNniwwYegfY/hIoXQ8hFEBaYLqapqadz+X+Kg==
-----END AGE ENCRYPTED FILE-----
- recipient: age1f6mzqe0cejajzt0c7nwdjz4xvs4hjct9d8hrgj60e7unzyfd7prsn0npe5
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWK1lPTE9ac01kazdEVHd1
c25FWFVZVDhkeUYyeXdqeGFabEZtY0haeGhBCnpRQ2wwTG96cmlTZXl3WHc2UytL
YzVYdEZ1U2EzVXltZ2FibERnRWM3Yk0KLS0tIHVpaDVIM1N5M2hMNHY0anNmK0c0
cnp5ZU1lMzJrRlNFQ2VLSmxGUElOMjQKrbR6dL1UwkRTwdHFrq6HAvt4R8SsAbqE
V3tS9utgx5PEDQkVC/7ueuXFyeQyJFya7lvZREvJOLRTRDl6PbC/Ew==
-----END AGE ENCRYPTED FILE-----
- recipient: age1t0ufylv5xfwhmcamu4gpwtay4wcuyqgzlkht4t04s9qjl8xjks9skxrt02
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWdDFhVU16KzhwMmdpUHRo
TWNTaWdlN1FWYzhFb00zWGpON29JTEhuRDE0CmxmdkQ4ZkYrWnJIblBDK3dIVUN5
K2pKNmRkWnB4OVNreVJOV3JCUjNPY0UKLS0tIGVBaUN3VTZWOUkrcFZNTVV4S0RH
TTVLamdEaEZOYk55cldCVzBuWm1UTEEKjrVRYcy6P3JyPlgSrAxm127TqQzfi7mj
McQxS+qNleBjIvfWDhb8I7dsVt/3CSfZ+HHVZ3APhHLAT+av+pyi3w==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2025-07-16T18:40:40Z"
mac: ENC[AES256_GCM,data:b063J4BNm2C+fbpgLkOy/8kx7kDqajlZqOc0wdTFIAIRTUZVblYqJ6FScnTvkO4Gew82fYXtFgocKoRDJmZ2XrxpfxHdvThRzfsVtjH3E4sxq6onhSjRly3DxMBcshxaREN+roBi0U0h4vMdOwk5ZHtyYvMY5A3lSnDVghyvPfU=,iv:FilwMMqWpT6irXliYsnt8bO74D1uXpzLX9+e/WkI4WA=,tag:7TEtIEzA1S3sGvta74YtNA==,type:str]
unencrypted_suffix: _unencrypted
version: 3.10.2

0
environments/production.yaml.gotmpl Normal file
View File

10
helmfile.yaml
View File

@ -1,7 +1,15 @@
---
environments:
default:
values:
- ./environments/default.yaml.gotmpl
secrets:
- ./environments/default.secrets.yaml
production:
values:
- ./environments/production.yaml.gotmpl
secrets:
- ./environments/production.secrets.yaml
---
repositories:
- name: prometheus-community
@ -12,4 +20,4 @@ releases:
namespace: monitoring
chart: prometheus-community/kube-prometheus-stack
values:
- ./values/values.yaml
- ./values/prometheus.yaml.gotmpl

62
values/prometheus.yaml.gotmpl Normal file
View File

@ -0,0 +1,62 @@
grafana:
adminPassword: {{ .StateValues.grafana.ADMIN_PASSWORD }}
alertmanager:
config:
global:
resolve_timeout: 1m
route:
receiver: 'discord'
group_by: ['alertname', 'cluster', 'job', 'env']
group_wait: 15s
group_interval: 15s
repeat_interval: 1m
routes:
- match: # port not exposed?
alertname: KubeProxyDown
receiver: "null"
- match: # port not exposed?
alertname: KubeSchedulerDown
receiver: "null"
- match: # port not exposed?
alertname: KubeControllerManagerDown
receiver: "null"
- match: # dummy alert
alertname: Watchdog
receiver: "null"
# # discord has delivery problems sometimes (amount of messages)
# # better mute it temporarily via silence in graphana.
#- match:
# alertname: AlertmanagerFailedToSendAlerts
# receiver: "null"
receivers:
- name: 'null'
- name: 'discord'
discord_configs:
- webhook_url: {{ .StateValues.discord.WEBHOOK_URL }}
username: {{ .StateValues.discord.USERNAME }}
additionalPrometheusRulesMap:
high-cpu-usage:
groups:
- name: Node
rules:
- alert: HighCPUUsage
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
for: 10s
labels:
severity: warning
annotations:
summary: "High CPU usage detected"
description: 'CPU usage is above 80% on {{ "{{" }} $labels.instance {{ "}}" }}'
node-not-ready:
groups:
- name: Node
rules:
- alert: NodeNotReady
expr: sum(kube_node_status_condition{condition="Ready",status!="true"}) > 0
labels:
severity: critical
annotations:
summary: "Node not ready"
description: 'Node {{ "{{" }} $labels.instance {{ "}}" }} is not ready'

2
values/values.yaml
View File

@ -1,2 +0,0 @@
grafana:
adminPassword: it4c