diff --git a/environments/default.secrets.yaml b/environments/default.secrets.yaml new file mode 100644 index 0000000..663eb80 --- /dev/null +++ b/environments/default.secrets.yaml @@ -0,0 +1,74 @@ +discord: + WEBHOOK_URL: ENC[AES256_GCM,data:A0qxSbqbLKdYt1WM0xquidn8yKrkFBWF+ktsuTyyWy3HLcxxB9UEUIeRHrRS5NdJY0fjk9UoB/XIuxwAzwDK0Z5TSOefxNYhZSz2OH24Lqg8Wzk2q2gE6fqwsWlF0RGwJl3IhjgOdltP2Co7mHMaamnB6gQvPHcW1Q==,iv:DOvbvEgjJnw5KUdW1+7J4EKsdWzcWz/Vf1RVMG3T9rg=,tag:SwO1402li3RaHJrZt/gy4w==,type:str] + USERNAME: ENC[AES256_GCM,data:8OplmPnNPIk1pKzzRHlCte997YXDOxq+EqM=,iv:WEtU82UE3ur+uZt1YfAd0zCRiEvDoR6vxOY1VT2emng=,tag:ZoPZvUV7ZocjbfqzSiQs0g==,type:str] +grafana: + ADMIN_PASSWORD: ENC[AES256_GCM,data:oBA4kA==,iv:u3pSJCfnG8DwuXOeY5y0x2JGR6gyZnXf0/tqgWT0EQI=,tag:BtJMEr3jfaXVKkDf3AHqvg==,type:str] +sops: + age: + - recipient: age1al36hkk8can83zpxq8qyy07gpv83hdw9vchfly5f264kanz405as283a00 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLd1BvUUVRbFZQemNtcFZ6 + dUliNmpIUDcvL2F1cENvWldsUE9FWVFxZ21rCm9GWkxKZ05qVjhMNy9ueW43d1Mz + TTI2RzFsR1B3RlFWVitwcUpqRTdEQjQKLS0tIENZeEJCSlJMcHVMaXB1dFB3YmhL + enVVbGVWcmJoM1hJNTlzSlhpaS8rUWsK9Y1sjUnFjB3s2wHVvMU3bVC1LIYvrz8t + n/QaIHUIEf0NB/ZPj6r6hplCnf+EJVKuVl5pu4xw2ED9PvXQ6UUZvQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAydThhbUlBTGFIOElBUStr + WHdMNzBSbnlyYlFyVHhMbGJUSmozUjRINUhFCkNFbVBzTTl1cmVSRlRFL29VUFF0 + Qy9sQk8yc0Q1aGljMk1Ob1NFVkZQd2sKLS0tIGpidFhscFAwc2pVRWxtVFY1OFo3 + bzljNTc1MDQ4ckNQNzFjNDFGeVV5TzQKdIqZMcxhtjmPD8nsIHi8XbcZHcefo32l + AXXquc/+5+OBocUvAMZ9UWOdx8QCQAmaZ5YtXEePp+FFZKBcnPCRMQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1zycwtk6dkxj6vuqhj9jw7932ythky9p3att6df4z9qasyw8v5dxquejcmp + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3clQ3NVM4eEpJTzgxVTR1 + cm9vMm1qTGkvWElpckxvOXBRMzMrUlNLaVhZCjJvRElJa1ptU2szZXZjUEZ0RXd5 + cndZWXI2RHhuYzRnOFBLV0lZelQzKzAKLS0tIGpnVzdqWEV5RlV0UVdLUTVneklT + SEw3RkdrN0xOWndLb01nd1ovR01JZ1EKCvlakyb1WQeDaeDHHdrQEzO9fIynZsjk + ci8ccnOuZYjCHOc6U4enjlD559IZdniOPA72qdEFgquCtMwDi72buA== + -----END AGE ENCRYPTED FILE----- + - recipient: age15arcg8x6ltnsacwalvny0h2d4d4wkdmax328mw3v5vda9zm97uqshtavmr + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhcys4T2J1MkRHNHN2b2hB + akt4OEdYclBHaC9WNVdUdVhhalFaRzdDL1JVCkZDcElHclowaXFIRHJhaHluVW9j + d0VoVUZMcWlQclBrUXlRb3R3UzdpVzQKLS0tIEdyZ0dTc0lKOGJDTlNBUnZlcnp6 + Z1dZeWRsUkVpMzF4RWtMd0pqV3g5RHcKdmPPkfoMaHwmdfVm+vnaWpuzgEK4NREx + NSt4JDmqxDV0j4iQMzMyULgHdeyvxnXpHiyNh4FnKzZljh8J1O8/yw== + -----END AGE ENCRYPTED FILE----- + - recipient: age1khw2eps099audp3uu5s9rk07qznllh5c8a43gv5dtpnq2a7lue6qrehn5s + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYL3lnR2dZMmVpS3lMa1kz + b0lIeVVsUzUwSWszNzBVdWpCak5Rb0lKcFY4CnN0ckFjcDZtRDZsMkcxRWMvOHo4 + d01ySkJRemEzQ3dGK2NBU3pIZ0ROU0EKLS0tIFIwaVlhc2h0ZThwclBBMWNTc2dF + emdXSnhBV1VMbXp6ai9MaTBSZkNzYUUKkvZSOuYITTnDdm8RLk6h4inF3AqpfjX6 + TByKxFuoRWQNu0mB1RNniwwYegfY/hIoXQ8hFEBaYLqapqadz+X+Kg== + -----END AGE ENCRYPTED FILE----- + - recipient: age1f6mzqe0cejajzt0c7nwdjz4xvs4hjct9d8hrgj60e7unzyfd7prsn0npe5 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWK1lPTE9ac01kazdEVHd1 + c25FWFVZVDhkeUYyeXdqeGFabEZtY0haeGhBCnpRQ2wwTG96cmlTZXl3WHc2UytL + YzVYdEZ1U2EzVXltZ2FibERnRWM3Yk0KLS0tIHVpaDVIM1N5M2hMNHY0anNmK0c0 + cnp5ZU1lMzJrRlNFQ2VLSmxGUElOMjQKrbR6dL1UwkRTwdHFrq6HAvt4R8SsAbqE + V3tS9utgx5PEDQkVC/7ueuXFyeQyJFya7lvZREvJOLRTRDl6PbC/Ew== + -----END AGE ENCRYPTED FILE----- + - recipient: age1t0ufylv5xfwhmcamu4gpwtay4wcuyqgzlkht4t04s9qjl8xjks9skxrt02 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWdDFhVU16KzhwMmdpUHRo + TWNTaWdlN1FWYzhFb00zWGpON29JTEhuRDE0CmxmdkQ4ZkYrWnJIblBDK3dIVUN5 + K2pKNmRkWnB4OVNreVJOV3JCUjNPY0UKLS0tIGVBaUN3VTZWOUkrcFZNTVV4S0RH + TTVLamdEaEZOYk55cldCVzBuWm1UTEEKjrVRYcy6P3JyPlgSrAxm127TqQzfi7mj + McQxS+qNleBjIvfWDhb8I7dsVt/3CSfZ+HHVZ3APhHLAT+av+pyi3w== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2025-07-16T18:39:47Z" + mac: ENC[AES256_GCM,data:WbsGhvKwgIYzYhKM/sPkLESi0c79oYDzfgDzxt3jYHjhCIBhrX/gmwBXIwCOopiUrCQcecFi15Fj+F2w7v4wq6KMrHpqga+7DMfMzz2dWO0ZKxcomgvGkRREfcm6y2lFUZHOjMAF9msOiUkkW5dRtxkcTZlNk3+JPGwivI90RZA=,iv:J0IehP59T5P75lENjIq8fiFsnuGAZgAtyrQXw+ybmjA=,tag:WdiouOZNJqSmIVMl8CPUcA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.10.2 diff --git a/environments/default.yaml.gotmpl b/environments/default.yaml.gotmpl new file mode 100644 index 0000000..e69de29 diff --git a/environments/production.secrets.yaml b/environments/production.secrets.yaml new file mode 100644 index 0000000..ef8e27e --- /dev/null +++ b/environments/production.secrets.yaml @@ -0,0 +1,74 @@ +discord: + WEBHOOK_URL: ENC[AES256_GCM,data:A0qxSbqbLKdYt1WM0xquidn8yKrkFBWF+ktsuTyyWy3HLcxxB9UEUIeRHrRS5NdJY0fjk9UoB/XIuxwAzwDK0Z5TSOefxNYhZSz2OH24Lqg8Wzk2q2gE6fqwsWlF0RGwJl3IhjgOdltP2Co7mHMaamnB6gQvPHcW1Q==,iv:DOvbvEgjJnw5KUdW1+7J4EKsdWzcWz/Vf1RVMG3T9rg=,tag:SwO1402li3RaHJrZt/gy4w==,type:str] + USERNAME: ENC[AES256_GCM,data:izME9gTtPKLWvrG2E4SpzQpbd1UlKnNdiNELjpE=,iv:QhHD/UsCl+E97qJPcKUIXR4SLCuplJ42P9WW2flc3+8=,tag:g9g0So9eGV6wOHNmoW1X7g==,type:str] +grafana: + ADMIN_PASSWORD: ENC[AES256_GCM,data:oBA4kA==,iv:u3pSJCfnG8DwuXOeY5y0x2JGR6gyZnXf0/tqgWT0EQI=,tag:BtJMEr3jfaXVKkDf3AHqvg==,type:str] +sops: + age: + - recipient: age1al36hkk8can83zpxq8qyy07gpv83hdw9vchfly5f264kanz405as283a00 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLd1BvUUVRbFZQemNtcFZ6 + dUliNmpIUDcvL2F1cENvWldsUE9FWVFxZ21rCm9GWkxKZ05qVjhMNy9ueW43d1Mz + TTI2RzFsR1B3RlFWVitwcUpqRTdEQjQKLS0tIENZeEJCSlJMcHVMaXB1dFB3YmhL + enVVbGVWcmJoM1hJNTlzSlhpaS8rUWsK9Y1sjUnFjB3s2wHVvMU3bVC1LIYvrz8t + n/QaIHUIEf0NB/ZPj6r6hplCnf+EJVKuVl5pu4xw2ED9PvXQ6UUZvQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1llp6k66265q3rzqemxpnq0x3562u20989vcjf65fl9s3hjhgcscq6mhnjw + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAydThhbUlBTGFIOElBUStr + WHdMNzBSbnlyYlFyVHhMbGJUSmozUjRINUhFCkNFbVBzTTl1cmVSRlRFL29VUFF0 + Qy9sQk8yc0Q1aGljMk1Ob1NFVkZQd2sKLS0tIGpidFhscFAwc2pVRWxtVFY1OFo3 + bzljNTc1MDQ4ckNQNzFjNDFGeVV5TzQKdIqZMcxhtjmPD8nsIHi8XbcZHcefo32l + AXXquc/+5+OBocUvAMZ9UWOdx8QCQAmaZ5YtXEePp+FFZKBcnPCRMQ== + -----END AGE ENCRYPTED FILE----- + - recipient: age1zycwtk6dkxj6vuqhj9jw7932ythky9p3att6df4z9qasyw8v5dxquejcmp + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3clQ3NVM4eEpJTzgxVTR1 + cm9vMm1qTGkvWElpckxvOXBRMzMrUlNLaVhZCjJvRElJa1ptU2szZXZjUEZ0RXd5 + cndZWXI2RHhuYzRnOFBLV0lZelQzKzAKLS0tIGpnVzdqWEV5RlV0UVdLUTVneklT + SEw3RkdrN0xOWndLb01nd1ovR01JZ1EKCvlakyb1WQeDaeDHHdrQEzO9fIynZsjk + ci8ccnOuZYjCHOc6U4enjlD559IZdniOPA72qdEFgquCtMwDi72buA== + -----END AGE ENCRYPTED FILE----- + - recipient: age15arcg8x6ltnsacwalvny0h2d4d4wkdmax328mw3v5vda9zm97uqshtavmr + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhcys4T2J1MkRHNHN2b2hB + akt4OEdYclBHaC9WNVdUdVhhalFaRzdDL1JVCkZDcElHclowaXFIRHJhaHluVW9j + d0VoVUZMcWlQclBrUXlRb3R3UzdpVzQKLS0tIEdyZ0dTc0lKOGJDTlNBUnZlcnp6 + Z1dZeWRsUkVpMzF4RWtMd0pqV3g5RHcKdmPPkfoMaHwmdfVm+vnaWpuzgEK4NREx + NSt4JDmqxDV0j4iQMzMyULgHdeyvxnXpHiyNh4FnKzZljh8J1O8/yw== + -----END AGE ENCRYPTED FILE----- + - recipient: age1khw2eps099audp3uu5s9rk07qznllh5c8a43gv5dtpnq2a7lue6qrehn5s + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYL3lnR2dZMmVpS3lMa1kz + b0lIeVVsUzUwSWszNzBVdWpCak5Rb0lKcFY4CnN0ckFjcDZtRDZsMkcxRWMvOHo4 + d01ySkJRemEzQ3dGK2NBU3pIZ0ROU0EKLS0tIFIwaVlhc2h0ZThwclBBMWNTc2dF + emdXSnhBV1VMbXp6ai9MaTBSZkNzYUUKkvZSOuYITTnDdm8RLk6h4inF3AqpfjX6 + TByKxFuoRWQNu0mB1RNniwwYegfY/hIoXQ8hFEBaYLqapqadz+X+Kg== + -----END AGE ENCRYPTED FILE----- + - recipient: age1f6mzqe0cejajzt0c7nwdjz4xvs4hjct9d8hrgj60e7unzyfd7prsn0npe5 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWK1lPTE9ac01kazdEVHd1 + c25FWFVZVDhkeUYyeXdqeGFabEZtY0haeGhBCnpRQ2wwTG96cmlTZXl3WHc2UytL + YzVYdEZ1U2EzVXltZ2FibERnRWM3Yk0KLS0tIHVpaDVIM1N5M2hMNHY0anNmK0c0 + cnp5ZU1lMzJrRlNFQ2VLSmxGUElOMjQKrbR6dL1UwkRTwdHFrq6HAvt4R8SsAbqE + V3tS9utgx5PEDQkVC/7ueuXFyeQyJFya7lvZREvJOLRTRDl6PbC/Ew== + -----END AGE ENCRYPTED FILE----- + - recipient: age1t0ufylv5xfwhmcamu4gpwtay4wcuyqgzlkht4t04s9qjl8xjks9skxrt02 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBWdDFhVU16KzhwMmdpUHRo + TWNTaWdlN1FWYzhFb00zWGpON29JTEhuRDE0CmxmdkQ4ZkYrWnJIblBDK3dIVUN5 + K2pKNmRkWnB4OVNreVJOV3JCUjNPY0UKLS0tIGVBaUN3VTZWOUkrcFZNTVV4S0RH + TTVLamdEaEZOYk55cldCVzBuWm1UTEEKjrVRYcy6P3JyPlgSrAxm127TqQzfi7mj + McQxS+qNleBjIvfWDhb8I7dsVt/3CSfZ+HHVZ3APhHLAT+av+pyi3w== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2025-07-16T18:40:40Z" + mac: ENC[AES256_GCM,data:b063J4BNm2C+fbpgLkOy/8kx7kDqajlZqOc0wdTFIAIRTUZVblYqJ6FScnTvkO4Gew82fYXtFgocKoRDJmZ2XrxpfxHdvThRzfsVtjH3E4sxq6onhSjRly3DxMBcshxaREN+roBi0U0h4vMdOwk5ZHtyYvMY5A3lSnDVghyvPfU=,iv:FilwMMqWpT6irXliYsnt8bO74D1uXpzLX9+e/WkI4WA=,tag:7TEtIEzA1S3sGvta74YtNA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.10.2 diff --git a/environments/production.yaml.gotmpl b/environments/production.yaml.gotmpl new file mode 100644 index 0000000..e69de29 diff --git a/helmfile.yaml b/helmfile.yaml index f4ef3e2..c51cc95 100644 --- a/helmfile.yaml +++ b/helmfile.yaml @@ -1,7 +1,15 @@ --- environments: default: + values: + - ./environments/default.yaml.gotmpl + secrets: + - ./environments/default.secrets.yaml production: + values: + - ./environments/production.yaml.gotmpl + secrets: + - ./environments/production.secrets.yaml --- repositories: - name: prometheus-community @@ -12,4 +20,4 @@ releases: namespace: monitoring chart: prometheus-community/kube-prometheus-stack values: - - ./values/values.yaml \ No newline at end of file + - ./values/prometheus.yaml.gotmpl \ No newline at end of file diff --git a/values/prometheus.yaml.gotmpl b/values/prometheus.yaml.gotmpl new file mode 100644 index 0000000..5fb5912 --- /dev/null +++ b/values/prometheus.yaml.gotmpl @@ -0,0 +1,62 @@ +grafana: + adminPassword: {{ .StateValues.grafana.ADMIN_PASSWORD }} + +alertmanager: + config: + global: + resolve_timeout: 1m + route: + receiver: 'discord' + group_by: ['alertname', 'cluster', 'job', 'env'] + group_wait: 15s + group_interval: 15s + repeat_interval: 1m + routes: + - match: # port not exposed? + alertname: KubeProxyDown + receiver: "null" + - match: # port not exposed? + alertname: KubeSchedulerDown + receiver: "null" + - match: # port not exposed? + alertname: KubeControllerManagerDown + receiver: "null" + - match: # dummy alert + alertname: Watchdog + receiver: "null" + # # discord has delivery problems sometimes (amount of messages) + # # better mute it temporarily via silence in graphana. + #- match: + # alertname: AlertmanagerFailedToSendAlerts + # receiver: "null" + receivers: + - name: 'null' + - name: 'discord' + discord_configs: + - webhook_url: {{ .StateValues.discord.WEBHOOK_URL }} + username: {{ .StateValues.discord.USERNAME }} + +additionalPrometheusRulesMap: + high-cpu-usage: + groups: + - name: Node + rules: + - alert: HighCPUUsage + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 10s + labels: + severity: warning + annotations: + summary: "High CPU usage detected" + description: 'CPU usage is above 80% on {{ "{{" }} $labels.instance {{ "}}" }}' + node-not-ready: + groups: + - name: Node + rules: + - alert: NodeNotReady + expr: sum(kube_node_status_condition{condition="Ready",status!="true"}) > 0 + labels: + severity: critical + annotations: + summary: "Node not ready" + description: 'Node {{ "{{" }} $labels.instance {{ "}}" }} is not ready' \ No newline at end of file diff --git a/values/values.yaml b/values/values.yaml deleted file mode 100644 index 7dbd657..0000000 --- a/values/values.yaml +++ /dev/null @@ -1,2 +0,0 @@ -grafana: - adminPassword: it4c \ No newline at end of file