# Kubernetes alert rules — https://n9e.github.io/alert-rules/kubernetes
# Generated from the Nightingale integration templates: https://github.com/ccfos/nightingale/tree/main/integrations
# Many expressions are adapted from kubernetes-mixin (https://github.com/kubernetes-monitoring/kubernetes-mixin, Apache-2.0).
# Thresholds are starting points: review them before enabling.
groups:
  - name: "kubernetes-default"
    rules:
      - alert: "Deployment Pod availability below 70% (replicas above 9 pods)"
        expr: |-
          kube_deployment_status_replicas_unavailable and kube_deployment_status_replicas_available{} / kube_deployment_status_replicas{} * 100 < 70 and kube_deployment_status_replicas{} > 9
        for: 1m
        labels:
          severity: critical
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Job failed"
        expr: |-
          kube_job_status_failed{} > 0
        for: 1m
        labels:
          severity: info
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Pod startup timeout"
        expr: |-
          kube_pod_container_status_waiting_reason{} == 1
        for: 1m
        labels:
          severity: warning
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Pod state abnormal"
        expr: |-
          kube_pod_status_phase{phase=~"Pending|Unknown|Failed"} > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Pod restarting frequently"
        expr: |-
          increase(kube_pod_container_status_restarts_total{}[5m]) > 3
        for: 1m
        labels:
          severity: warning
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Container CPU usage exceeds 80%"
        expr: |-
          100 * (sum(rate(container_cpu_usage_seconds_total{id!="/"}[2m])) by (pod_name,namespace,container) / sum(label_replace(kube_pod_container_resource_limits_cpu_cores{},"pod_name","$1","pod","(.*)")) by (pod_name,namespace,container)) > 80 and sum(label_replace(kube_pod_container_resource_limits_cpu_cores{},"pod_name","$1","pod","(.)")) by (pod_name,namespace,container) != 0
        for: 1m
        labels:
          severity: info
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Container memory usage exceeds 80%"
        expr: |-
          (sum(container_memory_working_set_bytes{id!="/"}) BY (instance,name,container,pod_name,namespace) / sum(container_spec_memory_limit_bytes{id!="/"} > 0) BY (instance,name,container,pod_name,namespace) * 100) > 80
        for: 1m
        labels:
          severity: info
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Node CPU usage exceeds 90%"
        expr: |-
          100-cpu_usage_idle{cpu="cpu-total"} > 90
        for: 1m
        labels:
          severity: info
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Node memory usage exceeds 90%"
        expr: |-
          mem_used_percent > 90
        for: 1m
        labels:
          severity: warning
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Node not ready"
        expr: |-
          kube_node_status_condition{condition="Ready",status="true"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Node disk usage exceeds 90%"
        expr: |-
          disk_used_percent > 90
        for: 1m
        labels:
          severity: info
        annotations:
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeClientCertificateExpiration-S2"
        expr: |-
          apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
        labels:
          severity: warning
        annotations:
          summary: "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeclientcertificateexpiration"
      - alert: "KubeClientCertificateExpiration-S1"
        expr: |-
          apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
        labels:
          severity: critical
        annotations:
          summary: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeclientcertificateexpiration"
      - alert: "AggregatedAPIErrors"
        expr: |-
          sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2
        labels:
          severity: warning
        annotations:
          summary: "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "AggregatedAPIDown"
        expr: |-
          (1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeAPIDown"
        expr: |-
          absent(up{job="apiserver"} == 1)
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "KubeAPI has disappeared from Prometheus target discovery."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeAPIErrorBudgetBurn-S1-120s"
        expr: |-
          sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
          and
          sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "The API server is burning too much error budget."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeapierrorbudgetburn"
      - alert: "KubeAPIErrorBudgetBurn-S1-900s"
        expr: |-
          sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
          and
          sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "The API server is burning too much error budget."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeapierrorbudgetburn"
      - alert: "KubeAPIErrorBudgetBurn-S2-3600s"
        expr: |-
          sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
          and
          sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
        for: 60m
        labels:
          severity: warning
        annotations:
          summary: "The API server is burning too much error budget."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeapierrorbudgetburn"
      - alert: "KubeAPIErrorBudgetBurn-S2-10800s"
        expr: |-
          sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
          and
          sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
        for: 180m
        labels:
          severity: warning
        annotations:
          summary: "The API server is burning too much error budget."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeapierrorbudgetburn"
      - alert: "Deployment replicas mismatch"
        expr: |-
          kube_deployment_spec_replicas{job="kube-state-metrics"} != kube_deployment_status_replicas_available{job="kube-state-metrics"}
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Deployment {{ $labels.deployment }} in namespace {{ $labels.namespace }} has mismatched replicas."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "StatefulSet replicas mismatch"
        expr: |-
          kube_statefulset_status_replicas_ready{job="kube-state-metrics"} != kube_statefulset_spec_replicas{job="kube-state-metrics"}
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "StatefulSet {{ $labels.statefulset }} in namespace {{ $labels.namespace }} has mismatched replicas."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "DaemonSet has unready Pods"
        expr: |-
          kube_daemonset_status_number_ready{job="kube-state-metrics"} < kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "DaemonSet {{ $labels.daemonset }} in namespace {{ $labels.namespace }} has unready pods."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "HPA scaling failed"
        expr: |-
          kube_hpa_status_failed_scale_up_total{job="kube-state-metrics"} > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "HPA {{ $labels.hpa }} in namespace {{ $labels.namespace }} has scale up failures."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Pod in Failed state"
        expr: |-
          kube_pod_status_phase{job="kube-state-metrics",phase="Failed"} == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has failed."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Job failed"
        expr: |-
          kube_job_status_failed{job="kube-state-metrics"} > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Job {{ $labels.job_name }} in namespace {{ $labels.namespace }} has failed."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "CPUThrottlingHigh"
        expr: |-
          sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
            /
          sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
            > ( 25 / 100 )
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeAggregatedAPIDown"
        expr: |-
          (1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeaggregatedapidown"
      - alert: "KubeAggregatedAPIErrors"
        expr: |-
          sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeAPITerminatedRequests"
        expr: |-
          sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))  / (  sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeClientErrors"
        expr: |-
          (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by (cluster, instance, job, namespace)
            /
          sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by (cluster, instance, job, namespace))
          > 0.01
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'"
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeContainerWaiting"
        expr: |-
          sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0
        for: 60m
        labels:
          severity: warning
        annotations:
          summary: "pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubecontainerwaiting"
      - alert: "KubeControllerManagerDown"
        expr: |-
          absent(up{job="kube-controller-manager"} == 1)
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "KubeControllerManager has disappeared from Prometheus target discovery."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeCPUOvercommit"
        expr: |-
          sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="kube-state-metrics",}) by (cluster) - (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
          and
          (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubecpuovercommit"
      - alert: "KubeCPUQuotaOvercommit"
        expr: |-
          sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) by (cluster)
            /
          sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) by (cluster)
            > 1.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Cluster {{ $labels.cluster }}  has overcommitted CPU resource requests for Namespaces."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeDaemonSetMisScheduled"
        expr: |-
          kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubedaemonsetmisscheduled"
      - alert: "KubeDaemonSetNotScheduled"
        expr: |-
          kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
            -
          kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeDaemonSetRolloutStuck"
        expr: |-
          (
            (
              kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"}
                !=
              kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
            ) or (
              kube_daemonset_status_number_misscheduled{job="kube-state-metrics"}
                !=
              0
            ) or (
              kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}
                !=
              kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
            ) or (
              kube_daemonset_status_number_available{job="kube-state-metrics"}
                !=
              kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
            )
          ) and (
            changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics"}[5m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeDeploymentGenerationMismatch"
        expr: |-
          kube_deployment_status_observed_generation{job="kube-state-metrics"}
            !=
          kube_deployment_metadata_generation{job="kube-state-metrics"}
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeDeploymentReplicasMismatch"
        expr: |-
          (
            kube_deployment_spec_replicas{job="kube-state-metrics"}
              >
            kube_deployment_status_replicas_available{job="kube-state-metrics"}
          ) and (
            changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubedeploymentreplicasmismatch"
      - alert: "KubeDeploymentRolloutStuck"
        expr: |-
          kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics"}
          != 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubedeploymentrolloutstuck"
      - alert: "KubeHpaMaxedOut"
        expr: |-
          kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
            ==
          kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has been running at max replicas for longer than 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubehpamaxedout"
      - alert: "KubeHpaReplicasMismatch"
        expr: |-
          (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"}
            !=
          kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"})
            and
          (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
            >
          kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"})
            and
          (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
            <
          kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"})
            and
          changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has not matched the desired number of replicas for longer than 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeJobFailed"
        expr: |-
          kube_job_failed{job="kube-state-metrics"}  > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubejobfailed"
      - alert: "KubeJobNotCompleted"
        expr: |-
          time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics"}
            and
          kube_job_status_active{job="kube-state-metrics"} > 0) > 43200
        labels:
          severity: warning
        annotations:
          summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ \"43200\" | humanizeDuration }} to complete."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeletClientCertificateExpiration"
        expr: |-
          kubelet_certificate_manager_client_ttl_seconds < 604800
        labels:
          severity: warning
        annotations:
          summary: "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeletClientCertificateRenewalErrors"
        expr: |-
          increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes)."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeletDown"
        expr: |-
          absent(up{job="kubelet", metrics_path="/metrics"} == 1)
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Kubelet has disappeared from Prometheus target discovery."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubeletdown"
      - alert: "KubeletPlegDurationHigh"
        expr: |-
          node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeletPodStartUpLatencyHigh"
        expr: |-
          histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeletServerCertificateExpiration"
        expr: |-
          kubelet_certificate_manager_server_ttl_seconds < 604800
        labels:
          severity: warning
        annotations:
          summary: "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeletServerCertificateRenewalErrors"
        expr: |-
          increase(kubelet_server_expiration_renew_errors[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes)."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeletTooManyPods"
        expr: |-
          count by(cluster, node) (
            (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
          )
          /
          max by(cluster, node) (
            kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
          ) > 0.95
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubelettoomanypods"
      - alert: "KubeMemoryOvercommit"
        expr: |-
          sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
          and
          (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubememoryovercommit"
      - alert: "KubeMemoryQuotaOvercommit"
        expr: |-
          sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) by (cluster)
            /
          sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)
            > 1.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Cluster {{ $labels.cluster }}  has overcommitted memory resource requests for Namespaces."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeNodeNotReady"
        expr: |-
          kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.node }} has been unready for more than 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubenodenotready"
      - alert: "KubeNodeReadinessFlapping"
        expr: |-
          sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready"}[15m])) by (cluster, node) > 2
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeNodeUnreachable"
        expr: |-
          (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.node }} is unreachable and some workloads may be rescheduled."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubePersistentVolumeErrors"
        expr: |-
          kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubePersistentVolumeFillingUp"
        expr: |-
          (
            kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
              /
            kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
          ) < 0.03
          and
          kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0
          unless on(namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
          unless on(namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubepersistentvolumefillingup"
      - alert: "KubePersistentVolumeInodesFillingUp"
        expr: |-
          (
            kubelet_volume_stats_inodes_free{job="kubelet", metrics_path="/metrics"}
              /
            kubelet_volume_stats_inodes{job="kubelet", metrics_path="/metrics"}
          ) < 0.03
          and
          kubelet_volume_stats_inodes_used{job="kubelet", metrics_path="/metrics"} > 0
          unless on(namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
          unless on(namespace, persistentvolumeclaim)
          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubePodCrashLooping"
        expr: |-
          max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics"}[5m]) >= 1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: \"CrashLoopBackOff\")."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubepodcrashlooping"
      - alert: "KubePodNotReady"
        expr: |-
          sum by (namespace, pod, cluster) (
            max by(namespace, pod, cluster) (
              kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown|Failed"}
            ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) (
              1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"})
            )
          ) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubepodnotready"
      - alert: "KubeQuotaAlmostFull"
        expr: |-
          kube_resourcequota{job="kube-state-metrics", type="used"}
            / ignoring(instance, job, type)
          (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
            > 0.9 < 1
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubequotaalmostfull"
      - alert: "KubeQuotaExceeded"
        expr: |-
          kube_resourcequota{job="kube-state-metrics", type="used"}
            / ignoring(instance, job, type)
          (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
            > 1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeQuotaFullyUsed"
        expr: |-
          kube_resourcequota{job="kube-state-metrics", type="used"}
            / ignoring(instance, job, type)
          (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
            == 1
        for: 15m
        labels:
          severity: info
        annotations:
          summary: "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeSchedulerDown"
        expr: |-
          absent(up{job="kube-scheduler"} == 1)
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "KubeScheduler has disappeared from Prometheus target discovery."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeStatefulSetGenerationMismatch"
        expr: |-
          kube_statefulset_status_observed_generation{job="kube-state-metrics"}
            !=
          kube_statefulset_metadata_generation{job="kube-state-metrics"}
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeStatefulSetReplicasMismatch"
        expr: |-
          (
            kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
              !=
            kube_statefulset_status_replicas{job="kube-state-metrics"}
          ) and (
            changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes/kubestatefulsetreplicasmismatch"
      - alert: "KubeStatefulSetUpdateNotRolledOut"
        expr: |-
          (
            max without (revision) (
              kube_statefulset_status_current_revision{job="kube-state-metrics"}
                unless
              kube_statefulset_status_update_revision{job="kube-state-metrics"}
            )
              *
            (
              kube_statefulset_replicas{job="kube-state-metrics"}
                !=
              kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
            )
          )  and (
            changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "KubeVersionMismatch"
        expr: |-
          count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "There are {{ $value }} different semantic versions of Kubernetes components running."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Node running too many Pods"
        expr: |-
          count by(node) (
            (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
          )
          /
          max by(node) (
            kube_node_status_capacity_pods{job="kube-state-metrics"} != 1
          ) > 0.95
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Node readiness flapping"
        expr: |-
          sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Pod startup duration high"
        expr: |-
          histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Client certificate expiring - S1"
        expr: |-
          kubelet_certificate_manager_client_ttl_seconds < 86400
        labels:
          severity: critical
        annotations:
          summary: "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Server certificate expiring - S1"
        expr: |-
          kubelet_certificate_manager_server_ttl_seconds < 86400
        labels:
          severity: critical
        annotations:
          summary: "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"
      - alert: "Kubelet down"
        expr: |-
          absent(up{job="kubelet"} == 1)
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Kubelet has disappeared from Prometheus target discovery."
          runbook_url: "https://n9e.github.io/alert-rules/kubernetes"