# Prometheus alert rules — https://n9e.github.io/alert-rules/prometheus
# Generated from the Nightingale integration templates: https://github.com/ccfos/nightingale/tree/main/integrations
# Many expressions are adapted from kube-prometheus (https://github.com/prometheus-operator/kube-prometheus, Apache-2.0).
# Thresholds are starting points: review them before enabling.
groups:
  - name: "prometheus-default"
    rules:
      - alert: "ConfigReloaderSidecarErrors"
        expr: |-
          max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace.\nAs a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusOperatorListErrors"
        expr: |-
          (sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusOperatorNodeLookupErrors"
        expr: |-
          rate(prometheus_operator_node_address_lookup_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Errors while reconciling Prometheus in {{ $labels.namespace }} Namespace."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusOperatorNotReady"
        expr: |-
          min by (cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="prometheus-operator",namespace="monitoring"}[5m]) == 0)
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus operator in {{ $labels.namespace }} namespace isn't ready to reconcile {{ $labels.controller }} resources."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusOperatorReconcileErrors"
        expr: |-
          (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]))) / (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $value | humanizePercentage }} of reconciling operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusOperatorRejectedResources"
        expr: |-
          min_over_time(prometheus_operator_managed_resources{state="rejected",job="prometheus-operator",namespace="monitoring"}[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus operator in {{ $labels.namespace }} namespace rejected {{ printf \"%0.0f\" $value }} {{ $labels.controller }}/{{ $labels.resource }} resources."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus/prometheusoperatorrejectedresources"
      - alert: "PrometheusOperatorSyncFailed"
        expr: |-
          min_over_time(prometheus_operator_syncs{status="failed",job="prometheus-operator",namespace="monitoring"}[5m]) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Controller {{ $labels.controller }} in {{ $labels.namespace }} namespace fails to reconcile {{ $value }} objects."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusOperatorWatchErrors"
        expr: |-
          (sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[5m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.4
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusBadConfig"
        expr: |-
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          max_over_time(prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring"}[5m]) == 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus/prometheusbadconfig"
      - alert: "PrometheusDuplicateTimestamps"
        expr: |-
          rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf \"%.4g\" $value  }} samples/s with different values but duplicated timestamp."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusErrorSendingAlertsToAnyAlertmanager"
        expr: |-
          min without (alertmanager) (
            rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring",alertmanager!~``}[5m])
          /
            rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring",alertmanager!~``}[5m])
          )
          * 100
          > 3
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "{{ printf \"%.1f\" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusErrorSendingAlertsToSomeAlertmanagers"
        expr: |-
          (
            rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m])
          /
            rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m])
          )
          * 100
          > 1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "{{ printf \"%.1f\" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusHighQueryLoad"
        expr: |-
          avg_over_time(prometheus_engine_queries{job="prometheus-k8s",namespace="monitoring"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0.8
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has less than 20% available capacity in its query engine for the last 15 minutes."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusLabelLimitHit"
        expr: |-
          increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf \"%.0f\" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusMissingRuleEvaluations"
        expr: |-
          increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf \"%.0f\" $value }} rule group evaluations in the last 5m."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusNotConnectedToAlertmanagers"
        expr: |-
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-k8s",namespace="monitoring"}[5m]) < 1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus/prometheusnotconnectedtoalertmanagers"
      - alert: "PrometheusNotificationQueueRunningFull"
        expr: |-
          # Without min_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          (
            predict_linear(prometheus_notifications_queue_length{job="prometheus-k8s",namespace="monitoring"}[5m], 60 * 30)
          >
            min_over_time(prometheus_notifications_queue_capacity{job="prometheus-k8s",namespace="monitoring"}[5m])
          )
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusNotIngestingSamples"
        expr: |-
          (
            rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m]) <= 0
          and
            (
              sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-k8s",namespace="monitoring"}) > 0
            or
              sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-k8s",namespace="monitoring"}) > 0
            )
          )
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusOutOfOrderTimestamps"
        expr: |-
          rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf \"%.4g\" $value  }} samples/s with timestamps arriving out of order."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusRemoteStorageFailures"
        expr: |-
          (
            (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring"}[5m]))
          /
            (
              (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring"}[5m]))
            +
              (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]))
            )
          )
          * 100
          > 1
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf \"%.1f\" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}"
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusRemoteWriteBehind"
        expr: |-
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          (
            max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
          - ignoring(remote_name, url) group_right
            max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
          )
          > 120
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf \"%.1f\" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus/prometheusremotewritebehind"
      - alert: "PrometheusRemoteWriteDesiredShards"
        expr: |-
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          (
            max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-k8s",namespace="monitoring"}[5m])
          >
            max_over_time(prometheus_remote_storage_shards_max{job="prometheus-k8s",namespace="monitoring"}[5m])
          )
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance=\"%s\",job=\"prometheus-k8s\",namespace=\"monitoring\"}` $labels.instance | query | first | value }}."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus/prometheusremotewritedesiredshards"
      - alert: "PrometheusRuleFailures"
        expr: |-
          increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf \"%.0f\" $value }} rules in the last 5m."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus/prometheusrulefailures"
      - alert: "PrometheusScrapeBodySizeLimitHit"
        expr: |-
          increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf \"%.0f\" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusScrapeSampleLimitHit"
        expr: |-
          increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf \"%.0f\" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusSDRefreshFailure"
        expr: |-
          increase(prometheus_sd_refresh_failures_total{job="prometheus-k8s",namespace="monitoring"}[10m]) > 0
        for: 20m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to refresh SD with mechanism {{$labels.mechanism}}."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusTargetLimitHit"
        expr: |-
          increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf \"%.0f\" $value }} targets because the number of targets exceeded the configured target_limit."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus/prometheustargetlimithit"
      - alert: "PrometheusTargetSyncFailure"
        expr: |-
          increase(prometheus_target_sync_failed_total{job="prometheus-k8s",namespace="monitoring"}[30m]) > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "{{ printf \"%.0f\" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} have failed to sync because invalid configuration was supplied."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusTSDBCompactionsFailing"
        expr: |-
          increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
        for: 240m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"
      - alert: "PrometheusTSDBReloadsFailing"
        expr: |-
          increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
        for: 240m
        labels:
          severity: warning
        annotations:
          summary: "Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h."
          runbook_url: "https://n9e.github.io/alert-rules/prometheus"