prometheus_alerts.yml

"groups":
- "name": "jaeger_alerts"
  "rules":
  - "alert": "JaegerHTTPServerErrs"
    "annotations":
      "message": |
        {{ $labels.job }} {{ $labels.instance }} is experiencing {{ printf "%.2f" $value }}% HTTP errors.
    "expr": "100 * sum(rate(jaeger_agent_http_server_errors_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_agent_http_server_total[1m])) by (instance, job, namespace)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"
  - "alert": "JaegerRPCRequestsErrors"
    "annotations":
      "message": |
        {{ $labels.job }} {{ $labels.instance }} is experiencing {{ printf "%.2f" $value }}% RPC HTTP errors.
    "expr": "100 * sum(rate(jaeger_client_jaeger_rpc_http_requests{status_code=~\"4xx|5xx\"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_client_jaeger_rpc_http_requests[1m])) by (instance, job, namespace)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"
  - "alert": "JaegerClientSpansDropped"
    "annotations":
      "message": |
        service {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans.
    "expr": "100 * sum(rate(jaeger_reporter_spans{result=~\"dropped|err\"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_reporter_spans[1m])) by (instance, job, namespace)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"
  - "alert": "JaegerAgentSpansDropped"
    "annotations":
      "message": |
        agent {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans.
    "expr": "100 * sum(rate(jaeger_agent_reporter_batches_failures_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_agent_reporter_batches_submitted_total[1m])) by (instance, job, namespace)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"
  - "alert": "JaegerCollectorDroppingSpans"
    "annotations":
      "message": |
        collector {{ $labels.job }} {{ $labels.instance }} is dropping {{ printf "%.2f" $value }}% spans.
    "expr": "100 * sum(rate(jaeger_collector_spans_dropped_total[1m])) by (instance, job, namespace) / sum(rate(jaeger_collector_spans_received_total[1m])) by (instance, job, namespace)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"
  - "alert": "JaegerSamplingUpdateFailing"
    "annotations":
      "message": |
        {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating sampling policies.
    "expr": "100 * sum(rate(jaeger_sampler_queries{result=\"err\"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_sampler_queries[1m])) by (instance, job, namespace)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"
  - "alert": "JaegerThrottlingUpdateFailing"
    "annotations":
      "message": |
        {{ $labels.job }} {{ $labels.instance }} is failing {{ printf "%.2f" $value }}% in updating throttling policies.
    "expr": "100 * sum(rate(jaeger_throttler_updates{result=\"err\"}[1m])) by (instance, job, namespace) / sum(rate(jaeger_throttler_updates[1m])) by (instance, job, namespace)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"
  - "alert": "JaegerQueryReqsFailing"
    "annotations":
      "message": |
        {{ $labels.job }} {{ $labels.instance }} is seeing {{ printf "%.2f" $value }}% query errors on {{ $labels.operation }}.
    "expr": "100 * sum(rate(jaeger_query_requests_total{result=\"err\"}[1m])) by (instance, job, namespace, operation) / sum(rate(jaeger_query_requests_total[1m])) by (instance, job, namespace, operation)> 1"
    "for": "15m"
    "labels":
      "severity": "warning"