groups:
  - name: cmpp-service-recording
    interval: 15s
    rules:
      - record: cmpp:service_api:requests_per_second
        expr: sum(rate(cmpp_api_http_requests_total[5m]))
      - record: cmpp:service_api:error_percent
        expr: 100 * sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001)
      - record: cmpp:service_api:latency_p95_seconds
        expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m])))
      - record: cmpp:service_api:event_loop_p99_seconds
        expr: cmpp_api_nodejs_event_loop_lag_p99_seconds
      - record: cmpp:service_gateway:submits_per_second
        expr: sum(rate(cmpp_gateway_submit_total[5m]))
      - record: cmpp:service_gateway:failure_percent
        expr: 100 * sum(rate(cmpp_gateway_submit_total{result="failed"}[5m])) / clamp_min(sum(rate(cmpp_gateway_submit_total[5m])), 0.001)
      - record: cmpp:service_gateway:queue_pending
        expr: cmpp_gateway_submit_queue_pending
      - record: cmpp:service_gateway:queue_lag
        expr: cmpp_gateway_submit_queue_lag
      - record: cmpp:service_gateway:queue_oldest_seconds
        expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds
      - record: cmpp:service_postgresql:connection_percent
        expr: 100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)
      - record: cmpp:service_postgresql:deadlocks_15m
        expr: sum(increase(pg_stat_database_deadlocks[15m]))
      - record: cmpp:service_redis:memory_percent
        expr: (100 * redis_memory_used_bytes / redis_memory_max_bytes) and on(instance) (redis_memory_max_bytes > 0)
      - record: cmpp:service_redis:memory_used_bytes
        expr: redis_memory_used_bytes
      - record: cmpp:service_redis:evictions_5m
        expr: increase(redis_evicted_keys_total[5m])
      - record: cmpp:service_redis:connected_clients
        expr: redis_connected_clients
      - record: cmpp:service_nginx:connections_active
        expr: nginx_connections_active
      - record: cmpp:service_nginx:requests_per_second
        expr: rate(nginx_http_requests_total[5m])
      - record: cmpp:service_minio:capacity_percent
        expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes)
      - record: cmpp:service_minio:usage_bytes
        expr: minio_cluster_usage_total_bytes
      - record: cmpp:service_minio:objects
        expr: minio_cluster_usage_object_total
      - record: cmpp:service_minio:drives_offline
        expr: minio_cluster_drive_offline_total

  - name: cmpp-host-resources
    rules:
      - alert: NodeExporterDown
        expr: up{job="node"} == 0
        for: 2m
        labels:
          severity: critical
          service: node-exporter
        annotations:
          summary: 主机指标采集不可用
          description: Prometheus连续2分钟无法采集Node Exporter。
          currentValue: "{{ $value }}"
          threshold: "up = 1"

      - alert: HostCpuUsageWarning
        expr: (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80) and (100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) <= 90)
        for: 10m
        labels:
          severity: warning
          service: host
        annotations:
          summary: CPU使用率持续偏高
          description: 主机CPU使用率连续10分钟高于80%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "80%"

      - alert: HostCpuUsageCritical
        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 5m
        labels:
          severity: critical
          service: host
        annotations:
          summary: CPU使用率严重超限
          description: 主机CPU使用率连续5分钟高于90%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "90%"

      - alert: HostMemoryUsageWarning
        expr: ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85) and ((1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 <= 95)
        for: 10m
        labels:
          severity: warning
          service: host
        annotations:
          summary: 内存使用率持续偏高
          description: 主机可用内存连续10分钟低于15%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "85%"

      - alert: HostMemoryUsageCritical
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
        for: 5m
        labels:
          severity: critical
          service: host
        annotations:
          summary: 内存使用率严重超限
          description: 主机可用内存连续5分钟低于5%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "95%"

      - alert: HostRootDiskUsageWarning
        expr: ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
        for: 15m
        labels:
          severity: warning
          service: host
        annotations:
          summary: 根文件系统空间不足
          description: 根文件系统使用率连续15分钟高于80%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "80%"

      - alert: HostRootDiskUsageCritical
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
        for: 5m
        labels:
          severity: critical
          service: host
        annotations:
          summary: 根文件系统空间严重不足
          description: 根文件系统使用率连续5分钟高于90%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "90%"

      - alert: HostRootInodeUsageWarning
        expr: ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 80) and ((1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 <= 90)
        for: 15m
        labels:
          severity: warning
          service: host
        annotations:
          summary: 根文件系统inode余量偏低
          description: 根文件系统inode使用率连续15分钟高于80%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "80%"

      - alert: HostRootInodeUsageCritical
        expr: (1 - node_filesystem_files_free{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_files{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}) * 100 > 90
        for: 5m
        labels:
          severity: critical
          service: host
        annotations:
          summary: 根文件系统inode严重不足
          description: 根文件系统inode使用率连续5分钟高于90%。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "90%"

      - alert: HostCpuIowaitWarning
        expr: (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 20) and (avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 <= 35)
        for: 10m
        labels:
          severity: warning
          service: host
        annotations:
          summary: CPU iowait持续偏高
          description: 主机CPU iowait连续10分钟高于20%，请检查磁盘I/O。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "20%"

      - alert: HostCpuIowaitCritical
        expr: avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 35
        for: 10m
        labels:
          severity: critical
          service: host
        annotations:
          summary: CPU iowait严重超限
          description: 主机CPU iowait连续10分钟高于35%，磁盘I/O可能已成为瓶颈。
          currentValue: "{{ printf \"%.1f\" $value }}%"
          threshold: "35%"

  - name: cmpp-core-services
    rules:
      - alert: CmppCoreServiceInactive
        expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
        for: 2m
        labels:
          severity: critical
          service: "{{ $labels.name }}"
        annotations:
          summary: CMPP核心服务未处于active状态
          description: "systemd服务 {{ $labels.name }} 连续2分钟未处于active状态。"
          currentValue: "{{ $value }}"
          threshold: "active = 1"

  - name: cmpp-api-runtime
    rules:
      - alert: CmppApiMetricsDown
        expr: up{job="cmpp-api"} == 0
        for: 2m
        labels: { severity: critical, service: api }
        annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
      - alert: CmppApiHttpErrorRateWarning
        expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
        for: 5m
        labels: { severity: warning, service: api }
        annotations: { summary: "API 5xx错误率偏高", description: "API 5xx错误率连续5分钟高于1%，且窗口内至少5次错误。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "1%" }
      - alert: CmppApiHttpErrorRateCritical
        expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
        for: 5m
        labels: { severity: critical, service: api }
        annotations: { summary: "API 5xx错误率严重超限", description: "API 5xx错误率连续5分钟高于5%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "5%" }
      - alert: CmppApiLatencyWarning
        expr: (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) > 1) and (histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[10m]))) <= 3)
        for: 10m
        labels: { severity: warning, service: api }
        annotations: { summary: "API P95响应偏慢", description: "API P95响应时间连续10分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }
      - alert: CmppApiLatencyCritical
        expr: histogram_quantile(0.95, sum by (le) (rate(cmpp_api_http_request_duration_seconds_bucket[5m]))) > 3
        for: 5m
        labels: { severity: critical, service: api }
        annotations: { summary: "API P95响应严重超时", description: "API P95响应时间连续5分钟超过3秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "3s" }
      - alert: CmppApiEventLoopLagWarning
        expr: (cmpp_api_nodejs_event_loop_lag_p99_seconds > 0.2) and (cmpp_api_nodejs_event_loop_lag_p99_seconds <= 1)
        for: 10m
        labels: { severity: warning, service: api }
        annotations: { summary: "API事件循环延迟偏高", description: "Node.js事件循环P99延迟连续10分钟超过200ms。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "0.2s" }
      - alert: CmppApiEventLoopLagCritical
        expr: cmpp_api_nodejs_event_loop_lag_p99_seconds > 1
        for: 5m
        labels: { severity: critical, service: api }
        annotations: { summary: "API事件循环严重阻塞", description: "Node.js事件循环P99延迟连续5分钟超过1秒。", currentValue: "{{ printf \"%.3f\" $value }}s", threshold: "1s" }

  - name: cmpp-gateway-runtime
    rules:
      - alert: CmppGatewayMetricsDown
        expr: up{job="cmpp-gateway"} == 0
        for: 2m
        labels: { severity: critical, service: gateway }
        annotations: { summary: "Gateway指标采集不可用", description: "Prometheus连续2分钟无法读取Gateway指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
      - alert: CmppGatewaySubmitWorkerDown
        expr: cmpp_gateway_submit_worker_up == 0
        for: 1m
        labels: { severity: critical, service: gateway }
        annotations: { summary: "Gateway提交消费者未运行", description: "Gateway进程存活，但提交消费者未成功初始化。", currentValue: "{{ $value }}", threshold: "1" }
      - alert: CmppGatewayUpstreamConnectionShortage
        expr: cmpp_gateway_upstream_connections{state="connected"} < cmpp_gateway_upstream_connections{state="desired"}
        for: 2m
        labels: { severity: critical, service: gateway }
        annotations: { summary: "Gateway上游连接不足", description: "实际上游CMPP连接数连续2分钟低于期望数。", currentValue: "{{ $value }}", threshold: "connected = desired" }
      - alert: CmppGatewayQueueDelayedWarning
        expr: (cmpp_gateway_submit_queue_oldest_pending_age_seconds > 30) and (cmpp_gateway_submit_queue_oldest_pending_age_seconds <= 120)
        for: 2m
        labels: { severity: warning, service: gateway }
        annotations: { summary: "Gateway提交队列开始延迟", description: "Redis Stream最旧pending消息等待超过30秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "30s" }
      - alert: CmppGatewayQueueDelayedCritical
        expr: cmpp_gateway_submit_queue_oldest_pending_age_seconds > 120
        for: 2m
        labels: { severity: critical, service: gateway }
        annotations: { summary: "Gateway提交队列严重延迟", description: "Redis Stream最旧pending消息等待超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }

  - name: cmpp-data-services
    rules:
      - alert: PostgresExporterDown
        expr: up{job="postgresql"} == 0 or pg_up == 0
        for: 2m
        labels: { severity: critical, service: postgresql }
        annotations: { summary: "PostgreSQL指标或数据库不可用", description: "PostgreSQL Exporter或其数据库连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
      - alert: PostgresConnectionsWarning
        expr: (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.70) and (sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) <= 0.85)
        for: 10m
        labels: { severity: warning, service: postgresql }
        annotations: { summary: "PostgreSQL连接使用率偏高", description: "数据库连接数连续10分钟超过上限的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
      - alert: PostgresConnectionsCritical
        expr: sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1) > 0.85
        for: 5m
        labels: { severity: critical, service: postgresql }
        annotations: { summary: "PostgreSQL连接即将耗尽", description: "数据库连接数连续5分钟超过上限的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
      - alert: PostgresDeadlocksDetected
        expr: sum(increase(pg_stat_database_deadlocks[15m])) > 0
        for: 1m
        labels: { severity: warning, service: postgresql }
        annotations: { summary: "PostgreSQL发生死锁", description: "15分钟窗口内检测到数据库死锁。", currentValue: "{{ $value }}", threshold: "0" }
      - alert: RedisExporterDown
        expr: up{job="redis"} == 0 or redis_up == 0
        for: 2m
        labels: { severity: critical, service: redis }
        annotations: { summary: "Redis指标或服务不可用", description: "Redis Exporter或Redis连接连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }
      - alert: RedisMemoryWarning
        expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.70) and (redis_memory_used_bytes / redis_memory_max_bytes <= 0.85)
        for: 10m
        labels: { severity: warning, service: redis }
        annotations: { summary: "Redis内存使用率偏高", description: "Redis内存连续10分钟超过maxmemory的70%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "70%" }
      - alert: RedisMemoryCritical
        expr: (redis_memory_max_bytes > 0) and (redis_memory_used_bytes / redis_memory_max_bytes > 0.85)
        for: 5m
        labels: { severity: critical, service: redis }
        annotations: { summary: "Redis内存即将耗尽", description: "Redis内存连续5分钟超过maxmemory的85%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "85%" }
      - alert: RedisUnexpectedEvictions
        expr: increase(redis_evicted_keys_total[5m]) > 0
        for: 1m
        labels: { severity: critical, service: redis }
        annotations: { summary: "Redis发生Key淘汰", description: "Redis承载队列和运行状态，5分钟内不应出现淘汰。", currentValue: "{{ $value }}", threshold: "0" }
      - alert: RedisRejectedConnections
        expr: increase(redis_rejected_connections_total[5m]) > 0
        for: 1m
        labels: { severity: critical, service: redis }
        annotations: { summary: "Redis拒绝连接", description: "5分钟内Redis出现被拒绝连接。", currentValue: "{{ $value }}", threshold: "0" }

  - name: cmpp-storage-and-edge
    rules:
      - alert: MinioMetricsDown
        expr: up{job="minio"} == 0
        for: 2m
        labels: { severity: critical, service: minio }
        annotations: { summary: "MinIO指标采集不可用", description: "Prometheus连续2分钟无法读取MinIO原生指标。", currentValue: "{{ $value }}", threshold: "up = 1" }
      - alert: MinioCapacityWarning
        expr: (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 80) and (100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) <= 90)
        for: 15m
        labels: { severity: warning, service: minio }
        annotations: { summary: "MinIO存储容量偏高", description: "MinIO可用容量使用率连续15分钟超过80%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "80%" }
      - alert: MinioCapacityCritical
        expr: 100 * (1 - minio_cluster_capacity_usable_free_bytes / minio_cluster_capacity_usable_total_bytes) > 90
        for: 5m
        labels: { severity: critical, service: minio }
        annotations: { summary: "MinIO存储容量即将耗尽", description: "MinIO可用容量使用率连续5分钟超过90%。", currentValue: "{{ printf \"%.1f\" $value }}%", threshold: "90%" }
      - alert: MinioDriveOffline
        expr: minio_cluster_drive_offline_total > 0
        for: 1m
        labels: { severity: critical, service: minio }
        annotations: { summary: "MinIO存储盘离线", description: "MinIO检测到离线存储盘。", currentValue: "{{ $value }}", threshold: "0" }
      - alert: NginxExporterDown
        expr: up{job="nginx"} == 0 or nginx_up == 0
        for: 2m
        labels: { severity: critical, service: nginx }
        annotations: { summary: "Nginx指标或状态页不可用", description: "Nginx Exporter或回环stub_status连续2分钟不可用。", currentValue: "{{ $value }}", threshold: "up = 1" }

  - name: cmpp-monitoring-self
    rules:
      - alert: PrometheusScrapeSlow
        expr: scrape_duration_seconds / scrape_interval_seconds > 0.8
        for: 5m
        labels: { severity: warning, service: prometheus }
        annotations: { summary: "Prometheus采集接近超时", description: "采集耗时连续5分钟超过采集周期的80%。", currentValue: "{{ printf \"%.2f\" $value }}", threshold: "80%" }
      - alert: PrometheusRuleEvaluationFailures
        expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
        for: 1m
        labels: { severity: critical, service: prometheus }
        annotations: { summary: "Prometheus告警规则计算失败", description: "5分钟内出现告警规则计算失败。", currentValue: "{{ $value }}", threshold: "0" }