# nginx.conf
server {
    listen 127.0.0.1:8080;
    server_name localhost;

    location /nginx_status {
        stub_status on;
        allow 127.0.0.1;   # 僅允許本機(jī)訪問(wèn)
        deny all;
    }
}

Active connections: 3 
server accepts handled requests
 12345 12345 67890 
Reading: 0 Writing: 2 Waiting: 1 

# 1. 安裝依賴
sudo apt update && sudo apt install -y build-essential libpcre3-dev libssl-dev zlib1g-dev

# 2. 下載 Nginx 源碼（與當(dāng)前版本嚴(yán)格一致！）
wget https://nginx.org/download/nginx-1.24.0.tar.gz
tar -zxvf nginx-1.24.0.tar.gz

# 3. 下載 vts 模塊
git clone https://github.com/vozlt/nginx-module-vts.git

# 4. 重新編譯 Nginx（保留原有配置參數(shù)，追加 --add-module）
cd nginx-1.24.0
./configure \
  --prefix=/etc/nginx \
  --sbin-path=/usr/sbin/nginx \
  --modules-path=/usr/lib/nginx/modules \
  --conf-path=/etc/nginx/nginx.conf \
  --error-log-path=/var/log/nginx/error.log \
  --http-log-path=/var/log/nginx/access.log \
  --pid-path=/var/run/nginx.pid \
  --with-http_ssl_module \
  --with-http_v2_module \
  --with-http_realip_module \
  --add-module=../nginx-module-vts  # ?? 關(guān)鍵：添加 vts 模塊

make && sudo make install

# nginx.conf 全局塊
vhost_traffic_status_zone;  # 必須聲明，定義共享內(nèi)存區(qū)

http {
    # ... 其他 http 配置 ...

    # 新增 vts 狀態(tài)頁(yè)（Prometheus 將從此抓?。?
    server {
        listen 127.0.0.1:8081;
        server_name _;

        location /status/format/json {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format json;  # 返回 JSON
        }

        # 可選：提供 HTML 可視化界面（調(diào)試用）
        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }
}

{
  "nginx_version": "1.24.0",
  "load_timestamp": 1715823456,
  "serverZones": {
    "example.com": {
      "requestCounter": 12345,
      "inBytes": 87654321,
      "outBytes": 234567890,
      "responses": {
        "1xx": 0,
        "2xx": 11234,
        "3xx": 567,
        "4xx": 123,
        "5xx": 42
      },
      "cached": 0,
      "uncached": 12345,
      "processing": 3,
      "connections": {
        "active": 3,
        "reading": 0,
        "writing": 2,
        "waiting": 1
      }
    }
  },
  "upstreamZones": {
    "backend-api": {
      "peers": [
        {
          "server": "10.0.1.10:8080",
          "requestCounter": 6789,
          "inBytes": 45678901,
          "outBytes": 123456789,
          "responses": {"1xx":0,"2xx":6234,"3xx":123,"4xx":45,"5xx":12},
          "sent": 123456789,
          "received": 45678901,
          "fails": 0,
          "unavail": 0,
          "healthChecks": {"checks":123,"fails":0,"unhealthy":0,"lastPassed":1715823455}
        }
      ]
    }
  }
}

global:
  scrape_interval: 15s
  evaluation_interval: 15s
scrape_configs:
  # 1. 抓取本機(jī) Nginx vts 指標(biāo)（Prometheus 格式）
  - job_name: 'nginx-vts'
    static_configs:
      - targets: ['127.0.0.1:8081']  # vts metrics 端口
    metrics_path: /status/format/prometheus  # ?? 關(guān)鍵：使用 Prometheus 格式
    relabel_configs:
      # 為所有指標(biāo)添加固定標(biāo)簽，便于多實(shí)例區(qū)分
      - source_labels: [__address__]
        target_label: instance
        replacement: nginx-prod-main
      - target_label: job
        replacement: nginx-vts
      # 重寫(xiě)指標(biāo)名前綴（可選，避免與其他 exporter 沖突）
      - action: labelmap
        regex: __meta_vts_(.+)
  # 2. 抓取 Prometheus 自身指標(biāo)（用于監(jiān)控 Prometheus 健康）
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  # 3. 抓取 Node Exporter（服務(wù)器基礎(chǔ)指標(biāo)，與 Nginx 關(guān)聯(lián)分析）
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']

# 下載 Prometheus（略去下載步驟，假設(shè)已解壓到 /opt/prometheus）
cd /opt/prometheus
./prometheus --config.file=prometheus.yml --web.listen-address="0.0.0.0:9090"

# 查看 Nginx 總請(qǐng)求數(shù)（按 server zone）
sum by (server) (nginx_vts_server_request_counter_total)

# 查看 5xx 錯(cuò)誤率（過(guò)去5分鐘）
sum(rate(nginx_vts_server_responses_total{code="5xx"}[5m])) 
/ 
sum(rate(nginx_vts_server_request_counter_total[5m]))

# 查看 upstream peer 的失敗次數(shù)
nginx_vts_upstream_peer_fails_total{upstream="backend-api"}

// pom.xml 關(guān)鍵依賴
<dependencies>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    <dependency>
        <groupId>io.micrometer</groupId>
        <artifactId>micrometer-registry-prometheus</artifactId>
    </dependency>
    <!-- 添加 Actuator，暴露 /actuator/prometheus -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
    </dependency>
</dependencies>

// src/main/java/com/example/UserController.java
@RestController
@RequestMapping("/api/v1/users")
public class UserController {
    private final MeterRegistry meterRegistry;
    private final Counter errorCounter;
    public UserController(MeterRegistry meterRegistry) {
        this.meterRegistry = meterRegistry;
        // 自定義錯(cuò)誤計(jì)數(shù)器，帶業(yè)務(wù)維度
        this.errorCounter = Counter.builder("user.service.errors")
                .description("Count of business errors in user service")
                .tag("type", "database_timeout") // 可擴(kuò)展為 network, cache, auth...
                .register(meterRegistry);
    }
    @GetMapping("/{id}")
    public ResponseEntity<User> getUser(@PathVariable Long id) {
        try {
            // 模擬數(shù)據(jù)庫(kù)查詢（可能超時(shí)）
            User user = fetchUserFromDB(id);
            return ResponseEntity.ok(user);
        } catch (DatabaseTimeoutException e) {
            // 記錄業(yè)務(wù)錯(cuò)誤，同時(shí)觸發(fā) Nginx 502（若上游超時(shí)）
            errorCounter.increment();
            // 記錄到 Micrometer，會(huì)被 Prometheus 抓取
            meterRegistry.counter("user.db.timeout.total", "service", "user-service").increment();
            return ResponseEntity.status(500).build();
        }
    }
    private User fetchUserFromDB(Long id) throws DatabaseTimeoutException {
        // 模擬隨機(jī)超時(shí)（用于測(cè)試告警）
        if (Math.random() > 0.95) {
            throw new DatabaseTimeoutException("Simulated DB timeout");
        }
        return new User(id, "John Doe", "john@example.com");
    }
}
// 自定義異常
class DatabaseTimeoutException extends RuntimeException {
    public DatabaseTimeoutException(String msg) {
        super(msg);
    }
}

# application.yml
management:
  endpoints:
    web:
      exposure:
        include: health,info,prometheus  # ?? 關(guān)鍵：暴露 prometheus 端點(diǎn)
  endpoint:
    prometheus:
      scrape-interval: 15s
# 暴露 Actuator 端點(diǎn)在 /actuator
server:
  port: 8080

# HELP user_service_errors Count of business errors in user service
# TYPE user_service_errors counter
user_service_errors{type="database_timeout",} 3.0
# HELP user_db_timeout_total  
# TYPE user_db_timeout_total counter
user_db_timeout_total{service="user-service",} 3.0

# nginx.conf
upstream backend-api {
    server 127.0.0.1:8080 max_fails=3 fail_timeout=30s;
    # 可添加更多實(shí)例實(shí)現(xiàn)負(fù)載均衡
    # server 10.0.1.11:8080;
}

server {
    listen 80;
    server_name example.com;

    location /api/ {
        proxy_pass http://backend-api;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        # 關(guān)鍵：傳遞原始請(qǐng)求路徑，vts 會(huì)自動(dòng)記錄
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        # 設(shè)置超時(shí)，避免長(zhǎng)連接拖垮 Nginx
        proxy_connect_timeout 5s;
        proxy_send_timeout 30s;
        proxy_read_timeout 30s;

        # 啟用健康檢查（vts 會(huì)自動(dòng)采集）
        health_check interval=5 rise=2 fall=3;
    }
}

// Nginx 層 5xx 錯(cuò)誤率（過(guò)去5分鐘）
sum(rate(nginx_vts_server_responses_total{code="5xx", server="example.com"}[5m])) 
/ 
sum(rate(nginx_vts_server_request_counter_total{server="example.com"}[5m]))

// Java 應(yīng)用層 DB 超時(shí)錯(cuò)誤率（過(guò)去5分鐘）
sum(rate(user_db_timeout_total{service="user-service"}[5m]))
/
sum(rate(http_server_requests_seconds_count{application="user-service"}[5m]))

groups:
- name: nginx-alerts
  rules:
  # 規(guī)則1：Nginx 連接耗盡風(fēng)險(xiǎn)（緊急！）
  - alert: NginxConnectionExhaustionWarning
    expr: |
      (nginx_vts_server_connections_active{server="example.com"} 
       / nginx_vts_server_worker_connections{server="example.com"}) * 100 > 80
    for: 2m
    labels:
      severity: warning
      team: infra
    annotations:
      summary: "Nginx connection usage > 80% on {{ $labels.instance }}"
      description: "Active connections: {{ $value }}%. Check upstream latency or increase worker_connections."
  # 規(guī)則2：上游服務(wù)全宕機(jī)（嚴(yán)重?。?
  - alert: UpstreamAllDown
    expr: |
      count by (upstream) (
        nginx_vts_upstream_peer_state{upstream="backend-api", state="down"}
      ) == count by (upstream) (
        nginx_vts_upstream_peer_state{upstream="backend-api"}
      )
    for: 1m
    labels:
      severity: critical
      team: backend
    annotations:
      summary: "All peers DOWN in upstream {{ $labels.upstream }}"
      description: "No healthy backend instances available. Immediate investigation required!"
  # 規(guī)則3：5xx 錯(cuò)誤率突增（注意?。?
  - alert: HighNginx5xxRate
    expr: |
      sum(rate(nginx_vts_server_responses_total{code="5xx", server="example.com"}[5m])) 
      / 
      sum(rate(nginx_vts_server_request_counter_total{server="example.com"}[5m])) > 0.05
    for: 3m
    labels:
      severity: warning
      team: sre
    annotations:
      summary: "5xx error rate > 5% for 3 minutes on {{ $labels.instance }}"
      description: "Current rate: {{ $value | humanize }}%. Correlate with upstream and application metrics."

global:
  resolve_timeout: 5m
route:
  group_by: ['alertname', 'team']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 1h
  receiver: 'wechat-notifications'
receivers:
- name: 'wechat-notifications'
  wechat_configs:
  - send_resolved: true
    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_WEBHOOK_KEY'  # 替換為你的企業(yè)微信機(jī)器人 key
    message: |
      {{ define "__text_alert_list" }}{{ range .Alerts }}
      [{{ .Labels.severity }}] {{ .Labels.alertname }}
      {{ .Annotations.summary }}
      {{ .Annotations.description }}
      Details: {{ .GeneratorURL }}
      {{ end }}{{ end }}
      {{ if gt (len .Alerts.Firing) 0 -}}
      ?? FIRING:
      {{ template "__text_alert_list" .Alerts.Firing }}
      {{ end }}
      {{ if gt (len .Alerts.Resolved) 0 -}}
      ? RESOLVED:
      {{ template "__text_alert_list" .Alerts.Resolved }}
      {{ end }}

./alertmanager --config.file=alertmanager.yml --web.external-url=http://localhost:9093

alerting:
  alertmanagers:
  - static_configs:
    - targets: ['localhost:9093']

# helm install ingress-nginx ingress-nginx/ingress-nginx \
#   --set controller.metrics.enabled=true \
#   --set controller.metrics.serviceMonitor.enabled=true \
#   --set controller.stats.enabled=true

# 查看所有 Ingress 的 5xx 錯(cuò)誤率（按 namespace 分組）
sum by (namespace, ingress) (
  rate(nginx_ingress_controller_requests_total{status=~"5.."}[5m])
)
/
sum by (namespace, ingress) (
  rate(nginx_ingress_controller_requests_total[5m])
)

# 檢測(cè) SSL 證書(shū)即將過(guò)期（< 7 天）
min by (host) (
  nginx_ingress_controller_ssl_expire_time_seconds - time()
) < 604800

- action: labeldrop
  regex: "(exported_instance|job)"