http {
    # 1. 提升錯(cuò)誤日志級(jí)別（生產(chǎn)謹(jǐn)慎用 info，可臨時(shí)切 warn）
    error_log /var/log/nginx/error.log warn;

    # 2. 擴(kuò)展 access log 格式，包含上游關(guān)鍵指標(biāo)
    log_format upstream '$remote_addr - $remote_user [$time_local] '
                         '"$request" $status $body_bytes_sent '
                         '"$http_referer" "$http_user_agent" '
                         'rt=$request_time uct="$upstream_connect_time" '
                         'uht="$upstream_header_time" urt="$upstream_response_time" '
                         'ups="$upstream_addr"';

    access_log /var/log/nginx/access.log upstream;

    # 3. 啟用 upstream 日志（記錄每個(gè) upstream server 的狀態(tài)變化）
    upstream_conf;
}

@RestController
public class UnsafeController {
    @GetMapping("/unsafe-json")
    public void unsafeJsonResponse(HttpServletResponse response) throws IOException {
        // ?? 嚴(yán)重錯(cuò)誤：未設(shè)置狀態(tài)碼、Content-Type、未使用 ResponseEntity
        response.getOutputStream().write("{\"code\":0,\"msg\":\"success\"}".getBytes(StandardCharsets.UTF_8));
        // 缺少 response.getOutputStream().flush()，且未關(guān)閉流！
    }
}

@RestController
@RequestMapping("/api")
public class SafeController {
    // ? 正確：使用 ResponseEntity，自動(dòng)設(shè)置狀態(tài)碼、Content-Type、Content-Length
    @GetMapping("/users")
    public ResponseEntity<List<User>> getUsers() {
        List<User> users = userService.findAll();
        return ResponseEntity.ok()
                .header("X-Processed-By", "Java-SpringBoot") // 自定義 Header
                .body(users);
    }
    // ? 正確：對(duì)流式響應(yīng)，使用 StreamingResponseBody 并顯式設(shè)置 Header
    @GetMapping(value = "/large-report", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
    public ResponseEntity<StreamingResponseBody> downloadReport() {
        return ResponseEntity.ok()
                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=report.csv")
                .header(HttpHeaders.CONTENT_TYPE, "text/csv;charset=UTF-8")
                .body(outputStream -> {
                    try (PrintWriter writer = new PrintWriter(outputStream, true, StandardCharsets.UTF_8)) {
                        writer.println("id,name,email");
                        userService.exportAllUsers(writer); // 流式寫入
                    }
                });
    }
}

upstream java_backend {
    server 10.0.1.5:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.6:8080 max_fails=3 fail_timeout=30s;

    # ? 關(guān)鍵：允許上游返回非標(biāo)準(zhǔn)但可解析的狀態(tài)碼（如 Spring Boot Actuator 的 200/503）
    #      但禁止 502（避免循環(huán)）
    proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
    proxy_next_upstream_tries 3;
    proxy_next_upstream_timeout 10s;

    # ? 關(guān)鍵：強(qiáng)制 Nginx 驗(yàn)證上游響應(yīng)頭完整性
    proxy_buffering on;
    proxy_buffer_size 128k;
    proxy_buffers 4 256k;
    proxy_busy_buffers_size 256k;
    proxy_max_temp_file_size 0; # 禁用臨時(shí)文件，避免磁盤 IO 影響

    # ? 關(guān)鍵：設(shè)置合理的讀取超時(shí)，避免因上游慢導(dǎo)致 502（實(shí)為 504）
    proxy_read_timeout 60s;
    proxy_send_timeout 60s;
}

# 模擬一個(gè)“非法”響應(yīng)（無(wú)狀態(tài)行）—— 觸發(fā) 502
printf "Content-Type: application/json\r\n\r\n{\"error\":\"bad\"}" | nc 10.0.1.5 8080

# 使用 curl 查看 Nginx 是否透?jìng)魃嫌?Header（調(diào)試 502 時(shí)關(guān)鍵?。?
curl -I -v https://api.example.com/api/users
# 觀察響應(yīng)頭中是否有 X-Upstream-Addr, X-Upstream-Response-Time 等 Nginx 注入頭

@Component
public class DatabaseHealthIndicator implements ReactiveHealthIndicator {
    private final DataSource dataSource;
    public DatabaseHealthIndicator(DataSource dataSource) {
        this.dataSource = dataSource;
    }
    @Override
    public Mono<Health> health() {
        return Mono.fromCallable(() -> {
            try (Connection conn = dataSource.getConnection()) {
                // ? 關(guān)鍵：只執(zhí)行輕量級(jí)檢查（如 SELECT 1），避免長(zhǎng)事務(wù)
                try (Statement stmt = conn.createStatement()) {
                    stmt.execute("SELECT 1");
                    return Health.up()
                            .withDetail("database", "PostgreSQL")
                            .withDetail("version", getDbVersion(conn))
                            .build();
                }
            } catch (Exception ex) {
                // ?? 重要：捕獲所有異常，避免 HealthCheck 拋出 RuntimeException 導(dǎo)致整個(gè) endpoint 失效
                return Health.down()
                        .withDetail("error", ex.getMessage())
                        .withDetail("timestamp", Instant.now().toString())
                        .build();
            }
        });
    }
    private String getDbVersion(Connection conn) throws SQLException {
        return conn.getMetaData().getDatabaseProductVersion();
    }
}

upstream java_backend {
    # ? 開(kāi)啟主動(dòng)健康檢查（需 nginx-plus 或開(kāi)源版編譯時(shí)加入 http_upstream_check_module）
    check interval=3 rise=2 fall=5 timeout=1 type=http;
    check_http_send "HEAD /actuator/health HTTP/1.1\r\nHost: localhost\r\n\r\n";
    check_http_expect_alive http_2xx http_3xx;

    server 10.0.1.5:8080;
    server 10.0.1.6:8080;
}

# 暴露健康檢查狀態(tài)頁(yè)（供運(yùn)維監(jiān)控）
server {
    listen 8081;
    location /upstream_status {
        check_status;
        access_log off;
        allow 127.0.0.1;
        deny all;
    }
}

// 使用 Resilience4j 實(shí)現(xiàn)優(yōu)雅降級(jí)
@Value("${resilience4j.circuitbreaker.instances.payment.timeout-duration:3s}")
private Duration timeoutDuration;
@Bean
public CircuitBreaker circuitBreaker() {
    CircuitBreakerConfig config = CircuitBreakerConfig.custom()
            .failureRateThreshold(50) // 錯(cuò)誤率 >50% 觸發(fā)熔斷
            .waitDurationInOpenState(Duration.ofSeconds(60)) // 保持 OPEN 60 秒
            .ringBufferSizeInHalfOpenState(10) // HALF_OPEN 狀態(tài)下最多試 10 次
            .recordExceptions(TimeoutException.class, SQLException.class)
            .ignoreExceptions(BusinessException.class) // 業(yè)務(wù)異常不計(jì)入失敗
            .build();
    return CircuitBreakerRegistry.of(config).circuitBreaker("paymentService");
}
// Controller 中使用
@GetMapping("/order/{id}")
public ResponseEntity<Order> getOrder(@PathVariable Long id) {
    Supplier<Order> orderSupplier = () -> paymentService.getOrder(id);
    try {
        Order order = circuitBreaker.executeSupplier(orderSupplier);
        return ResponseEntity.ok(order);
    } catch (CallNotPermittedException e) {
        // ? 熔斷開(kāi)啟時(shí)，主動(dòng)返回 503 + 友好提示
        return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
                .header("X-RateLimit-Reset", String.valueOf(System.currentTimeMillis() + 60_000))
                .body(Order.builder()
                        .status("SERVICE_UNAVAILABLE")
                        .message("Payment service is temporarily unavailable. Please try again later.")
                        .build());
    } catch (Exception e) {
        throw new RuntimeException("Failed to fetch order", e);
    }
}

# 1. 下載 Arthas（Alibaba 開(kāi)源 Java 診斷神器）
curl -O https://arthas.aliyun.com/arthas-boot.jar

# 2. Attach 到 Java 進(jìn)程（PID 從 jps 獲?。?
java -jar arthas-boot.jar 12345

# 3. 監(jiān)控慢方法（如 findAllUsers 耗時(shí) > 5s）
[arthas@12345]$ trace com.example.service.UserService findAllUsers --condition 'duration>5000'

# 4. 查看線程堆棧（定位阻塞點(diǎn)）
[arthas@12345]$ thread -n 3  # 顯示 CPU 最高 3 個(gè)線程
[arthas@12345]$ thread 25   # 查看線程 ID 25 的完整堆棧

# application.yml
spring:
  datasource:
    hikari:
      # ? 關(guān)鍵：連接池大小必須匹配數(shù)據(jù)庫(kù)最大連接數(shù)
      maximum-pool-size: 20
      minimum-idle: 5
      # ? 關(guān)鍵：連接超時(shí)必須 < Nginx proxy_read_timeout
      connection-timeout: 30000  # 30s
      validation-timeout: 3000
      idle-timeout: 600000
      max-lifetime: 1800000
  # ? 關(guān)鍵：WebMvc 線程池（Tomcat/Jetty）
  web:
    server:
      tomcat:
        threads:
          max: 200          # Tomcat 最大工作線程
          min-spare: 10     # 最小空閑線程
  # ? 關(guān)鍵：異步任務(wù)線程池（@Async）
  task:
    execution:
      pool:
        core-size: 10
        max-size: 50
        queue-capacity: 100

upstream java_backend {
    server 10.0.1.5:8080;
    server 10.0.1.6:8080;

    # ? 黃金公式：proxy_read_timeout ≥ (DB連接超時(shí) + 業(yè)務(wù)邏輯P99 + GC停頓P99)
    #    假設(shè) DB 超時(shí) 30s，業(yè)務(wù) P99 15s，GC P99 2s → 至少 47s，建議設(shè)為 60s
    proxy_read_timeout 60s;
    proxy_send_timeout 60s;
    proxy_connect_timeout 5s; # 網(wǎng)絡(luò)層握手，通常 1-3s 足夠

    # ? 關(guān)鍵：?jiǎn)⒂?keepalive，復(fù)用連接，避免反復(fù)握手耗時(shí)
    keepalive 32;
}

server {
    location /api/ {
        proxy_pass http://java_backend;
        proxy_http_version 1.1;
        proxy_set_header Connection '';
        # ? 關(guān)鍵：透?jìng)髡鎸?shí)客戶端 IP，用于后端日志追蹤
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

@Service
public class ReportService {
    private final RestTemplate restTemplate;
    public ReportService(RestTemplateBuilder builder) {
        // ? 為 RestTemplate 設(shè)置獨(dú)立超時(shí)，避免拖垮整個(gè)應(yīng)用
        this.restTemplate = builder
                .setConnectTimeout(Duration.ofSeconds(5))   // 連接超時(shí)
                .setReadTimeout(Duration.ofSeconds(30))     // 讀取超時(shí)（必須 < Nginx proxy_read_timeout）
                .build();
    }
    public Report generateReport(ReportRequest request) {
        try {
            // ? 使用 CompletableFuture 實(shí)現(xiàn)異步調(diào)用 + 超時(shí)控制
            return CompletableFuture.supplyAsync(() -> {
                try {
                    // 調(diào)用外部報(bào)表服務(wù)
                    return restTemplate.postForObject(
                            "https://report-service/v1/generate",
                            request,
                            Report.class);
                } catch (ResourceAccessException ex) {
                    throw new ReportGenerationException("External report service unavailable", ex);
                }
            }, Executors.newFixedThreadPool(5))
            .orTimeout(45, TimeUnit.SECONDS) // ? 整體超時(shí) 45s < Nginx 60s
            .exceptionally(throwable -> {
                if (throwable instanceof TimeoutException) {
                    throw new ReportGenerationException("Report generation timeout after 45s");
                }
                throw new RuntimeException(throwable);
            })
            .join();
        } catch (CompletionException e) {
            throw new ReportGenerationException("Failed to generate report", e.getCause());
        }
    }
}

# Dockerfile
FROM openjdk:17-jdk-slim

# ? 關(guān)鍵：使用 JVM 容器感知參數(shù)（JDK 10+ 原生支持）
#      -XX:+UseContainerSupport 自動(dòng)識(shí)別容器內(nèi)存限制
#      -XX:MaxRAMPercentage=75.0 將最大堆設(shè)為容器內(nèi)存的 75%
ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+UseG1GC"

COPY app.jar /app.jar
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app.jar"]

# Kubernetes deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: java-app
spec:
  template:
    spec:
      containers:
      - name: app
        image: my-java-app:1.0
        resources:
          requests:
            memory: "1536Mi"   # ? 必須 ≥ JVM MaxRAMPercentage 計(jì)算出的堆上限
            cpu: "500m"
          limits:
            memory: "2Gi"       # ? 容器內(nèi)存上限，JVM 以此為基準(zhǔn)
            cpu: "1000m"
        env:
        - name: JAVA_OPTS
          value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+UseG1GC"

@Component
public class GcWatcher {
    private static final Logger log = LoggerFactory.getLogger(GcWatcher.class);
    @EventListener
    public void handleGcEvent(GarbageCollectionEvent event) {
        if ("G1 Old Generation".equals(event.getGarbageCollectorName()) &&
                event.getDuration() > 2000 && // GC 耗時(shí) >2s
                System.currentTimeMillis() - event.getStartTime() < 60_000) { // 近 1 分鐘內(nèi)
            long gcCount = ManagementFactory.getGarbageCollectorMXBean(
                    "G1 Old Generation").getCollectionCount();
            // ? 連續(xù) 3 次長(zhǎng)時(shí)間 GC，觸發(fā)自愈
            if (gcCount >= 3) {
                log.error("Critical: 3+ long GC detected in 60s. Triggering graceful shutdown.");
                Runtime.getRuntime().addShutdownHook(new Thread(() -> {
                    try {
                        // 等待正在處理的請(qǐng)求完成（Spring Boot Actuator 提供）
                        Thread.sleep(30_000);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    }
                }));
                System.exit(143); // SIGTERM
            }
        }
    }
}

# 在 Nginx 配置中注入指標(biāo)
log_format metrics '$remote_addr - $remote_user [$time_local] '
                    '"$request" $status $body_bytes_sent '
                    '$request_time $upstream_response_time '
                    '$upstream_addr $upstream_status';

# 通過(guò) Logstash 或 Filebeat 將 metrics 日志發(fā)送至 Elasticsearch
# Kibana 中創(chuàng)建告警：當(dāng) 5xx 率 >5% 且持續(xù) 2 分鐘 → 觸發(fā) K8s API 擴(kuò)容