解决紫金接口请求报错412问题

tenant/insurance/quotation-summary/src/main/resources/application-dev.yaml

@@ -227,7 +227,9 @@ zijin:
 
   crawler:
     baseUrl: https://kscd.zking.com
-    cookie-path: https://test.baoxianzhanggui.com/ck-api/ck
+    cookie-path: http://39.98.41.104:8888/zjbx/ck
+    # Python外放接口的Cookie更新地址，生成ck的JS算法有时效性，需主动调用此接口刷新Cookie
+    update-ck-path: http://39.98.41.104:8888/zjbx/update
 
 #    # 为接入方提供获取token服务
 #    accessTokenUrl: https://open.zking.com/api/v1/public/token/getAccessToken

tenant/insurance/quotation-zijin/src/main/java/com/jzg/quotation/zijin/crawler/component/ZiJinCrawlerRequestComponent.java

@@ -9,7 +9,6 @@ import com.jzg.quotation.zijin.crawler.entity.response.*;
 import com.jzg.quotation.zijin.crawler.enums.RequestUrl;
 import com.jzg.quotation.zijin.crawler.properties.ZiJinCrawlerProperties;
 import com.jzg.quotation.zijin.crawler.util.DESEncryption;
-import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.jetbrains.annotations.NotNull;
@@ -49,6 +48,16 @@ public class ZiJinCrawlerRequestComponent {
     // 用于调用Python外放接口获取Cookie
     private final RestTemplate restTemplate = new RestTemplate();
 
+
+    /**
+     * Cookie缓存过期时间戳（毫秒），超过此时间需重新调用Python更新接口刷新Cookie
+     */
+    private volatile long cookieExpireTime;
+    /**
+     * Cookie缓存有效期：10分钟（生成ck的JS算法具有时效性，超过10分钟需主动调用更新接口刷新）
+     */
+    private static final long COOKIE_TTL_MS = 10 * 60 * 1000;
+
     /**
      * 获取验证码
      *
@@ -116,7 +125,6 @@ public class ZiJinCrawlerRequestComponent {
     public ZiJinCrawlerCheckCaptchaResponse checkCaptchaAndGetToken(ZiJinCrawlerCheckCaptchaRequest ziJinCrawlerCheckCaptchaRequest, String token) throws IOException {
         log.info("ziJinCrawlerCheckCaptchaRequest请求参数的值：{}", JSON.toJSONString(ziJinCrawlerCheckCaptchaRequest));
         HttpHeaders httpHeaders = setHttpHeaders();
-        // 注意：不要手动设置Content-Length，由底层WebClient根据body自动计算，否则会被WAF判定为异常请求(412)
         // 需要单独设置这个
         httpHeaders.set("Authorization", token);
         httpHeaders.set("loginToken", token);
@@ -514,40 +522,57 @@ public class ZiJinCrawlerRequestComponent {
 
 
     /**
-     * Python外放接口返回的Cookie信息
-     * 用于动态设置请求头中的Cookie、x-csrf-token、User-Agent等，避免硬编码导致WAF校验失败
+     * 默认User-Agent，与Python端curl-cffi的impersonate=chrome131保持一致
      */
-    @Data
-    public static class CookieInfo {
-        /** 完整的Cookie字符串，包含WAF挑战Cookie和csrfToken */
-        private String cookie;
-        /** CSRF Token值，用于x-csrf-token请求头 */
-        private String csrfToken;
-        /** User-Agent字符串，需与获取Cookie时Python使用的UA保持一致以通过WAF指纹校验 */
-        private String ua;
+    private static final String DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36";
+
+    /**
+     * 调用Python外放接口更新Cookie
+     * 由于生成ck的JS算法具有时效性，需主动调用此接口让Python重新执行JS挑战刷新Cookie
+     *
+     * @throws IOException 如果调用更新接口时发生错误
+     */
+    public void updateCookie() throws IOException {
+        String updateUrl = ziJinCrawlerProperties.getUpdateCkPath();
+        if (updateUrl == null || updateUrl.trim().isEmpty()) {
+            throw new IOException("紫金crawler.update-ck-path未配置");
+        }
+        try {
+            log.info("调用Python更新Cookie接口: {}", updateUrl);
+            ResponseEntity<String> response = restTemplate.getForEntity(updateUrl, String.class);
+            String body = response.getBody();
+            log.info("更新Cookie接口响应: {}", body);
+        } catch (Exception e) {
+            log.error("调用Python更新Cookie接口失败: {}", e.getMessage(), e);
+            throw new IOException("更新Cookie失败: " + e.getMessage(), e);
+        }
     }
 
     /**
-     * 调用Python外放接口获取Cookie及关联信息
-     * 返回包含cookie、csrfToken、ua的完整信息对象，用于动态构建请求头
+     * 获取Cookie字符串（带缓存）
+     * <p>
+     * 由于生成ck的JS算法具有时效性，Cookie缓存10分钟后自动过期。
+     * 过期时会先调用Python更新接口刷新Cookie，再重新获取。
      * <p>
      * Python接口返回示例：
      * <pre>{@code
      * {
-     *     "ok": true,
-     *     "cookie": "c7IBxtKJpAH2O=...; c7IBxtKJpAH2P=...; csrfToken=uZvxgMXq-TPNbLU8jJhSOpxmaynt9YmmuxH0",
-     *     "pLen": 343,
-     *     "csrfToken": "uZvxgMXq-TPNbLU8jJhSOpxmaynt9YmmuxH0",
-     *     "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-     *     "impersonate": "chrome131",
-     *     "elapsed_ms": 3801
+     *     "c7IBxtKJpAH2P": "aqEdSy_6gjpocI0...",
+     *     "c7IBxtKJpAH2O": "qaAIGG3xrOsNPZx..."
      * }
      * }</pre>
      *
-     * @return CookieInfo Cookie信息对象
+     * @return String Cookie字符串，格式: "key1=value1; key2=value2"
      * @throws IOException 如果获取Cookie时发生错误
      */
-    public CookieInfo getCookieInfo() throws IOException {
+    public String getCookie() throws IOException {
+        // 检查缓存是否有效
+        long now = System.currentTimeMillis();
+        if (now < cookieExpireTime) {
+            log.info("紫金Cookie JS算法剩余有效时间: {}ms", (cookieExpireTime - now));
+            updateCookie();
+        }
+        // 调用Python接口获取刷新后的Cookie
         String cookieUrl = ziJinCrawlerProperties.getCookiePath();
         if (cookieUrl == null || cookieUrl.trim().isEmpty()) {
             throw new IOException("紫金crawler.cookie-path未配置");
@@ -559,23 +584,25 @@ public class ZiJinCrawlerRequestComponent {
                 throw new IOException("获取Cookie失败: Python接口返回空内容");
             }
             log.info("获取Cookie响应：{}", body);
-            // 解析JSON响应，提取cookie、csrfToken、ua字段
+            // Python返回Cookie键值对Map，遍历拼接为标准Cookie格式
             com.alibaba.fastjson2.JSONObject jsonObject = JSON.parseObject(body);
-            if (!jsonObject.getBooleanValue("ok")) {
-                throw new IOException("获取Cookie失败: Python接口返回ok=false");
-            }
-            CookieInfo cookieInfo = new CookieInfo();
-            cookieInfo.setCookie(jsonObject.getString("cookie"));
-            cookieInfo.setCsrfToken(jsonObject.getString("csrfToken"));
-            cookieInfo.setUa(jsonObject.getString("ua"));
-            if (cookieInfo.getCookie() == null || cookieInfo.getCookie().trim().isEmpty()) {
-                throw new IOException("获取Cookie失败: 响应中cookie字段为空");
+            StringBuilder cookieBuilder = new StringBuilder();
+            for (String key : jsonObject.keySet()) {
+                String value = jsonObject.getString(key);
+                if (value != null && !value.trim().isEmpty()) {
+                    if (cookieBuilder.length() > 0) {
+                        cookieBuilder.append("; ");
+                    }
+                    cookieBuilder.append(key).append("=").append(value);
+                }
             }
-            if (cookieInfo.getCsrfToken() == null || cookieInfo.getCsrfToken().trim().isEmpty()) {
-                throw new IOException("获取Cookie失败: 响应中csrfToken字段为空");
+            String cookie = cookieBuilder.toString();
+            if (cookie.trim().isEmpty()) {
+                throw new IOException("获取Cookie失败: 拼接的Cookie字符串为空");
             }
-            log.info("紫金Cookie获取成功，耗时: {}ms", jsonObject.getString("elapsed_ms"));
-            return cookieInfo;
+            cookieExpireTime = System.currentTimeMillis() + COOKIE_TTL_MS;
+            log.info("紫金Cookie获取并缓存成功，有效期: {}分钟", COOKIE_TTL_MS / 60000);
+            return cookie;
         } catch (IOException e) {
             throw e;
         } catch (Exception e) {
@@ -584,17 +611,6 @@ public class ZiJinCrawlerRequestComponent {
         }
     }
 
-    /**
-     * 获取Cookie字符串
-     * 通过调用Python外放接口获取Cookie，返回完整的Cookie字符串（包含WAF挑战Cookie和csrfToken）
-     *
-     * @return String Cookie字符串
-     * @throws IOException 如果获取Cookie时发生错误
-     */
-    public String getCookie() throws IOException {
-        return getCookieInfo().getCookie();
-    }
-
     /**
      * 从User-Agent字符串中解析Chrome主版本号
      * 例如: "Mozilla/5.0 ... Chrome/131.0.0.0 Safari/537.36" → "131"
@@ -637,10 +653,9 @@ public class ZiJinCrawlerRequestComponent {
     @NotNull
     private HttpHeaders setHttpHeaders(String token) throws IOException {
         HttpHeaders headers = new HttpHeaders();
-        // 动态获取Cookie信息（包含WAF挑战Cookie、csrfToken、ua），避免硬编码导致WAF校验失败
-        CookieInfo cookieInfo = getCookieInfo();
-        // Cookie字符串已包含csrfToken，直接使用Python返回的完整Cookie
-        headers.set("Cookie", cookieInfo.getCookie());
+        // 通过Python外放接口获取WAF挑战Cookie，拼接为标准Cookie格式
+        String cookie = getCookie();
+        headers.set("Cookie", "csrfToken=yKKI5-FlPLoNFbMVKORknSef;" + cookie);
         headers.set("Accept", "application/json, text/plain, */*, application/problem+json");
         // headers.set("Accept-Encoding", "gzip, deflate, br, zstd");
         headers.set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
@@ -655,19 +670,14 @@ public class ZiJinCrawlerRequestComponent {
         headers.set("Sec-Fetch-Mode", "cors");
         headers.set("Sec-Fetch-Site", "same-origin");
         headers.set("Tokentype", "DEFULT");
-        // User-Agent需与Python获取Cookie时使用的UA保持一致，否则WAF指纹校验会失败
-        String ua = cookieInfo.getUa();
-        if (ua == null || ua.trim().isEmpty()) {
-            ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
-        }
-        headers.set("User-Agent", ua);
+        headers.set("User-Agent", DEFAULT_UA);
         // 从UA中解析Chrome版本号，构造与UA一致的sec-ch-ua头，避免版本号不匹配
-        String chromeVersion = extractChromeVersion(ua);
+        String chromeVersion = extractChromeVersion(DEFAULT_UA);
         headers.set("sec-ch-ua", "\"Google Chrome\";v=\"" + chromeVersion + "\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"" + chromeVersion + "\"");
         headers.set("sec-ch-ua-mobile", "?0");
         headers.set("sec-ch-ua-platform", "\"Windows\"");
         // x-csrf-token使用Python返回的动态值，不再硬编码
-        headers.set("x-csrf-token", cookieInfo.getCsrfToken());
+        headers.set("x-csrf-token", "yKKI5-FlPLoNFbMVKORknSef");
         headers.set("content-type", "application/json");
         if (StringUtils.hasText(token)) {
             headers.set("Authorization", token);

tenant/insurance/quotation-zijin/src/main/java/com/jzg/quotation/zijin/crawler/properties/ZiJinCrawlerProperties.java

@@ -21,4 +21,10 @@ public class ZiJinCrawlerProperties {
      */
     private String cookiePath;
 
+    /**
+     * Python外放接口的Cookie更新地址
+     * 由于生成ck的JS算法有时效性，需主动调用此接口让Python重新执行JS挑战刷新Cookie
+     */
+    private String updateCkPath;
+
 }