|
|
@@ -12,20 +12,25 @@ import com.jzg.quotation.huanong.crawler.properties.HuaNongCrawlerProperties;
|
|
|
import com.jzg.quotation.huanong.crawler.utils.RequestEncryptor;
|
|
|
import lombok.RequiredArgsConstructor;
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
+import org.springframework.core.ParameterizedTypeReference;
|
|
|
import org.springframework.core.io.FileSystemResource;
|
|
|
+import org.springframework.http.HttpEntity;
|
|
|
import org.springframework.http.HttpHeaders;
|
|
|
import org.springframework.http.ResponseEntity;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
import org.springframework.util.LinkedMultiValueMap;
|
|
|
import org.springframework.util.MultiValueMap;
|
|
|
import org.springframework.util.StringUtils;
|
|
|
+import org.springframework.web.reactive.function.BodyInserters;
|
|
|
|
|
|
-import java.io.File;
|
|
|
import java.io.IOException;
|
|
|
import java.net.URI;
|
|
|
import java.nio.file.Files;
|
|
|
import java.nio.file.Path;
|
|
|
-import java.util.*;
|
|
|
+import java.util.Collections;
|
|
|
+import java.util.HashMap;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.Optional;
|
|
|
|
|
|
import static com.jzg.quotation.huanong.crawler.constant.enums.RequestUrl.*;
|
|
|
|
|
|
@@ -192,30 +197,64 @@ public class HuaNongCrawlerRequestComponent {
|
|
|
return response.getData();
|
|
|
}
|
|
|
|
|
|
+ private HttpHeaders getBizUploadHttpHeaders() {
|
|
|
+ HttpHeaders headers = new HttpHeaders();
|
|
|
+ headers.set("Host", "nile.chinahuanong.com.cn");
|
|
|
+ headers.set("Connection", "keep-alive");
|
|
|
+ headers.set("Cache-Control", "max-age=0");
|
|
|
+ headers.set("sec-ch-ua", "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"138\", \"Google Chrome\";v=\"138\"");
|
|
|
+ headers.set("sec-ch-ua-mobile", "?0");
|
|
|
+ headers.set("sec-ch-ua-platform", "\"Windows\"");
|
|
|
+ headers.set("Upgrade-Insecure-Requests", "1");
|
|
|
+ headers.set("Origin", "https://cx.chinahuanong.com.cn");
|
|
|
+ headers.set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
|
|
|
+ headers.set("Sec-Fetch-Site", "same-site");
|
|
|
+ headers.set("Sec-Fetch-Mode", "navigate");
|
|
|
+ headers.set("Sec-Fetch-User", "?1");
|
|
|
+ headers.set("Sec-Fetch-Dest", "document");
|
|
|
+ headers.set("Referer", "https://cx.chinahuanong.com.cn/");
|
|
|
+ headers.set("Accept-Encoding", "gzip, deflate, br, zstd");
|
|
|
+ headers.set("Accept-Language", "zh-CN,zh;q=0.9");
|
|
|
+ return headers;
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 获取 accessToken
|
|
|
*
|
|
|
- * @param data data数据
|
|
|
- * @param headers 请求头
|
|
|
+ * @param data data数据
|
|
|
* @return accessToken
|
|
|
*/
|
|
|
- public String bizUpload(String data, HttpHeaders headers) {
|
|
|
- MultiValueMap<String, Object> parts = new LinkedMultiValueMap<>();
|
|
|
- parts.add("data", data);
|
|
|
+ public String bizUpload(String data, HttpHeaders httpHeaders) {
|
|
|
String url = properties.getUploadUrl() + RequestUrl.BIZ_UPLOAD.getUri();
|
|
|
- ResponseEntity<String> response = requestComponent.formData(url, parts, String.class, headers);
|
|
|
+ BodyInserters.FormInserter<String> fromFormData = BodyInserters.fromFormData("data", data);
|
|
|
+ ResponseEntity<String> response = requestComponent.formUrlencoded(url, fromFormData, String.class, getBizUploadHttpHeaders());
|
|
|
+
|
|
|
String fragment = Optional.ofNullable(response)
|
|
|
.map(ResponseEntity::getHeaders)
|
|
|
.map(HttpHeaders::getLocation)
|
|
|
.map(URI::getFragment)
|
|
|
.orElse("");
|
|
|
|
|
|
+ Optional.ofNullable(response)
|
|
|
+ .map(HttpEntity::getHeaders)
|
|
|
+ .map(x -> x.get("Set-Cookie"))
|
|
|
+ .ifPresent(x -> {
|
|
|
+ httpHeaders.set("Cookie", String.join(";", x));
|
|
|
+ });
|
|
|
+
|
|
|
// 解析片段中的查询参数
|
|
|
String[] s = fragment.split("\\?");
|
|
|
+
|
|
|
if (s.length > 1) {
|
|
|
Map<String, String> params = parseQueryParams(s[1]);
|
|
|
- return params.get("accessToken");
|
|
|
+ String accessToken = params.get("accessToken");
|
|
|
+ if (fragment.contains("/error")) {
|
|
|
+ unlock(accessToken, httpHeaders);
|
|
|
+ return "ERROR";
|
|
|
+ }
|
|
|
+ return accessToken;
|
|
|
}
|
|
|
+
|
|
|
throw new SystemException("");
|
|
|
}
|
|
|
|
|
|
@@ -236,19 +275,31 @@ public class HuaNongCrawlerRequestComponent {
|
|
|
* @param headers 请求头
|
|
|
* @return 上传影像响应
|
|
|
*/
|
|
|
- public HuaNongCrawlerFileUploadResponse fileUpload(String accessToken, String uuidFileName, String suffix, byte[] imageByte, HttpHeaders headers){
|
|
|
+ public HuaNongCrawlerFileUploadResponse fileUpload(String accessToken, String uuidFileName, String suffix, byte[] imageByte, HttpHeaders headers) {
|
|
|
try {
|
|
|
MultiValueMap<String, Object> parts = new LinkedMultiValueMap<>();
|
|
|
Path tempPath = Files.createTempFile(uuidFileName, suffix);
|
|
|
Files.write(tempPath, imageByte);
|
|
|
parts.add("file", new FileSystemResource(tempPath.toFile()));
|
|
|
- String url = properties.getUploadUrl() + String.format(FILE_UPLOAD.getUri(), accessToken, "");
|
|
|
+ String url = properties.getUploadUrl() + String.format(FILE_UPLOAD.getUri(), accessToken, uuidFileName);
|
|
|
ResponseEntity<HuaNongCrawlerFileUploadResponse> responseEntity = requestComponent.formData(url, parts, HuaNongCrawlerFileUploadResponse.class, headers);
|
|
|
return responseEntity.getBody();
|
|
|
- }catch (IOException e){
|
|
|
+ } catch (IOException e) {
|
|
|
throw new SystemException("文件读取异常");
|
|
|
}
|
|
|
}
|
|
|
+ public HuaNongCrawlerBizQueryResponse bizQuery(String accessToken, HttpHeaders headers) {
|
|
|
+ String url = properties.getUploadUrl() + String.format(BIZ_QUERY.getUri(), accessToken);
|
|
|
+ ResponseEntity<HuaNongCrawlerBizQueryResponse> responseEntity = requestComponent.baseGet(url, new LinkedMultiValueMap<>(), HuaNongCrawlerBizQueryResponse.class, headers);
|
|
|
+ return responseEntity.getBody();
|
|
|
+ }
|
|
|
+
|
|
|
+ public void unlock(String accessToken, HttpHeaders headers) {
|
|
|
+ String url = properties.getUploadUrl() + String.format(UNLOCK.getUri(), accessToken);
|
|
|
+ String o = requestComponent.get(url, Map.of(), UNLOCK.getRequestDescription(), new ParameterizedTypeReference<>() {
|
|
|
+ }, headers);
|
|
|
+ System.out.println(o);
|
|
|
+ }
|
|
|
|
|
|
/**
|
|
|
* 提交影像
|
|
|
@@ -259,9 +310,12 @@ public class HuaNongCrawlerRequestComponent {
|
|
|
*/
|
|
|
public void fileSubmit(HuaNongCrawlerFileSubmitRequest request, String accessToken, HttpHeaders headers) {
|
|
|
String url = properties.getUploadUrl() + String.format(FILE_SUBMIT.getUri(), accessToken, System.currentTimeMillis());
|
|
|
- ResponseEntity<HuaNongCrawlerFileUploadResponse> responseEntity = requestComponent.objectPost(url, request, HuaNongCrawlerFileUploadResponse.class, headers);
|
|
|
+ ResponseEntity<HuaNongCrawlerFileSubmitResponse> responseEntity = requestComponent.objectPost(url, request, HuaNongCrawlerFileSubmitResponse.class, headers);
|
|
|
+ System.out.println();
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+
|
|
|
/**
|
|
|
* 查询上传目录
|
|
|
*
|