Doghole/emo-grab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

获取指标说明的能力

Browse Source
This commit is contained in:
Doghole
2025-05-21 15:51:44 +08:00
parent 06c351a956
commit 1f329e3b2a
131 changed files with 4461 additions and 3126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/quant/rich/emoney/EmoneyAutoApplication.java
View File

@@ -2,12 +2,14 @@ package quant.rich.emoney;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableAsync
@EnableScheduling
@SpringBootApplication
@EnableCaching(proxyTargetClass=true)
public class EmoneyAutoApplication {
public static void main(String[] args) {

127
src/main/java/quant/rich/emoney/client/EmoneyClient.java
View File

@@ -26,15 +26,27 @@ import quant.rich.emoney.util.EncryptUtils;
import quant.rich.emoney.util.SpringContextHolder;
import okhttp3.OkHttpClient;
/**
* 益盟操盘手基本请求客户端，提供基本功能
* <p><b>请求头顺序</b></p>
* <p>
* X-Protocol-Id > X-Request-Id > EM-Sign > Authorization >
* X-Android-Agent > Emapp-ViewMode > Content-Type > Content-Length >
* Host > Connection: "Keep-Alive" > Accept-Encoding: "gzip" > User-Agent</p>
* <p>从 X-Protocol-Id 到 Emapp-ViewMode，由本例添加，剩余为 okhttp 默认添加，
* User-Agent 由 ByteBuddy 重写 header 方法控制添加</p>
* @see quant.rich.emoney.patch.okhttp.PatchOkHttp
*/
@Data
@Slf4j
@Accessors(chain = true)
public class EmoneyClient implements Cloneable {
private static final String MBS_URL = "https://mbs.emoney.cn/";
//private static final String LOGIN_URL = "https://emapp.emoney.cn/user/auth/login";
private static final String LOGIN_URL = "http://localhost:7790/user/auth/login";
private static final String LOGIN_URL = "https://emapp.emoney.cn/user/auth/login";
private static final String RELOGIN_URL = "https://emapp.emoney.cn/user/auth/ReLogin";
private static final String LOGIN_X_PROTOCOL_ID = "user%2Fauth%2Flogin";
private static final String RELOGIN_X_PROTOCOL_ID = "user%2Fauth%2FReLogin";
private static volatile EmoneyRequestConfig emoneyRequestConfig;
@@ -68,8 +80,10 @@ public class EmoneyClient implements Cloneable {
/**
* 使用系统管理的用户名密码登录
* <p>建议仅在调试时使用，其他情况请用 {@code loginWithManaged()}</p>
* @return
*/
@Deprecated
public static Boolean loginWithUsernamePassword() {
ObjectNode formObject = getEmoneyRequestConfig().getUsernamePasswordLoginObject();
return login(formObject);
@@ -77,10 +91,12 @@ public class EmoneyClient implements Cloneable {
/**
* 使用给定的用户名密码登录
* <p>建议仅在调试时使用，其他情况请用 {@code loginWithManaged()}</p>
* @param username 用户名
* @param password 密码，可以是明文，也可是密文
* @return
*/
@Deprecated
public static Boolean loginWithUsernamePassword(String username, String password) {
ObjectNode formObject = getEmoneyRequestConfig().getUsernamePasswordLoginObject(username, password);
return login(formObject);
@@ -88,13 +104,80 @@ public class EmoneyClient implements Cloneable {
/**
* 匿名登录
* <p>建议仅在调试时使用，其他情况请用 {@code loginWithManaged()}</p>
* @return
*/
@Deprecated
public static Boolean loginWithAnonymous() {
ObjectNode formObject = getEmoneyRequestConfig().getAnonymousLoginObject();
return login(formObject);
}
/**
* 触发重登陆验证
* @return
*/
public static Boolean relogin() {
EmoneyRequestConfig emoneyRequestConfig = getEmoneyRequestConfig();
ObjectNode reloginObject = emoneyRequestConfig.getReloginObject();
if (reloginObject == null) {
// 无登录信息，直接触发登录
return loginWithManaged();
}
String token = reloginObject.get("token").asText();
try {
OkHttpClient okHttpClient = OkHttpClientProvider.getInstance();
MediaType type = MediaType.parse("application/json");
byte[] content = reloginObject.toString().getBytes("utf-8");
RequestBody body = RequestBody.create(
content, type);
Request.Builder requestBuilder = new Request.Builder()
.url(RELOGIN_URL)
.post(body)
.header("X-Protocol-Id", RELOGIN_X_PROTOCOL_ID)
.header("X-Request-Id", "1")
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", RELOGIN_X_PROTOCOL_ID))
.header("Authorization", token)
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode());
Request request = requestBuilder.build();
final Call call = okHttpClient.newCall(request);
Response response = call.execute();
if (response.code() != 200) {
// 不是 200，重新登录
log.debug("ReLogin 重登录验证返回状态码 {}, 触发登录", response.code());
return loginWithManaged();
}
String responseText = response.body().string();
ObjectNode loginResult = (ObjectNode) new ObjectMapper().readTree(responseText);
Integer code = loginResult.get("result").get("code").asInt();
if (code != 0 ||
!token.equals(loginResult.get("detail").get("token").asText())) {
// 我也不知道会不会进入这里，不过尽可能写完所有可能
log.debug("ReLogin 返回消息 result.code = {}, token 与已有 authorization {}一致, 触发重登陆",
code, code == 0 ? "不" : "");
return loginWithManaged();
}
else {
log.debug("ReLogin 校验通过，不需要重登录");
return true;
}
}
catch (JsonProcessingException e) {
throw new EmoneyDecodeException("ReLogin 试图将返回数据解析成 JSON 时失败", e);
}
catch (Exception e) {
throw new EmoneyRequestException("执行 emoney ReLogin 请求/返回时出现错误", e);
}
}
/**
* 登录总方法，唯一要控制的只有 formObject
* @param formObject
@@ -103,7 +186,7 @@ public class EmoneyClient implements Cloneable {
private static Boolean login(ObjectNode formObject) {
try {
//OkHttpClient okHttpClient = new OkHttpClient();
EmoneyRequestConfig emoneyRequestConfig = getEmoneyRequestConfig();
OkHttpClient okHttpClient = OkHttpClientProvider.getInstance();
MediaType type = MediaType.parse("application/json");
//type.charset(StandardCharsets.UTF_8);
@@ -123,31 +206,40 @@ public class EmoneyClient implements Cloneable {
.header("X-Request-Id", "null")
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", LOGIN_X_PROTOCOL_ID))
.header("Authorization", "")
.header("X-Android-Agent", getEmoneyRequestConfig().getXAndroidAgent())
.header("Emapp-ViewMode", getEmoneyRequestConfig().getEmappViewMode());
//.header("User-Agent", getEmoneyRequestConfig().getOkHttpUserAgent())
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode());
//.header("User-Agent", emoneyRequestConfig.getOkHttpUserAgent())
Request request = requestBuilder.build();
final Call call = okHttpClient.newCall(request);
Response response = call.execute();
ObjectNode loginResult = (ObjectNode) new ObjectMapper().readTree(response.body().string());
String responseText = response.body().string();
ObjectNode loginResult = (ObjectNode) new ObjectMapper().readTree(responseText);
Integer code = loginResult.get("result").get("code").asInt();
if (code == 0) {
getEmoneyRequestConfig().setAuthorization(
loginResult.get("detail").get("token").asText())
.updateAuthorizationTime().saveOrUpdate();
emoneyRequestConfig
.setAuthorization(
loginResult
.get("detail").get("token").asText())
.setUid(
loginResult
.get("detail").get("uid").asInt())
.saveOrUpdate();
log.info("执行 emoney LOGIN 成功");
return true;
} else {
}
else {
String msg = loginResult.get("result").get("msg").asText();
throw new EmoneyResponseException("执行 emoney LOGIN 请求返回错误，code: " + code + ", msg: " + msg, code);
}
} catch (JsonProcessingException e) {
}
catch (JsonProcessingException e) {
throw new EmoneyDecodeException("试图将返回数据解析成 JSON 时失败", e);
} catch (Exception e) {
}
catch (Exception e) {
throw new EmoneyRequestException("执行 emoney LOGIN 请求/返回时出现错误", e);
}
}
@@ -168,7 +260,8 @@ public class EmoneyClient implements Cloneable {
throw new EmoneyIllegalRequestParamException("执行 emoney 请求错误，Protocol id 不能为 null!",
new IllegalArgumentException());
}
if (StringUtils.isBlank(getEmoneyRequestConfig().getAuthorization())) {
EmoneyRequestConfig emoneyRequestConfig = getEmoneyRequestConfig();
if (StringUtils.isBlank(emoneyRequestConfig.getAuthorization())) {
throw new EmoneyIllegalRequestParamException("执行 emoney 请求错误，Authorization 为空，是否未登录？",
new IllegalArgumentException());
}
@@ -190,9 +283,9 @@ public class EmoneyClient implements Cloneable {
.header("X-Protocol-Id", xProtocolId.toString())
.header("X-Request-Id", xRequestId == null ? "null" : xRequestId.toString())
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", xProtocolId.toString()))
.header("Authorization", getEmoneyRequestConfig().getAuthorization())
.header("X-Android-Agent", getEmoneyRequestConfig().getXAndroidAgent())
.header("Emapp-ViewMode", getEmoneyRequestConfig().getEmappViewMode())
.header("Authorization", emoneyRequestConfig.getAuthorization())
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode())
;
Request request = requestBuilder.build();

6
src/main/java/quant/rich/emoney/component/LockByCaller.java
View File

@@ -14,7 +14,11 @@ import java.lang.annotation.*;
public @interface LockByCaller {
/**
* 可选参数，用于 SpEL 表达式获取 key
* 例如：@LockByCaller(key = "#userId")
* 例如：<ul>
* <li>@LockByCaller(key = "#userId")</li>
* <li>@LockByCaller(key = "#userId + ':' + #userName")</li>
* </ul>
* 当不指定时，不校验参数，单纯校验 Caller
*/
String key() default "";
}

28
src/main/java/quant/rich/emoney/controller/manage/IndexInfoControllerV1.java
View File

@@ -18,6 +18,7 @@ import quant.rich.emoney.controller.common.BaseController;
import quant.rich.emoney.entity.config.IndexInfoConfig;
import quant.rich.emoney.pojo.dto.LayPageResp;
import quant.rich.emoney.pojo.dto.R;
import quant.rich.emoney.service.IndexDetailService;
@Controller
@RequestMapping("/admin/v1/manage/indexInfo")
@@ -25,11 +26,30 @@ public class IndexInfoControllerV1 extends BaseController {
@Autowired
IndexInfoConfig indexInfo;
@Autowired
IndexDetailService indexDetailService;
@GetMapping({"", "/", "/index"})
public String index() {
return "/admin/v1/manage/indexInfo/index";
}
@GetMapping("/getIndexDetail")
@ResponseBody
public R<?> getIndexDetail(String indexCode) {
return
R.judge(() ->
indexDetailService.getIndexDetail(indexCode));
}
@GetMapping("/forceRefreshAndGetIndexDetail")
@ResponseBody
public R<?> forceRefreshAndGetIndexDetail(String indexCode) {
return
R.judge(() ->
indexDetailService.forceRefreshAndGetIndexDetail(indexCode));
}
@GetMapping("/configIndOnline")
@ResponseBody
@@ -53,17 +73,11 @@ public class IndexInfoControllerV1 extends BaseController {
return R.ok(map);
}
@GetMapping("/getIndexDetail")
@ResponseBody
public R<?> getIndexDetail(String code) {
return null;
}
@GetMapping("/getConfigIndOnlineByUrl")
@ResponseBody
public R<?> getConfigOnlineByUrl(String url) {
return R.judge(() -> indexInfo.getOnlineConfigByUrl(url));
return R.judge(() -> indexInfo.getOnlineConfigByUrl());
}
@GetMapping("/getIndexInfoConfig")

100
src/main/java/quant/rich/emoney/entity/config/EmoneyRequestConfig.java
View File

@@ -1,6 +1,6 @@
package quant.rich.emoney.entity.config;
import java.time.LocalDateTime;
import java.io.Serializable;
import java.util.Objects;
import org.apache.commons.lang3.ObjectUtils;
@@ -10,12 +10,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
import lombok.AccessLevel;
import lombok.Data;
import lombok.Getter;
@@ -67,11 +63,9 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
private String authorization;
/**
* 当前 authorization 更新时间
* UID
*/
@JsonSerialize(using = LocalDateTimeSerializer.class)
@JsonDeserialize(using = LocalDateTimeDeserializer.class)
private LocalDateTime authorizationUpdateTime;
private Integer uid;
/**
* <b>用于：</b><ul>
@@ -186,6 +180,12 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
* <b>来源：</b>本例管理
*/
private String emappViewMode = "1";
/**
* OkHttp 用于注入 User-Agent 规则的 id
*/
@JsonIgnore
private Serializable userAgentPatchRuleId;
@Getter(AccessLevel.PRIVATE)
@Autowired
@@ -215,7 +215,7 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
initFromRandomDeviceInfo();
}
else {
// 都不是 null，则由 fingerprint 来检查各项
// 都不是 null，则由 fingerprint 来检查各项。
// model 和 softwareType 本应交由 deviceInfoConfig 检查以
// 应对可能的通过修改本地 json 来进行攻击的方式，可是本身
// deviceInfoConfig 对 model 和 softwareType 的信息也来源
@@ -248,12 +248,22 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
}
// 注入 OkHttp
PatchOkHttp.apply(
patchOkHttp();
}
/**
* 注入 User-Agent patch 规则
*/
private EmoneyRequestConfig patchOkHttp() {
userAgentPatchRuleId = PatchOkHttp.apply(
PatchOkHttpRule.when()
.hostEndsWith("emoney.cn")
.not(r -> r.hostMatches("appstatic"))
.or(a -> a.hostContains("emapp"))
.or(b -> b.hasHeaderName("X-Protocol-Id"))
.overrideIf("User-Agent", getOkHttpUserAgent()).build());
.overrideIf("User-Agent", getOkHttpUserAgent()).build()
.setId(userAgentPatchRuleId));
return this;
}
/**
@@ -451,6 +461,45 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
return node;
}
/**
* 根据本例信息获取 Relogin ObjectNode
* @return 如果 authorization 和 uid 任意 null 则本例返回 null
*/
@JsonIgnore
public ObjectNode getReloginObject() {
if (ObjectUtils.anyNull(getAuthorization(), getUid())) {
return null;
}
ObjectMapper mapper = new ObjectMapper();
ObjectNode node = mapper.createObjectNode();
ObjectNode exIdentify = mapper.createObjectNode();
exIdentify.put("IMEI", "");
exIdentify.put("AndroidID", getAndroidId());
exIdentify.put("MAC", "");
exIdentify.put("OSFingerPrint", getFingerprint());
String exIdentifyString = exIdentify.toString().replace("/", "\\/");
// 这玩意最好按照顺序来，当前这个顺序是 5.8.1 的顺序
String guid = getGuid();
node.put("appVersion", getEmoneyVersion());
node.put("productId", 4);
node.put("softwareType", getSoftwareType());
node.put("deviceName", getDeviceName());
node.put("ssid", "0");
node.put("platform", "android");
node.put("token", getAuthorization()); // 和登录不同的地方: token
node.put("exIdentify", exIdentifyString);
node.put("uid", getUid()); // 和登录不同的地方: uid
node.put("osVersion", getAndroidSdkLevel());
node.put("guid", guid);
node.put("channelId", "1711");
node.put("hardware", getHardware());
return node;
}
/**
* 设置密码：<ul>
* <li>null or empty，保存空字符串</li>
@@ -483,31 +532,8 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
return this;
}
/**
* 触发更新 Authorization 的更新时间为现在
* @return
*/
public EmoneyRequestConfig updateAuthorizationTime() {
this.authorizationUpdateTime = LocalDateTime.now();
public EmoneyRequestConfig afterSaving() {
patchOkHttp();
return this;
}
public EmoneyRequestConfig mergeTo(EmoneyRequestConfig other) {
boolean authorizationUpdated = !Objects.equals(other.authorization, this.authorization);
IConfig.super.mergeTo(other);
if (authorizationUpdated) {
other.updateAuthorizationTime();
}
return other;
}
public EmoneyRequestConfig mergeFrom(EmoneyRequestConfig other) {
boolean authorizationUpdated = !Objects.equals(other.authorization, this.authorization);
IConfig.super.mergeFrom(other);
if (authorizationUpdated) {
this.updateAuthorizationTime();
}
return this;
}
}

18
src/main/java/quant/rich/emoney/entity/config/IndexInfoConfig.java
View File

@@ -21,6 +21,7 @@ import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import quant.rich.emoney.client.OkHttpClientProvider;
import quant.rich.emoney.component.LockByCaller;
import quant.rich.emoney.interfaces.ConfigInfo;
import quant.rich.emoney.interfaces.IConfig;
@@ -33,8 +34,6 @@ import quant.rich.emoney.interfaces.IConfig;
@ConfigInfo(field = "indexInfo", name =" 指标信息配置", initDefault = true, managed = false)
public class IndexInfoConfig implements IConfig<IndexInfoConfig> {
private static final String CONFIG_IND_ONLINE_PATH = "./conf/extra/config_ind_online.json";
@JsonView(IConfig.Views.Persistence.class)
private String configIndOnlineUrl;
@@ -46,23 +45,18 @@ public class IndexInfoConfig implements IConfig<IndexInfoConfig> {
@Getter(AccessLevel.PRIVATE)
private EmoneyRequestConfig emoneyRequestConfig;
public IndexInfoConfig() {
try {
String configStr = FileUtil.readString(CONFIG_IND_ONLINE_PATH);
configIndOnline = new ObjectMapper().readTree(configStr);
}
catch (Exception e) {}
configIndOnlineUrl = "https://emapp-static.oss-cn-shanghai.aliyuncs.com/down13/emstock/config/ind_config/v1000003/config_ind_online.json";
}
public IndexInfoConfig() {}
public String getConfigIndOnlineStr() {
return getConfigIndOnline().toPrettyString();
}
public String getOnlineConfigByUrl(String url) throws IOException {
@LockByCaller
@JsonIgnore
public String getOnlineConfigByUrl() throws IOException {
synchronized (this) {
Request request = new Request.Builder()
.url(url)
.url(configIndOnlineUrl)
.header("Cache-Control", "no-cache")
.get()
.build();

31
src/main/java/quant/rich/emoney/entity/config/ProxyConfig.java
View File

@@ -2,6 +2,9 @@ package quant.rich.emoney.entity.config;
import java.net.InetSocketAddress;
import java.net.Proxy;
import org.apache.commons.lang3.ObjectUtils;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Data;
import lombok.experimental.Accessors;
@@ -77,18 +80,22 @@ public class ProxyConfig implements IConfig<ProxyConfig> {
return Proxy.NO_PROXY;
}
public static void main(String[] args) {
String proxyIp = "127.0.0.1";
int proxyPort = 7897;
ProxyConfig proxyConfig = new ProxyConfig();
proxyConfig.setProxyHost(proxyIp).setProxyPort(proxyPort)
.setProxyType(Proxy.Type.SOCKS)
.setIgnoreHttpsVerification(false);
IpInfo result = GeoIPUtil.getIpInfoThroughProxy(proxyConfig);
System.out.println("Proxy is usable with through-proxy ip: " + result);
public String getProxyUrl() {
if (ObjectUtils.anyNull(getProxyType(), getProxyHost(), getProxyPort())) {
return null;
}
StringBuilder sb = new StringBuilder();
if (getProxyType() == Proxy.Type.SOCKS) {
sb.append("socks5://");
}
else if (getProxyType() == Proxy.Type.HTTP) {
sb.append("http://");
}
else {
return null;
}
sb.append(getProxyHost()).append(':').append(getProxyPort());
return sb.toString();
}
}

41
src/main/java/quant/rich/emoney/patch/okhttp/PatchOkHttp.java
View File

@@ -2,19 +2,16 @@ package quant.rich.emoney.patch.okhttp;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.agent.ByteBuddyAgent;
import net.bytebuddy.agent.builder.AgentBuilder;
import net.bytebuddy.asm.Advice;
import okhttp3.Request;
import quant.rich.emoney.util.SpringContextHolder;
import net.bytebuddy.dynamic.ClassFileLocator;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.dynamic.loading.ClassReloadingStrategy;
import static net.bytebuddy.matcher.ElementMatchers.*;
import java.lang.instrument.Instrumentation;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.function.Consumer;
import org.slf4j.Logger;
@@ -23,31 +20,45 @@ import org.springframework.boot.SpringApplication;
import org.springframework.context.ConfigurableApplicationContext;
public class PatchOkHttp {
private static final Random random = new Random();
private static final Logger log = LoggerFactory.getLogger(PatchOkHttp.class);
private static final List<PatchOkHttpRule> rules = new CopyOnWriteArrayList<>();
private static final Set<PatchOkHttpRule> rules = new CopyOnWriteArraySet<>();
private static boolean logOnce = false;
private static boolean isHooked = false;
public static void apply(PatchOkHttpRule rule) {
/**
* 应用指定 Patch 规则
* @param rule
* @return 如果 rule 未设置 id，则生成随机 id 并返回，否则返回 rule.getId()
*/
public static Serializable apply(PatchOkHttpRule rule) {
if (rule.getId() == null) {
rule.setId(random.nextInt());
}
rules.add(rule);
// log.debug("apply() running in classloader {}", PatchOkHttp.class.getClassLoader());
log.debug("PatchOkHttp.apply() running in classloader {}", PatchOkHttp.class.getClassLoader());
if (!isHooked) hook();
return rule.getId();
}
public static void apply(PatchOkHttpRule.Builder builder) {
rules.add(builder.build());
if (!isHooked) hook();
public static Serializable apply(PatchOkHttpRule.Builder builder) {
return PatchOkHttp.apply(builder.build());
}
public static void apply(Consumer<PatchOkHttpRule.Builder> r) {
public static Serializable apply(Consumer<PatchOkHttpRule.Builder> r) {
PatchOkHttpRule.Builder builder = PatchOkHttpRule.when();
r.accept(builder);
rules.add(builder.build());
if (!isHooked) hook();
return PatchOkHttp.apply(builder.build());
}
public static void match(RequestContext ctx, String currentHeader, Consumer<String> consumer) {
// log.debug("match() running in classloader {}", PatchOkHttp.class.getClassLoader());
if (!logOnce) {
log.debug("PatchOkHttp.match() running in classloader {}", PatchOkHttp.class.getClassLoader());
logOnce = true;
}
for (PatchOkHttpRule rule : PatchOkHttp.rules) {
if (rule.matches(ctx)) {
rule.apply(ctx, currentHeader, consumer);

26
src/main/java/quant/rich/emoney/patch/okhttp/PatchOkHttpRule.java
View File

@@ -1,5 +1,6 @@
package quant.rich.emoney.patch.okhttp;
import java.io.Serializable;
import java.util.*;
import java.util.function.*;
import java.util.regex.Pattern;
@@ -7,9 +8,18 @@ import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import quant.rich.emoney.patch.okhttp.PatchOkHttp.HeaderInterceptor;
public class PatchOkHttpRule {
@Getter
@Setter
@Accessors(chain=true)
private Serializable id;
private final Predicate<RequestContext> condition;
private final List<HeaderAction> actions;
@@ -88,11 +98,11 @@ public class PatchOkHttpRule {
}
public Builder isHttp() {
return and(ctx -> ctx.scheme.equalsIgnoreCase("http"));
return and(ctx -> "http".equalsIgnoreCase(ctx.scheme));
}
public Builder isHttps() {
return and(ctx -> ctx.scheme.equalsIgnoreCase("https"));
return and(ctx -> "https".equalsIgnoreCase(ctx.scheme));
}
/**
@@ -160,4 +170,16 @@ public class PatchOkHttpRule {
return new PatchOkHttpRule(condition, actions);
}
}
public boolean equals(PatchOkHttpRule other) {
if (other == null) return false;
if (this.id == null || other.id == null) return false;
return this.id.equals(other.id);
}
public int hashCode() {
return Objects.hash(id);
}
}

34
src/main/java/quant/rich/emoney/patch/okhttp/RequestContext.java
View File

@@ -7,17 +7,16 @@ import java.util.Map;
import okhttp3.Request;
public class RequestContext {
public final Map<String, String> headers;
public final String scheme;
public final String host;
public final Request.Builder builder;
public Map<String, String> headers;
public String scheme = null;
public String host = null;
public String url = null;
/**
* 使用 OkHttp Request.Builder 来初始化上下文，包括 scheme、host 和 headers
* 使用 OkHttp Request.Builder 来初始化上下文
* @param builder
*/
public RequestContext(Request.Builder builder) {
this.builder = builder;
List<String> nvList = builder.getHeaders$okhttp().getNamesAndValues$okhttp();
Map<String, String> headerMap = new HashMap<>();
@@ -26,21 +25,20 @@ public class RequestContext {
}
this.headers = headerMap;
String url = builder.getUrl$okhttp().toString();
this.scheme = url.startsWith("https") ? "https" : "http";
this.host = url.replaceFirst("https?://", "").split("/")[0];
if (builder.getUrl$okhttp() != null) {
String url = builder.getUrl$okhttp().toString();
this.scheme = url.startsWith("https") ? "https" : "http";
this.host = url.replaceFirst("https?://", "").split("/")[0];
this.url = url;
}
}
/**
* 使用指定 headers、scheme 和 host 来初始化上下文，一般用在单元测试场景
* @param headers
* @param scheme
* @param host
* 使用 OkHttp Request 来初始化上下文
* @param builder
*/
public RequestContext(Map<String, String> headers, String scheme, String host) {
this.builder = null;
this.headers = headers;
this.scheme = scheme;
this.host = host;
public RequestContext(Request request) {
this(request.newBuilder());
}
}

47
src/main/java/quant/rich/emoney/pojo/dto/IndexDetail.java Normal file
View File

@@ -0,0 +1,47 @@
package quant.rich.emoney.pojo.dto;
import java.util.List;
import lombok.Data;
import lombok.experimental.Accessors;
public interface IndexDetail {
/**
* 获取 indexName
* @return
*/
public String getIndexName();
/**
* 获取 indexCode
* @return
*/
public String getIndexCode();
/**
* 详情接口
* @return
*/
public List<Detail> getDetails();
/**
* 清洗 HTML 内容
*/
public void sanitize();
@Data
@Accessors(chain=true)
public static class Detail {
private String content;
private DetailType type;
}
public static enum DetailType {
TITLE,
TEXT,
IMAGE
}
}

4
src/main/java/quant/rich/emoney/pojo/dto/IpInfo.java
View File

@@ -35,7 +35,7 @@ public class IpInfo {
if (StringUtils.isNoneBlank(getIpv6())) {
sb.append(", ").append("IPv6: ").append(getIpv6());
}
sb.append(getGeoString());
sb.append('(').append(getGeoString()).append(')');
return sb.toString();
}
@@ -45,7 +45,6 @@ public class IpInfo {
public String getGeoString() {
StringBuilder sb = new StringBuilder();
sb.append("(");
if (StringUtils.isNotBlank(getCountry())) {
// 国家
sb.append(getCountry());
@@ -60,7 +59,6 @@ public class IpInfo {
else {
sb.append("未知位置");
}
sb.append(")");
return sb.toString();
}

101
src/main/java/quant/rich/emoney/pojo/dto/NonParamsIndexDetail.java Normal file
View File

@@ -0,0 +1,101 @@
package quant.rich.emoney.pojo.dto;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Data;
import lombok.experimental.Accessors;
import quant.rich.emoney.util.HtmlSanitizer;
@Data
@Accessors(chain=true)
public class NonParamsIndexDetail implements IndexDetail {
@JsonView(IndexDetail.class)
private String id;
@JsonView(IndexDetail.class)
private String name;
@JsonView(IndexDetail.class)
private String nameCode;
@JsonView(IndexDetail.class)
private List<NonParamsIndexDetailData> data = new ArrayList<>();
@Data
@Accessors(chain=true)
public static class NonParamsIndexDetailData {
@JsonView(IndexDetail.class)
private String title;
@JsonView(IndexDetail.class)
private List<String> items;
@JsonView(IndexDetail.class)
private String image;
}
@Override
public String getIndexName() {
return name;
}
@Override
public String getIndexCode() {
return nameCode;
}
@Override
public List<Detail> getDetails() {
List<Detail> list = new ArrayList<>();
data.forEach(d -> {
if (StringUtils.isNotBlank(d.getTitle())) {
list.add(new Detail().setContent(d.getTitle()).setType(DetailType.TITLE));
}
if (!CollectionUtils.isEmpty(d.getItems())) {
d.getItems().forEach(i -> {
list.add(new Detail().setContent(i).setType(DetailType.TEXT));
});
}
String image = d.getImage();
if (StringUtils.isNotBlank(image)) {
// 判断是否是合法 base64 图片，如果不是，生成警告
if (!image.startsWith("data:image/") || !image.contains("base64")) {
list.add(new Detail()
.setContent(
new StringBuilder()
.append("配图 url: ")
.append(image)
.append(" 无法获取其地址，请检查或重试")
.toString()
)
.setType(DetailType.TEXT));
}
else {
list.add(new Detail().setContent(image).setType(DetailType.IMAGE));
}
}
});
return list;
}
@Override
public void sanitize() {
for (NonParamsIndexDetailData data : getData()) {
String title = data.getTitle();
if (StringUtils.isNotBlank(title)) {
data.setTitle(HtmlSanitizer.sanitize(title));
}
if (data.getItems() != null) {
List<String> items = new ArrayList<>();
for (String item : data.getItems()) {
items.add(HtmlSanitizer.sanitize(item));
}
data.setItems(items);
}
}
}
}

50
src/main/java/quant/rich/emoney/pojo/dto/ParamsIndexDetail.java Normal file
View File

@@ -0,0 +1,50 @@
package quant.rich.emoney.pojo.dto;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Data;
import lombok.experimental.Accessors;
import quant.rich.emoney.util.HtmlSanitizer;
@Data
@Accessors(chain=true)
public class ParamsIndexDetail implements IndexDetail {
@JsonView(IndexDetail.class)
private String id;
@JsonView(IndexDetail.class)
private String name;
@JsonView(IndexDetail.class)
private String code;
@JsonView(IndexDetail.class)
private List<String> descriptions = new ArrayList<>();
@Override
public String getIndexName() {
return name;
}
@Override
public String getIndexCode() {
return code;
}
@Override
public List<Detail> getDetails() {
List<Detail> list = new ArrayList<>();
descriptions.forEach(des -> {
list.add(new Detail().setContent(des).setType(DetailType.TEXT));
});
return list;
}
@Override
public void sanitize() {
List<String> descriptions = new ArrayList<>();
for (String description : getDescriptions()) {
descriptions.add(HtmlSanitizer.sanitize(description));
}
setDescriptions(descriptions);
}
}

3
src/main/java/quant/rich/emoney/service/ConfigService.java
View File

@@ -236,9 +236,8 @@ public class ConfigService implements InitializingBean {
String field = fieldClassCache.inverse().get(config.getClass());
ConfigInfo info = getConfigInfoByField(field);
String configJoString;
SmartViewWriter writer = new SmartViewWriter();
configJoString = writer.writeWithSmartView(config, IConfig.Views.Persistence.class);
String configJoString = writer.writeWithSmartView(config, IConfig.Views.Persistence.class);
if (info.save()) {
try {

536
src/main/java/quant/rich/emoney/service/IndexDetailService.java Normal file
View File

@@ -0,0 +1,536 @@
package quant.rich.emoney.service;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Value;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import quant.rich.emoney.client.OkHttpClientProvider;
import quant.rich.emoney.entity.config.EmoneyRequestConfig;
import quant.rich.emoney.entity.config.IndexInfoConfig;
import quant.rich.emoney.entity.config.SmartViewWriter;
import quant.rich.emoney.pojo.dto.IndexDetail;
import quant.rich.emoney.pojo.dto.NonParamsIndexDetail;
import quant.rich.emoney.pojo.dto.NonParamsIndexDetail.NonParamsIndexDetailData;
import quant.rich.emoney.util.EncryptUtils;
import quant.rich.emoney.util.SpringContextHolder;
import quant.rich.emoney.pojo.dto.ParamsIndexDetail;
/**
* 获取指标详情的服务
*/
@Slf4j
@Service
public class IndexDetailService {
@Autowired
IndexInfoConfig indexInfoConfig;
@Autowired
EmoneyRequestConfig emoneyRequestConfig;
static final String filePath = "./conf/extra/indexDetail/";
static final ObjectMapper mapper = new ObjectMapper();
static final Context jsContext = Context.create("js");
static final Pattern nonParamsIndexDetailPattern = Pattern.compile("\\[(\\{(id:\\d+,?|data:\\[\\{(title:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|items:\\[(\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?)+],?|image:\".*?\",?)+\\}+\\],?|name:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|nameCode:\"\\d+\",?)+\\},?)+\\]", Pattern.DOTALL);
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
/**
* 强制刷新并重新获取
* @param indexCode
* @return
*/
@CacheEvict(cacheNames="@indexDetailService.getIndexDetail(Serializable)", key="#indexCode.toString()")
public IndexDetail forceRefreshAndGetIndexDetail(Serializable indexCode) {
Path path = getIndexDetailPath(indexCode);
try {
Files.deleteIfExists(path);
} catch (IOException e) {
String msg = MessageFormat.format("本地 IndexDetail 文件删除失败，path: {0}, msg: {1}", path.toString(), e.getLocalizedMessage());
throw new RuntimeException(msg, e);
}
return getIndexDetail(indexCode);
}
@Cacheable(cacheNames="@indexDetailService.getIndexDetail(Serializable)", key="#indexCode.toString()")
public IndexDetail getIndexDetail(Serializable indexCode) {
if (indexCode == null) {
throw new NullPointerException("indexCode 不能为空");
}
if (!hasParams(indexCode)) {
return getNonParamsIndexDetail(indexCode);
}
else {
return getParamsIndexDetail(indexCode);
}
}
private ParamsIndexDetail getParamsIndexDetail(Serializable indexCode) {
// 先判断本地有没有
Path localFilePath = getIndexDetailPath(indexCode);
if (Files.exists(localFilePath)) {
ParamsIndexDetail detail = null;
try {
String str = Files.readString(localFilePath);
detail = mapper.readValue(str, ParamsIndexDetail.class);
} catch (IOException e) {
log.warn("无法获取本地无参数指标说明，将尝试重新从网络获取 indexCode: {}", indexCode, e);
}
if (detail != null) {
return detail;
}
}
// 从网络获取
return getParamsIndexDetailOnline(indexCode);
}
private ParamsIndexDetail getParamsIndexDetailOnline(Serializable indexCode) {
try {
OkHttpClient client = OkHttpClientProvider.getInstance();
String url = "https://emapp.emoney.cn/Config/AppIndicator/Get";
byte[] content = new StringBuilder().append("{\"code\":\"").append(indexCode).append("\"}").toString().getBytes();
MediaType type = MediaType.parse("application/json");
RequestBody body = RequestBody.create(content, type);
Request request = new Request.Builder()
.url(url)
.post(body)
.header("X-Protocol-Id", "Config%2FAppIndicator%2FGet")
.header("X-Request-Id", "null")
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", "Config%2FAppIndicator%2FGet"))
.header("Authorization", emoneyRequestConfig.getAuthorization())
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode())
.build();
final Response response = client.newCall(request).execute();
String responseText = response.body().string();
ObjectNode result = (ObjectNode)mapper.readTree(responseText);
Integer code = result.get("result").get("code").asInt();
if (code == 0) {
ParamsIndexDetail detail = mapper.treeToValue(result.get("detail"), ParamsIndexDetail.class);
if (detail == null) {
// 网络访问成功但为 null, 新建一空 detail
detail = new ParamsIndexDetail();
detail.setCode(indexCode.toString());
detail.getDescriptions().add("该指标说明接口返回为空");
}
// 清洗内容：凡是文本类型的内容的，都要清洗一遍，判断是否有脚本、
// 视频和图片等，有的要清除并记录，以免有泄露网址、客户端 IP 风险
detail.sanitize();
saveIndexDetail(detail);
return detail;
}
else {
log.error("网络获取 ParamsIndexDetail 失败，code: {}, msg: {}, indexCode: {}",
code, result.get("result").get("msg").asText(),
indexCode);
return null;
}
}
catch (Exception e) {
String message = MessageFormat.format("网络获取 ParamsIndexDetail 失败，indexCode: {0}，message: {1}",
indexCode, e.getLocalizedMessage());
log.error(message, e);
throw new RuntimeException(message, e);
}
}
/**
* 获取不带参数的指标信息，通过模拟 Webview 获取
* @param indexCode
* @return
*/
private NonParamsIndexDetail getNonParamsIndexDetail(Serializable indexCode) {
// 先判断本地有没有
Path localFilePath = getIndexDetailPath(indexCode);
if (Files.exists(localFilePath)) {
NonParamsIndexDetail detail = null;
try {
String str = Files.readString(localFilePath);
detail = mapper.readValue(str, NonParamsIndexDetail.class);
}
catch (IOException e) {
log.warn("无法获取本地无参数指标说明，将尝试重新从网络获取 indexCode: {}", indexCode, e);
}
if (detail != null) {
loadImages(detail);
saveIndexDetail(detail);
return detail;
}
}
// 从网络获取
return getNonParamsIndexDetailOnline(indexCode);
}
/**
* 从网络获取指定 indexCode 的无参指标详情
* <p>会一并尝试获取其他在本地未有的无参指标</p>
* @param indexCode
* @return
*/
private NonParamsIndexDetail getNonParamsIndexDetailOnline(Serializable indexCode) {
String url = buildNonParamsIndexUrl(indexCode);
Request request = new Request.Builder()
.url(url)
.header("Host", "appstatic.emoney.cn")
.header("Connection", "keep-alive")
.header("Upgrade-Insecure-Requests", "1")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "none")
.header("Sec-Fetch-Mode", "navigate")
.header("Sec-Fetch-User", "?1")
.header("Sec-Fetch-Dest", "document")
.header("Accept-Encoding", "gzip, deflate")
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7")
.build();
OkHttpClient client = OkHttpClientProvider.getInstance();
List<String> scripts = new ArrayList<>();
// 1. 获取页面内 scripts 列表，需要的信息都包含在里面
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.error("NonParamsIndexDetail 获取失败，status code: {}, indexCode: {}", response.code(), indexCode);
return null;
}
String responseBody = response.body().string();
Document doc = Jsoup.parseBodyFragment(responseBody);
doc.select("script[src]").forEach(el -> {
String absoluteURI = resolveUrl(url, el.attr("src"));
log.info("script uri: {}", absoluteURI);
if (absoluteURI != null) {
scripts.add(absoluteURI);
}
});
} catch (IOException e) {
String msg = MessageFormat.format("NonParamsIndexDetail 获取失败：指标页访问失败，indexCode: {0}", indexCode);
log.error(msg, e);
throw new RuntimeException(msg, e);
}
if (scripts.size() == 0) {
String msg = MessageFormat.format("NonParamsIndexDetail 获取失败：未能获取基本页面内脚本链接，indexCode: {0}", indexCode);
log.error(msg);
throw new RuntimeException(msg);
}
// 2. 加载 scripts，试图匹配包含在 js 中的指标说明
Request.Builder scriptBuilder = new Request.Builder()
.header("Host", "appstatic.emoney.cn")
.header("Connection", "keep-alive")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "*/*")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "same-origin")
.header("Sec-Fetch-Mode", "no-cors")
.header("Sec-Fetch-Dest", "script")
.header("Accept-Encoding", "gzip, deflate")
.header("Referer", url)
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
List<NonParamsIndexDetail> valids = new ArrayList<>();
scriptLoop:
for (String scriptUrl : scripts) {
Request scriptRequest = scriptBuilder.url(scriptUrl).build();
try (Response response = client.newCall(scriptRequest).execute()) {
if (!response.isSuccessful()) {
log.warn("NonParamsIndexDetail 获取失败：读取页面内 script 请求失败, response.isSuccessful() is false, indexCode: {}, scriptUrl: {}", indexCode, scriptUrl);
continue;
}
String responseBody = response.body().string();
// 尝试匹配
Matcher m = nonParamsIndexDetailPattern.matcher(responseBody);
while (m.find()) {
String find = m.group();
Optional<String> jsonStringify = stringifyJsArray(find);
if (jsonStringify.isPresent()) {
String jsonString = jsonStringify.get();
JsonNode root = mapper.readTree(jsonString);
boolean foundAny = false;
if (root.isArray()) {
ArrayNode array = (ArrayNode) root;
for (JsonNode obj : array) {
if (obj.isObject() &&
obj.has("name") &&
obj.get("name").isTextual() &&
obj.has("nameCode") && obj.get("nameCode").isTextual() &&
obj.has("data") && obj.get("data").isArray()) {
NonParamsIndexDetail detail = mapper.treeToValue(obj, NonParamsIndexDetail.class);
valids.add(detail);
foundAny = true;
}
}
}
if (foundAny) {
break scriptLoop;
}
}
}
} catch (IOException e) {
log.warn("NonParamsIndexDetail 获取失败：读取页面内 script 请求失败, response.isSuccessful() is true, indexCode: {}, scriptUrl: {}", indexCode, scriptUrl, e);
continue;
}
}
if (valids.isEmpty()) {
String msg = MessageFormat.format(
"NonParamsIndexDetail 获取失败：未从页面 js 脚本中获取到任何可能匹配的参数说明，indexCode: {}",
indexCode);
log.warn(msg);
throw new RuntimeException(msg);
}
// 直接保存在本地
Pattern numericPattern = Pattern.compile("^\\d+$");
NonParamsIndexDetail targetDetail = null;
for (NonParamsIndexDetail detail : valids) {
// 判断 nameCode 是否是合法的代码
if (!numericPattern.matcher(detail.getNameCode()).matches()) {
continue;
}
Path path = getIndexDetailPath(detail);
// 判断是否是需求的 detail
if (indexCode.toString().equals(detail.getIndexCode())) {
loadImages(detail);
targetDetail = detail;
}
// 清洗内容：凡是文本类型的内容的，都要清洗一遍，判断是否有脚本、
// 视频和图片等，有的要清除并记录，以免有泄露网址、客户端 IP 风险
detail.sanitize();
if (!Files.exists(path)) {
// 不存在则保存
saveIndexDetail(detail);
}
}
if (targetDetail == null) {
// 创建一空 target
targetDetail = new NonParamsIndexDetail();
targetDetail.setNameCode(indexCode.toString());
NonParamsIndexDetailData data = new NonParamsIndexDetailData();
List<String> items = List.of("该指标说明接口返回为空");
data.setItems(items);
targetDetail.getData().add(data);
Path path = getIndexDetailPath(targetDetail);
if (!Files.exists(path)) {
// 不存在则保存
saveIndexDetail(targetDetail);
}
}
return targetDetail;
}
/**
* 为指定 NonParamsIndexDetail 加载在线图片并转换成 base64
* <p>该方法涉及 authorization(token)/webviewUserAgent，需确保 EmoneyRequestConfig 已正确注入并登录</p>
* @param detail
* @return
* @see EmoneyRequestConfig
*/
private NonParamsIndexDetail loadImages(NonParamsIndexDetail detail) {
OkHttpClient client = OkHttpClientProvider.getInstance();
for (NonParamsIndexDetailData data : detail.getData()) {
String imageUrl = data.getImage();
if (StringUtils.isNotBlank(imageUrl)) {
if (imageUrl.startsWith("data:image/") && imageUrl.contains("base64")) {
continue;
}
else if (imageUrl.toLowerCase().startsWith("http")) {
// 需要转换
URI uri;
try {
uri = new URI(imageUrl);
}
catch (URISyntaxException e) {
log.warn("未经转换的 NonParamsIndexDetailData.image {} 非合法 http(s) 地址，请检查", imageUrl, e);
continue;
}
String host = uri.getHost();
// 从网络获取
Request request = new Request.Builder()
.url(imageUrl)
.header("Host", host)
.header("Connection", "keep-alive")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "same-origin")
.header("Sec-Fetch-Mode", "no-cors")
.header("Sec-Fetch-Dest", "image")
.header("Accept-Encoding", "gzip, deflate")
.header("Referer", buildNonParamsIndexUrl(detail.getIndexCode()))
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7")
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.warn("未经转换的 NonParamsIndexDetailData.image {} 获取图片详情失败，请检查或重试", imageUrl);
continue;
}
byte[] bytes = response.body().bytes();
String contentType = response.body().contentType().toString();
String base64 = Base64.getEncoder().encodeToString(bytes);
data.setImage("data:" + contentType + ";base64," + base64);
}
catch (IOException e) {
log.warn("未经转换的 NonParamsIndexDetailData.image {} 获取图片详情失败，请检查或重试", imageUrl, e);
continue;
}
}
else {
log.warn("未知的 NonParamsIndexDetailData.image 格式 {}", imageUrl);
}
}
}
return detail;
}
/**
* 保存指标详情到本地文件
* @param <Description>
* @param des
*/
private <Description extends IndexDetail> void saveIndexDetail(Description des) {
SmartViewWriter writer = new SmartViewWriter();
String joString = writer.writeWithSmartView(des, IndexDetail.class);
Path path = getIndexDetailPath(des);
try {
Files.writeString(path, joString);
}
catch (IOException e) {
log.error("写入指标详情到 {} 失败", path.toString(), e);
}
}
/**
* 获取指标详情本地路径 Path
* @param <Description>
* @param description
* @return
*/
private <Description extends IndexDetail> Path getIndexDetailPath(Description description) {
Path path = Path.of(new StringBuilder(filePath)
.append((description instanceof NonParamsIndexDetail) ? "nonParams/": "params/")
.append(description.getIndexCode())
.append(".json").toString());
return path;
}
/**
* 根据指标代码 indexCode 获取本地文件路径
* @param indexCode
* @return
*/
private Path getIndexDetailPath(Serializable indexCode) {
boolean hasParams = hasParams(indexCode);
Path path = Path.of(new StringBuilder(filePath)
.append(!hasParams ? "nonParams/": "params/")
.append(indexCode)
.append(".json").toString());
return path;
}
/**
* 根据指标代码 indexCode 判断是否有参数
* @param indexCode
* @return
*/
private boolean hasParams(Serializable indexCode) {
JsonNode configIndOnline = indexInfoConfig.getConfigIndOnline();
JsonNode jsonNode = configIndOnline.get("indMap").get(indexCode.toString());
if (jsonNode == null) {
throw new RuntimeException("无法根据指定 indexCode: " + indexCode + " 找到对应 indMap 内容");
}
ObjectNode node = (ObjectNode)jsonNode;
return node.has("indParam");
}
/**
* 获取 NonParamsIndexDetail URL
* <p>该 Url 涉及 authorization(token)，需确保 EmoneyRequestConfig 已正确注入并登录</p>
* @param indexCode
* @return
* @see EmoneyRequestConfig
*/
private String buildNonParamsIndexUrl(Serializable indexCode) {
StringBuilder urlBuilder = new StringBuilder();
urlBuilder.append("https://appstatic.emoney.cn/html/emapp/stock/note/?name=");
urlBuilder.append(indexCode.toString());
urlBuilder.append("&emoneyScaleType=0&emoneyLandMode=0&token=");
urlBuilder.append(emoneyRequestConfig.getAuthorization());
return urlBuilder.toString();
}
/**
* 根据页面解析绝对路径
*/
private static String resolveUrl(String pageUrl, String src) {
try {
URI base = URI.create(pageUrl);
return base.resolve(src).toString();
} catch (Exception e) {
log.warn("转换 URI 错误，pageUrl: {}, scriptSrc: {} {}", pageUrl, src, e);
return null;
}
}
/**
* 使用 GraalVM 的 JS 引擎尝试 JSON.stringify
* @param jsArrayText
* @return
*/
private static Optional<String> stringifyJsArray(String jsArrayText) {
try {
Value result = jsContext.eval("js", "JSON.stringify(" + jsArrayText + ")");
return result.isString() ? Optional.of(result.asString()) : Optional.empty();
} catch (Exception e) {
log.warn("Failed to call JSON.stringify on {}", jsArrayText, e);
return Optional.empty();
}
}
}

1
src/main/java/quant/rich/emoney/util/GeoIPUtil.java
View File

@@ -118,6 +118,7 @@ public class GeoIPUtil {
} catch (IOException | GeoIp2Exception e) {
log.warn("Get city response from GeoLite2-City database failed: {}", e.getMessage());
}
log.debug("ipInfo: {}", ipInfo);
return ipInfo;
}

134
src/main/java/quant/rich/emoney/util/HtmlSanitizer.java Normal file
View File

@@ -0,0 +1,134 @@
package quant.rich.emoney.util;
import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;
import java.util.Iterator;
import java.util.Random;
public class HtmlSanitizer {
public static final Random random = new Random();
public static String sanitize(String unsafeHtml) {
String nlRandom =
new StringBuilder("__NL").append(random.nextInt()).append("__").toString();
unsafeHtml = unsafeHtml.replaceAll("\n", nlRandom);
Document doc = Jsoup.parseBodyFragment(unsafeHtml);
Element body = doc.body();
NodeTraversor.traverse(new NodeVisitor() {
public void head(Node node, int depth) {
if (!(node instanceof Element)) return;
Element el = (Element) node;
String tag = el.tagName().toLowerCase();
// 清洗 <link rel="stylesheet" href="...">
if (tag.equals("link")) {
String href = el.attr("href");
if (isExternalUrl(href)) {
replaceWithNotice(el, "link[href]");
return;
}
}
// 清洗危险结构元素
switch (tag) {
case "script":
case "iframe":
case "object":
case "embed":
case "video":
case "audio":
case "source":
case "meta":
case "fencedframe":
replaceWithNotice(el, tag);
return;
}
// 清除 on* 属性
Iterator<Attribute> it = el.attributes().iterator();
while (it.hasNext()) {
Attribute attr = it.next();
if (attr.getKey().toLowerCase().startsWith("on")) {
it.remove();
}
}
// 清除 src 外链
if (el.hasAttr("src")) {
String src = el.attr("src");
if (isExternalUrl(src)) {
el.removeAttr("src");
el.appendChild(new TextNode("[已移除外链资源]"));
}
}
// 清除 style="...url(...)" 中的非 base64 URL
if (el.hasAttr("style")) {
String style = el.attr("style");
if (containsExternalUrl(style)) {
style = style.replaceAll("(?i)url\\((?!['\"]?data:)[^)]*\\)", "");
el.attr("style", style + " /* 外链背景图已清除 */");
}
}
// a 标签添加 rel="noreferrer"
if (tag.equals("a")) {
String rel = el.attr("rel");
if (!rel.toLowerCase().contains("noreferrer")) {
el.attr("rel", "noreferrer");
}
}
// style 标签内容清理 url(...)
if (tag.equals("style")) {
for (Node child : el.childNodes()) {
if (child instanceof TextNode) {
TextNode text = (TextNode) child;
String content = text.getWholeText();
// 移除非 base64 的 url(...)
String cleaned = content.replaceAll("(?i)url\\((?!['\"]?data:)[^)]*\\)", "/* 外链已清除 */");
text.text(cleaned);
}
}
}
}
public void tail(Node node, int depth) {
// nothing
}
}, body);
// SVG、MathML 类危险标签
doc.select("svg, math, foreignObject").forEach(el ->
replaceWithNotice(el, el.tagName())
);
unsafeHtml = body.html();
unsafeHtml = unsafeHtml.replaceAll(nlRandom, "\n");
return unsafeHtml;
}
private static boolean isExternalUrl(String url) {
return url != null
&& !url.trim().toLowerCase().startsWith("data:")
&& (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("//"));
}
private static boolean containsExternalUrl(String style) {
return style != null && style.toLowerCase().matches(".*url\\((?!['\"]?data:).+\\).*");
}
private static void replaceWithNotice(Element el, String tagName) {
el.replaceWith(new Element("div")
.addClass("blocked-element")
.text("该 " + tagName + " 元素可能包含危害安全的内容，已屏蔽。"));
}
}