获取指标说明的能力

This commit is contained in:
2025-05-21 15:51:44 +08:00
parent 06c351a956
commit 1f329e3b2a
131 changed files with 4461 additions and 3126 deletions

View File

@@ -2,12 +2,14 @@ package quant.rich.emoney;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling;
@EnableAsync
@EnableScheduling
@SpringBootApplication
@EnableCaching(proxyTargetClass=true)
public class EmoneyAutoApplication {
public static void main(String[] args) {

View File

@@ -26,15 +26,27 @@ import quant.rich.emoney.util.EncryptUtils;
import quant.rich.emoney.util.SpringContextHolder;
import okhttp3.OkHttpClient;
/**
* 益盟操盘手基本请求客户端,提供基本功能
* <p><b>请求头顺序</b></p>
* <p>
* X-Protocol-Id > X-Request-Id > EM-Sign > Authorization >
* X-Android-Agent > Emapp-ViewMode > Content-Type > Content-Length >
* Host > Connection: "Keep-Alive" > Accept-Encoding: "gzip" > User-Agent</p>
* <p>从 X-Protocol-Id 到 Emapp-ViewMode由本例添加剩余为 okhttp 默认添加,
* User-Agent 由 ByteBuddy 重写 header 方法控制添加</p>
* @see quant.rich.emoney.patch.okhttp.PatchOkHttp
*/
@Data
@Slf4j
@Accessors(chain = true)
public class EmoneyClient implements Cloneable {
private static final String MBS_URL = "https://mbs.emoney.cn/";
//private static final String LOGIN_URL = "https://emapp.emoney.cn/user/auth/login";
private static final String LOGIN_URL = "http://localhost:7790/user/auth/login";
private static final String LOGIN_URL = "https://emapp.emoney.cn/user/auth/login";
private static final String RELOGIN_URL = "https://emapp.emoney.cn/user/auth/ReLogin";
private static final String LOGIN_X_PROTOCOL_ID = "user%2Fauth%2Flogin";
private static final String RELOGIN_X_PROTOCOL_ID = "user%2Fauth%2FReLogin";
private static volatile EmoneyRequestConfig emoneyRequestConfig;
@@ -68,8 +80,10 @@ public class EmoneyClient implements Cloneable {
/**
* 使用系统管理的用户名密码登录
* <p>建议仅在调试时使用,其他情况请用 {@code loginWithManaged()}</p>
* @return
*/
@Deprecated
public static Boolean loginWithUsernamePassword() {
ObjectNode formObject = getEmoneyRequestConfig().getUsernamePasswordLoginObject();
return login(formObject);
@@ -77,10 +91,12 @@ public class EmoneyClient implements Cloneable {
/**
* 使用给定的用户名密码登录
* <p>建议仅在调试时使用,其他情况请用 {@code loginWithManaged()}</p>
* @param username 用户名
* @param password 密码,可以是明文,也可是密文
* @return
*/
@Deprecated
public static Boolean loginWithUsernamePassword(String username, String password) {
ObjectNode formObject = getEmoneyRequestConfig().getUsernamePasswordLoginObject(username, password);
return login(formObject);
@@ -88,13 +104,80 @@ public class EmoneyClient implements Cloneable {
/**
* 匿名登录
* <p>建议仅在调试时使用,其他情况请用 {@code loginWithManaged()}</p>
* @return
*/
@Deprecated
public static Boolean loginWithAnonymous() {
ObjectNode formObject = getEmoneyRequestConfig().getAnonymousLoginObject();
return login(formObject);
}
/**
* 触发重登陆验证
* @return
*/
public static Boolean relogin() {
EmoneyRequestConfig emoneyRequestConfig = getEmoneyRequestConfig();
ObjectNode reloginObject = emoneyRequestConfig.getReloginObject();
if (reloginObject == null) {
// 无登录信息,直接触发登录
return loginWithManaged();
}
String token = reloginObject.get("token").asText();
try {
OkHttpClient okHttpClient = OkHttpClientProvider.getInstance();
MediaType type = MediaType.parse("application/json");
byte[] content = reloginObject.toString().getBytes("utf-8");
RequestBody body = RequestBody.create(
content, type);
Request.Builder requestBuilder = new Request.Builder()
.url(RELOGIN_URL)
.post(body)
.header("X-Protocol-Id", RELOGIN_X_PROTOCOL_ID)
.header("X-Request-Id", "1")
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", RELOGIN_X_PROTOCOL_ID))
.header("Authorization", token)
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode());
Request request = requestBuilder.build();
final Call call = okHttpClient.newCall(request);
Response response = call.execute();
if (response.code() != 200) {
// 不是 200重新登录
log.debug("ReLogin 重登录验证返回状态码 {}, 触发登录", response.code());
return loginWithManaged();
}
String responseText = response.body().string();
ObjectNode loginResult = (ObjectNode) new ObjectMapper().readTree(responseText);
Integer code = loginResult.get("result").get("code").asInt();
if (code != 0 ||
!token.equals(loginResult.get("detail").get("token").asText())) {
// 我也不知道会不会进入这里,不过尽可能写完所有可能
log.debug("ReLogin 返回消息 result.code = {}, token 与已有 authorization {}一致, 触发重登陆",
code, code == 0 ? "" : "");
return loginWithManaged();
}
else {
log.debug("ReLogin 校验通过,不需要重登录");
return true;
}
}
catch (JsonProcessingException e) {
throw new EmoneyDecodeException("ReLogin 试图将返回数据解析成 JSON 时失败", e);
}
catch (Exception e) {
throw new EmoneyRequestException("执行 emoney ReLogin 请求/返回时出现错误", e);
}
}
/**
* 登录总方法,唯一要控制的只有 formObject
* @param formObject
@@ -103,7 +186,7 @@ public class EmoneyClient implements Cloneable {
private static Boolean login(ObjectNode formObject) {
try {
//OkHttpClient okHttpClient = new OkHttpClient();
EmoneyRequestConfig emoneyRequestConfig = getEmoneyRequestConfig();
OkHttpClient okHttpClient = OkHttpClientProvider.getInstance();
MediaType type = MediaType.parse("application/json");
//type.charset(StandardCharsets.UTF_8);
@@ -123,31 +206,40 @@ public class EmoneyClient implements Cloneable {
.header("X-Request-Id", "null")
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", LOGIN_X_PROTOCOL_ID))
.header("Authorization", "")
.header("X-Android-Agent", getEmoneyRequestConfig().getXAndroidAgent())
.header("Emapp-ViewMode", getEmoneyRequestConfig().getEmappViewMode());
//.header("User-Agent", getEmoneyRequestConfig().getOkHttpUserAgent())
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode());
//.header("User-Agent", emoneyRequestConfig.getOkHttpUserAgent())
Request request = requestBuilder.build();
final Call call = okHttpClient.newCall(request);
Response response = call.execute();
ObjectNode loginResult = (ObjectNode) new ObjectMapper().readTree(response.body().string());
String responseText = response.body().string();
ObjectNode loginResult = (ObjectNode) new ObjectMapper().readTree(responseText);
Integer code = loginResult.get("result").get("code").asInt();
if (code == 0) {
getEmoneyRequestConfig().setAuthorization(
loginResult.get("detail").get("token").asText())
.updateAuthorizationTime().saveOrUpdate();
emoneyRequestConfig
.setAuthorization(
loginResult
.get("detail").get("token").asText())
.setUid(
loginResult
.get("detail").get("uid").asInt())
.saveOrUpdate();
log.info("执行 emoney LOGIN 成功");
return true;
} else {
}
else {
String msg = loginResult.get("result").get("msg").asText();
throw new EmoneyResponseException("执行 emoney LOGIN 请求返回错误code: " + code + ", msg: " + msg, code);
}
} catch (JsonProcessingException e) {
}
catch (JsonProcessingException e) {
throw new EmoneyDecodeException("试图将返回数据解析成 JSON 时失败", e);
} catch (Exception e) {
}
catch (Exception e) {
throw new EmoneyRequestException("执行 emoney LOGIN 请求/返回时出现错误", e);
}
}
@@ -168,7 +260,8 @@ public class EmoneyClient implements Cloneable {
throw new EmoneyIllegalRequestParamException("执行 emoney 请求错误Protocol id 不能为 null!",
new IllegalArgumentException());
}
if (StringUtils.isBlank(getEmoneyRequestConfig().getAuthorization())) {
EmoneyRequestConfig emoneyRequestConfig = getEmoneyRequestConfig();
if (StringUtils.isBlank(emoneyRequestConfig.getAuthorization())) {
throw new EmoneyIllegalRequestParamException("执行 emoney 请求错误Authorization 为空,是否未登录?",
new IllegalArgumentException());
}
@@ -190,9 +283,9 @@ public class EmoneyClient implements Cloneable {
.header("X-Protocol-Id", xProtocolId.toString())
.header("X-Request-Id", xRequestId == null ? "null" : xRequestId.toString())
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", xProtocolId.toString()))
.header("Authorization", getEmoneyRequestConfig().getAuthorization())
.header("X-Android-Agent", getEmoneyRequestConfig().getXAndroidAgent())
.header("Emapp-ViewMode", getEmoneyRequestConfig().getEmappViewMode())
.header("Authorization", emoneyRequestConfig.getAuthorization())
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode())
;
Request request = requestBuilder.build();

View File

@@ -14,7 +14,11 @@ import java.lang.annotation.*;
public @interface LockByCaller {
/**
* 可选参数,用于 SpEL 表达式获取 key
* 例如:@LockByCaller(key = "#userId")
* 例如:<ul>
* <li>@LockByCaller(key = "#userId")</li>
* <li>@LockByCaller(key = "#userId + ':' + #userName")</li>
* </ul>
* 当不指定时,不校验参数,单纯校验 Caller
*/
String key() default "";
}

View File

@@ -18,6 +18,7 @@ import quant.rich.emoney.controller.common.BaseController;
import quant.rich.emoney.entity.config.IndexInfoConfig;
import quant.rich.emoney.pojo.dto.LayPageResp;
import quant.rich.emoney.pojo.dto.R;
import quant.rich.emoney.service.IndexDetailService;
@Controller
@RequestMapping("/admin/v1/manage/indexInfo")
@@ -25,11 +26,30 @@ public class IndexInfoControllerV1 extends BaseController {
@Autowired
IndexInfoConfig indexInfo;
@Autowired
IndexDetailService indexDetailService;
@GetMapping({"", "/", "/index"})
public String index() {
return "/admin/v1/manage/indexInfo/index";
}
@GetMapping("/getIndexDetail")
@ResponseBody
public R<?> getIndexDetail(String indexCode) {
return
R.judge(() ->
indexDetailService.getIndexDetail(indexCode));
}
@GetMapping("/forceRefreshAndGetIndexDetail")
@ResponseBody
public R<?> forceRefreshAndGetIndexDetail(String indexCode) {
return
R.judge(() ->
indexDetailService.forceRefreshAndGetIndexDetail(indexCode));
}
@GetMapping("/configIndOnline")
@ResponseBody
@@ -53,17 +73,11 @@ public class IndexInfoControllerV1 extends BaseController {
return R.ok(map);
}
@GetMapping("/getIndexDetail")
@ResponseBody
public R<?> getIndexDetail(String code) {
return null;
}
@GetMapping("/getConfigIndOnlineByUrl")
@ResponseBody
public R<?> getConfigOnlineByUrl(String url) {
return R.judge(() -> indexInfo.getOnlineConfigByUrl(url));
return R.judge(() -> indexInfo.getOnlineConfigByUrl());
}
@GetMapping("/getIndexInfoConfig")

View File

@@ -1,6 +1,6 @@
package quant.rich.emoney.entity.config;
import java.time.LocalDateTime;
import java.io.Serializable;
import java.util.Objects;
import org.apache.commons.lang3.ObjectUtils;
@@ -10,12 +10,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
import lombok.AccessLevel;
import lombok.Data;
import lombok.Getter;
@@ -67,11 +63,9 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
private String authorization;
/**
* 当前 authorization 更新时间
* UID
*/
@JsonSerialize(using = LocalDateTimeSerializer.class)
@JsonDeserialize(using = LocalDateTimeDeserializer.class)
private LocalDateTime authorizationUpdateTime;
private Integer uid;
/**
* <b>用于:</b><ul>
@@ -186,6 +180,12 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
* <b>来源:</b>本例管理
*/
private String emappViewMode = "1";
/**
* OkHttp 用于注入 User-Agent 规则的 id
*/
@JsonIgnore
private Serializable userAgentPatchRuleId;
@Getter(AccessLevel.PRIVATE)
@Autowired
@@ -215,7 +215,7 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
initFromRandomDeviceInfo();
}
else {
// 都不是 null则由 fingerprint 来检查各项
// 都不是 null则由 fingerprint 来检查各项
// model 和 softwareType 本应交由 deviceInfoConfig 检查以
// 应对可能的通过修改本地 json 来进行攻击的方式,可是本身
// deviceInfoConfig 对 model 和 softwareType 的信息也来源
@@ -248,12 +248,22 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
}
// 注入 OkHttp
PatchOkHttp.apply(
patchOkHttp();
}
/**
* 注入 User-Agent patch 规则
*/
private EmoneyRequestConfig patchOkHttp() {
userAgentPatchRuleId = PatchOkHttp.apply(
PatchOkHttpRule.when()
.hostEndsWith("emoney.cn")
.not(r -> r.hostMatches("appstatic"))
.or(a -> a.hostContains("emapp"))
.or(b -> b.hasHeaderName("X-Protocol-Id"))
.overrideIf("User-Agent", getOkHttpUserAgent()).build());
.overrideIf("User-Agent", getOkHttpUserAgent()).build()
.setId(userAgentPatchRuleId));
return this;
}
/**
@@ -451,6 +461,45 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
return node;
}
/**
* 根据本例信息获取 Relogin ObjectNode
* @return 如果 authorization 和 uid 任意 null 则本例返回 null
*/
@JsonIgnore
public ObjectNode getReloginObject() {
if (ObjectUtils.anyNull(getAuthorization(), getUid())) {
return null;
}
ObjectMapper mapper = new ObjectMapper();
ObjectNode node = mapper.createObjectNode();
ObjectNode exIdentify = mapper.createObjectNode();
exIdentify.put("IMEI", "");
exIdentify.put("AndroidID", getAndroidId());
exIdentify.put("MAC", "");
exIdentify.put("OSFingerPrint", getFingerprint());
String exIdentifyString = exIdentify.toString().replace("/", "\\/");
// 这玩意最好按照顺序来,当前这个顺序是 5.8.1 的顺序
String guid = getGuid();
node.put("appVersion", getEmoneyVersion());
node.put("productId", 4);
node.put("softwareType", getSoftwareType());
node.put("deviceName", getDeviceName());
node.put("ssid", "0");
node.put("platform", "android");
node.put("token", getAuthorization()); // 和登录不同的地方: token
node.put("exIdentify", exIdentifyString);
node.put("uid", getUid()); // 和登录不同的地方: uid
node.put("osVersion", getAndroidSdkLevel());
node.put("guid", guid);
node.put("channelId", "1711");
node.put("hardware", getHardware());
return node;
}
/**
* 设置密码:<ul>
* <li>null or empty保存空字符串</li>
@@ -483,31 +532,8 @@ public class EmoneyRequestConfig implements IConfig<EmoneyRequestConfig> {
return this;
}
/**
* 触发更新 Authorization 的更新时间为现在
* @return
*/
public EmoneyRequestConfig updateAuthorizationTime() {
this.authorizationUpdateTime = LocalDateTime.now();
public EmoneyRequestConfig afterSaving() {
patchOkHttp();
return this;
}
public EmoneyRequestConfig mergeTo(EmoneyRequestConfig other) {
boolean authorizationUpdated = !Objects.equals(other.authorization, this.authorization);
IConfig.super.mergeTo(other);
if (authorizationUpdated) {
other.updateAuthorizationTime();
}
return other;
}
public EmoneyRequestConfig mergeFrom(EmoneyRequestConfig other) {
boolean authorizationUpdated = !Objects.equals(other.authorization, this.authorization);
IConfig.super.mergeFrom(other);
if (authorizationUpdated) {
this.updateAuthorizationTime();
}
return this;
}
}

View File

@@ -21,6 +21,7 @@ import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import quant.rich.emoney.client.OkHttpClientProvider;
import quant.rich.emoney.component.LockByCaller;
import quant.rich.emoney.interfaces.ConfigInfo;
import quant.rich.emoney.interfaces.IConfig;
@@ -33,8 +34,6 @@ import quant.rich.emoney.interfaces.IConfig;
@ConfigInfo(field = "indexInfo", name =" 指标信息配置", initDefault = true, managed = false)
public class IndexInfoConfig implements IConfig<IndexInfoConfig> {
private static final String CONFIG_IND_ONLINE_PATH = "./conf/extra/config_ind_online.json";
@JsonView(IConfig.Views.Persistence.class)
private String configIndOnlineUrl;
@@ -46,23 +45,18 @@ public class IndexInfoConfig implements IConfig<IndexInfoConfig> {
@Getter(AccessLevel.PRIVATE)
private EmoneyRequestConfig emoneyRequestConfig;
public IndexInfoConfig() {
try {
String configStr = FileUtil.readString(CONFIG_IND_ONLINE_PATH);
configIndOnline = new ObjectMapper().readTree(configStr);
}
catch (Exception e) {}
configIndOnlineUrl = "https://emapp-static.oss-cn-shanghai.aliyuncs.com/down13/emstock/config/ind_config/v1000003/config_ind_online.json";
}
public IndexInfoConfig() {}
public String getConfigIndOnlineStr() {
return getConfigIndOnline().toPrettyString();
}
public String getOnlineConfigByUrl(String url) throws IOException {
@LockByCaller
@JsonIgnore
public String getOnlineConfigByUrl() throws IOException {
synchronized (this) {
Request request = new Request.Builder()
.url(url)
.url(configIndOnlineUrl)
.header("Cache-Control", "no-cache")
.get()
.build();

View File

@@ -2,6 +2,9 @@ package quant.rich.emoney.entity.config;
import java.net.InetSocketAddress;
import java.net.Proxy;
import org.apache.commons.lang3.ObjectUtils;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Data;
import lombok.experimental.Accessors;
@@ -77,18 +80,22 @@ public class ProxyConfig implements IConfig<ProxyConfig> {
return Proxy.NO_PROXY;
}
public static void main(String[] args) {
String proxyIp = "127.0.0.1";
int proxyPort = 7897;
ProxyConfig proxyConfig = new ProxyConfig();
proxyConfig.setProxyHost(proxyIp).setProxyPort(proxyPort)
.setProxyType(Proxy.Type.SOCKS)
.setIgnoreHttpsVerification(false);
IpInfo result = GeoIPUtil.getIpInfoThroughProxy(proxyConfig);
System.out.println("Proxy is usable with through-proxy ip: " + result);
public String getProxyUrl() {
if (ObjectUtils.anyNull(getProxyType(), getProxyHost(), getProxyPort())) {
return null;
}
StringBuilder sb = new StringBuilder();
if (getProxyType() == Proxy.Type.SOCKS) {
sb.append("socks5://");
}
else if (getProxyType() == Proxy.Type.HTTP) {
sb.append("http://");
}
else {
return null;
}
sb.append(getProxyHost()).append(':').append(getProxyPort());
return sb.toString();
}
}

View File

@@ -2,19 +2,16 @@ package quant.rich.emoney.patch.okhttp;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.agent.ByteBuddyAgent;
import net.bytebuddy.agent.builder.AgentBuilder;
import net.bytebuddy.asm.Advice;
import okhttp3.Request;
import quant.rich.emoney.util.SpringContextHolder;
import net.bytebuddy.dynamic.ClassFileLocator;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.dynamic.loading.ClassReloadingStrategy;
import static net.bytebuddy.matcher.ElementMatchers.*;
import java.lang.instrument.Instrumentation;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.function.Consumer;
import org.slf4j.Logger;
@@ -23,31 +20,45 @@ import org.springframework.boot.SpringApplication;
import org.springframework.context.ConfigurableApplicationContext;
public class PatchOkHttp {
private static final Random random = new Random();
private static final Logger log = LoggerFactory.getLogger(PatchOkHttp.class);
private static final List<PatchOkHttpRule> rules = new CopyOnWriteArrayList<>();
private static final Set<PatchOkHttpRule> rules = new CopyOnWriteArraySet<>();
private static boolean logOnce = false;
private static boolean isHooked = false;
public static void apply(PatchOkHttpRule rule) {
/**
* 应用指定 Patch 规则
* @param rule
* @return 如果 rule 未设置 id则生成随机 id 并返回,否则返回 rule.getId()
*/
public static Serializable apply(PatchOkHttpRule rule) {
if (rule.getId() == null) {
rule.setId(random.nextInt());
}
rules.add(rule);
// log.debug("apply() running in classloader {}", PatchOkHttp.class.getClassLoader());
log.debug("PatchOkHttp.apply() running in classloader {}", PatchOkHttp.class.getClassLoader());
if (!isHooked) hook();
return rule.getId();
}
public static void apply(PatchOkHttpRule.Builder builder) {
rules.add(builder.build());
if (!isHooked) hook();
public static Serializable apply(PatchOkHttpRule.Builder builder) {
return PatchOkHttp.apply(builder.build());
}
public static void apply(Consumer<PatchOkHttpRule.Builder> r) {
public static Serializable apply(Consumer<PatchOkHttpRule.Builder> r) {
PatchOkHttpRule.Builder builder = PatchOkHttpRule.when();
r.accept(builder);
rules.add(builder.build());
if (!isHooked) hook();
return PatchOkHttp.apply(builder.build());
}
public static void match(RequestContext ctx, String currentHeader, Consumer<String> consumer) {
// log.debug("match() running in classloader {}", PatchOkHttp.class.getClassLoader());
if (!logOnce) {
log.debug("PatchOkHttp.match() running in classloader {}", PatchOkHttp.class.getClassLoader());
logOnce = true;
}
for (PatchOkHttpRule rule : PatchOkHttp.rules) {
if (rule.matches(ctx)) {
rule.apply(ctx, currentHeader, consumer);

View File

@@ -1,5 +1,6 @@
package quant.rich.emoney.patch.okhttp;
import java.io.Serializable;
import java.util.*;
import java.util.function.*;
import java.util.regex.Pattern;
@@ -7,9 +8,18 @@ import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import quant.rich.emoney.patch.okhttp.PatchOkHttp.HeaderInterceptor;
public class PatchOkHttpRule {
@Getter
@Setter
@Accessors(chain=true)
private Serializable id;
private final Predicate<RequestContext> condition;
private final List<HeaderAction> actions;
@@ -88,11 +98,11 @@ public class PatchOkHttpRule {
}
public Builder isHttp() {
return and(ctx -> ctx.scheme.equalsIgnoreCase("http"));
return and(ctx -> "http".equalsIgnoreCase(ctx.scheme));
}
public Builder isHttps() {
return and(ctx -> ctx.scheme.equalsIgnoreCase("https"));
return and(ctx -> "https".equalsIgnoreCase(ctx.scheme));
}
/**
@@ -160,4 +170,16 @@ public class PatchOkHttpRule {
return new PatchOkHttpRule(condition, actions);
}
}
public boolean equals(PatchOkHttpRule other) {
if (other == null) return false;
if (this.id == null || other.id == null) return false;
return this.id.equals(other.id);
}
public int hashCode() {
return Objects.hash(id);
}
}

View File

@@ -7,17 +7,16 @@ import java.util.Map;
import okhttp3.Request;
public class RequestContext {
public final Map<String, String> headers;
public final String scheme;
public final String host;
public final Request.Builder builder;
public Map<String, String> headers;
public String scheme = null;
public String host = null;
public String url = null;
/**
* 使用 OkHttp Request.Builder 来初始化上下文,包括 scheme、host 和 headers
* 使用 OkHttp Request.Builder 来初始化上下文
* @param builder
*/
public RequestContext(Request.Builder builder) {
this.builder = builder;
List<String> nvList = builder.getHeaders$okhttp().getNamesAndValues$okhttp();
Map<String, String> headerMap = new HashMap<>();
@@ -26,21 +25,20 @@ public class RequestContext {
}
this.headers = headerMap;
String url = builder.getUrl$okhttp().toString();
this.scheme = url.startsWith("https") ? "https" : "http";
this.host = url.replaceFirst("https?://", "").split("/")[0];
if (builder.getUrl$okhttp() != null) {
String url = builder.getUrl$okhttp().toString();
this.scheme = url.startsWith("https") ? "https" : "http";
this.host = url.replaceFirst("https?://", "").split("/")[0];
this.url = url;
}
}
/**
* 使用指定 headers、scheme 和 host 来初始化上下文,一般用在单元测试场景
* @param headers
* @param scheme
* @param host
* 使用 OkHttp Request 来初始化上下文
* @param builder
*/
public RequestContext(Map<String, String> headers, String scheme, String host) {
this.builder = null;
this.headers = headers;
this.scheme = scheme;
this.host = host;
public RequestContext(Request request) {
this(request.newBuilder());
}
}

View File

@@ -0,0 +1,47 @@
package quant.rich.emoney.pojo.dto;
import java.util.List;
import lombok.Data;
import lombok.experimental.Accessors;
public interface IndexDetail {
/**
* 获取 indexName
* @return
*/
public String getIndexName();
/**
* 获取 indexCode
* @return
*/
public String getIndexCode();
/**
* 详情接口
* @return
*/
public List<Detail> getDetails();
/**
* 清洗 HTML 内容
*/
public void sanitize();
@Data
@Accessors(chain=true)
public static class Detail {
private String content;
private DetailType type;
}
public static enum DetailType {
TITLE,
TEXT,
IMAGE
}
}

View File

@@ -35,7 +35,7 @@ public class IpInfo {
if (StringUtils.isNoneBlank(getIpv6())) {
sb.append(", ").append("IPv6: ").append(getIpv6());
}
sb.append(getGeoString());
sb.append('(').append(getGeoString()).append(')');
return sb.toString();
}
@@ -45,7 +45,6 @@ public class IpInfo {
public String getGeoString() {
StringBuilder sb = new StringBuilder();
sb.append("(");
if (StringUtils.isNotBlank(getCountry())) {
// 国家
sb.append(getCountry());
@@ -60,7 +59,6 @@ public class IpInfo {
else {
sb.append("未知位置");
}
sb.append(")");
return sb.toString();
}

View File

@@ -0,0 +1,101 @@
package quant.rich.emoney.pojo.dto;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Data;
import lombok.experimental.Accessors;
import quant.rich.emoney.util.HtmlSanitizer;
@Data
@Accessors(chain=true)
public class NonParamsIndexDetail implements IndexDetail {
@JsonView(IndexDetail.class)
private String id;
@JsonView(IndexDetail.class)
private String name;
@JsonView(IndexDetail.class)
private String nameCode;
@JsonView(IndexDetail.class)
private List<NonParamsIndexDetailData> data = new ArrayList<>();
@Data
@Accessors(chain=true)
public static class NonParamsIndexDetailData {
@JsonView(IndexDetail.class)
private String title;
@JsonView(IndexDetail.class)
private List<String> items;
@JsonView(IndexDetail.class)
private String image;
}
@Override
public String getIndexName() {
return name;
}
@Override
public String getIndexCode() {
return nameCode;
}
@Override
public List<Detail> getDetails() {
List<Detail> list = new ArrayList<>();
data.forEach(d -> {
if (StringUtils.isNotBlank(d.getTitle())) {
list.add(new Detail().setContent(d.getTitle()).setType(DetailType.TITLE));
}
if (!CollectionUtils.isEmpty(d.getItems())) {
d.getItems().forEach(i -> {
list.add(new Detail().setContent(i).setType(DetailType.TEXT));
});
}
String image = d.getImage();
if (StringUtils.isNotBlank(image)) {
// 判断是否是合法 base64 图片,如果不是,生成警告
if (!image.startsWith("data:image/") || !image.contains("base64")) {
list.add(new Detail()
.setContent(
new StringBuilder()
.append("配图 url: ")
.append(image)
.append(" 无法获取其地址,请检查或重试")
.toString()
)
.setType(DetailType.TEXT));
}
else {
list.add(new Detail().setContent(image).setType(DetailType.IMAGE));
}
}
});
return list;
}
@Override
public void sanitize() {
for (NonParamsIndexDetailData data : getData()) {
String title = data.getTitle();
if (StringUtils.isNotBlank(title)) {
data.setTitle(HtmlSanitizer.sanitize(title));
}
if (data.getItems() != null) {
List<String> items = new ArrayList<>();
for (String item : data.getItems()) {
items.add(HtmlSanitizer.sanitize(item));
}
data.setItems(items);
}
}
}
}

View File

@@ -0,0 +1,50 @@
package quant.rich.emoney.pojo.dto;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonView;
import lombok.Data;
import lombok.experimental.Accessors;
import quant.rich.emoney.util.HtmlSanitizer;
@Data
@Accessors(chain=true)
public class ParamsIndexDetail implements IndexDetail {
@JsonView(IndexDetail.class)
private String id;
@JsonView(IndexDetail.class)
private String name;
@JsonView(IndexDetail.class)
private String code;
@JsonView(IndexDetail.class)
private List<String> descriptions = new ArrayList<>();
@Override
public String getIndexName() {
return name;
}
@Override
public String getIndexCode() {
return code;
}
@Override
public List<Detail> getDetails() {
List<Detail> list = new ArrayList<>();
descriptions.forEach(des -> {
list.add(new Detail().setContent(des).setType(DetailType.TEXT));
});
return list;
}
@Override
public void sanitize() {
List<String> descriptions = new ArrayList<>();
for (String description : getDescriptions()) {
descriptions.add(HtmlSanitizer.sanitize(description));
}
setDescriptions(descriptions);
}
}

View File

@@ -236,9 +236,8 @@ public class ConfigService implements InitializingBean {
String field = fieldClassCache.inverse().get(config.getClass());
ConfigInfo info = getConfigInfoByField(field);
String configJoString;
SmartViewWriter writer = new SmartViewWriter();
configJoString = writer.writeWithSmartView(config, IConfig.Views.Persistence.class);
String configJoString = writer.writeWithSmartView(config, IConfig.Views.Persistence.class);
if (info.save()) {
try {

View File

@@ -0,0 +1,536 @@
package quant.rich.emoney.service;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Value;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import quant.rich.emoney.client.OkHttpClientProvider;
import quant.rich.emoney.entity.config.EmoneyRequestConfig;
import quant.rich.emoney.entity.config.IndexInfoConfig;
import quant.rich.emoney.entity.config.SmartViewWriter;
import quant.rich.emoney.pojo.dto.IndexDetail;
import quant.rich.emoney.pojo.dto.NonParamsIndexDetail;
import quant.rich.emoney.pojo.dto.NonParamsIndexDetail.NonParamsIndexDetailData;
import quant.rich.emoney.util.EncryptUtils;
import quant.rich.emoney.util.SpringContextHolder;
import quant.rich.emoney.pojo.dto.ParamsIndexDetail;
/**
* 获取指标详情的服务
*/
@Slf4j
@Service
public class IndexDetailService {
@Autowired
IndexInfoConfig indexInfoConfig;
@Autowired
EmoneyRequestConfig emoneyRequestConfig;
static final String filePath = "./conf/extra/indexDetail/";
static final ObjectMapper mapper = new ObjectMapper();
static final Context jsContext = Context.create("js");
static final Pattern nonParamsIndexDetailPattern = Pattern.compile("\\[(\\{(id:\\d+,?|data:\\[\\{(title:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|items:\\[(\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?)+],?|image:\".*?\",?)+\\}+\\],?|name:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|nameCode:\"\\d+\",?)+\\},?)+\\]", Pattern.DOTALL);
static {
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
/**
* 强制刷新并重新获取
* @param indexCode
* @return
*/
@CacheEvict(cacheNames="@indexDetailService.getIndexDetail(Serializable)", key="#indexCode.toString()")
public IndexDetail forceRefreshAndGetIndexDetail(Serializable indexCode) {
Path path = getIndexDetailPath(indexCode);
try {
Files.deleteIfExists(path);
} catch (IOException e) {
String msg = MessageFormat.format("本地 IndexDetail 文件删除失败path: {0}, msg: {1}", path.toString(), e.getLocalizedMessage());
throw new RuntimeException(msg, e);
}
return getIndexDetail(indexCode);
}
@Cacheable(cacheNames="@indexDetailService.getIndexDetail(Serializable)", key="#indexCode.toString()")
public IndexDetail getIndexDetail(Serializable indexCode) {
if (indexCode == null) {
throw new NullPointerException("indexCode 不能为空");
}
if (!hasParams(indexCode)) {
return getNonParamsIndexDetail(indexCode);
}
else {
return getParamsIndexDetail(indexCode);
}
}
private ParamsIndexDetail getParamsIndexDetail(Serializable indexCode) {
// 先判断本地有没有
Path localFilePath = getIndexDetailPath(indexCode);
if (Files.exists(localFilePath)) {
ParamsIndexDetail detail = null;
try {
String str = Files.readString(localFilePath);
detail = mapper.readValue(str, ParamsIndexDetail.class);
} catch (IOException e) {
log.warn("无法获取本地无参数指标说明,将尝试重新从网络获取 indexCode: {}", indexCode, e);
}
if (detail != null) {
return detail;
}
}
// 从网络获取
return getParamsIndexDetailOnline(indexCode);
}
private ParamsIndexDetail getParamsIndexDetailOnline(Serializable indexCode) {
try {
OkHttpClient client = OkHttpClientProvider.getInstance();
String url = "https://emapp.emoney.cn/Config/AppIndicator/Get";
byte[] content = new StringBuilder().append("{\"code\":\"").append(indexCode).append("\"}").toString().getBytes();
MediaType type = MediaType.parse("application/json");
RequestBody body = RequestBody.create(content, type);
Request request = new Request.Builder()
.url(url)
.post(body)
.header("X-Protocol-Id", "Config%2FAppIndicator%2FGet")
.header("X-Request-Id", "null")
.header("EM-Sign", EncryptUtils.getEMSign(content, "POST", "Config%2FAppIndicator%2FGet"))
.header("Authorization", emoneyRequestConfig.getAuthorization())
.header("X-Android-Agent", emoneyRequestConfig.getXAndroidAgent())
.header("Emapp-ViewMode", emoneyRequestConfig.getEmappViewMode())
.build();
final Response response = client.newCall(request).execute();
String responseText = response.body().string();
ObjectNode result = (ObjectNode)mapper.readTree(responseText);
Integer code = result.get("result").get("code").asInt();
if (code == 0) {
ParamsIndexDetail detail = mapper.treeToValue(result.get("detail"), ParamsIndexDetail.class);
if (detail == null) {
// 网络访问成功但为 null, 新建一空 detail
detail = new ParamsIndexDetail();
detail.setCode(indexCode.toString());
detail.getDescriptions().add("该指标说明接口返回为空");
}
// 清洗内容:凡是文本类型的内容的,都要清洗一遍,判断是否有脚本、
// 视频和图片等,有的要清除并记录,以免有泄露网址、客户端 IP 风险
detail.sanitize();
saveIndexDetail(detail);
return detail;
}
else {
log.error("网络获取 ParamsIndexDetail 失败code: {}, msg: {}, indexCode: {}",
code, result.get("result").get("msg").asText(),
indexCode);
return null;
}
}
catch (Exception e) {
String message = MessageFormat.format("网络获取 ParamsIndexDetail 失败indexCode: {0}message: {1}",
indexCode, e.getLocalizedMessage());
log.error(message, e);
throw new RuntimeException(message, e);
}
}
/**
* 获取不带参数的指标信息,通过模拟 Webview 获取
* @param indexCode
* @return
*/
private NonParamsIndexDetail getNonParamsIndexDetail(Serializable indexCode) {
// 先判断本地有没有
Path localFilePath = getIndexDetailPath(indexCode);
if (Files.exists(localFilePath)) {
NonParamsIndexDetail detail = null;
try {
String str = Files.readString(localFilePath);
detail = mapper.readValue(str, NonParamsIndexDetail.class);
}
catch (IOException e) {
log.warn("无法获取本地无参数指标说明,将尝试重新从网络获取 indexCode: {}", indexCode, e);
}
if (detail != null) {
loadImages(detail);
saveIndexDetail(detail);
return detail;
}
}
// 从网络获取
return getNonParamsIndexDetailOnline(indexCode);
}
/**
* 从网络获取指定 indexCode 的无参指标详情
* <p>会一并尝试获取其他在本地未有的无参指标</p>
* @param indexCode
* @return
*/
private NonParamsIndexDetail getNonParamsIndexDetailOnline(Serializable indexCode) {
String url = buildNonParamsIndexUrl(indexCode);
Request request = new Request.Builder()
.url(url)
.header("Host", "appstatic.emoney.cn")
.header("Connection", "keep-alive")
.header("Upgrade-Insecure-Requests", "1")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "none")
.header("Sec-Fetch-Mode", "navigate")
.header("Sec-Fetch-User", "?1")
.header("Sec-Fetch-Dest", "document")
.header("Accept-Encoding", "gzip, deflate")
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7")
.build();
OkHttpClient client = OkHttpClientProvider.getInstance();
List<String> scripts = new ArrayList<>();
// 1. 获取页面内 scripts 列表,需要的信息都包含在里面
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.error("NonParamsIndexDetail 获取失败status code: {}, indexCode: {}", response.code(), indexCode);
return null;
}
String responseBody = response.body().string();
Document doc = Jsoup.parseBodyFragment(responseBody);
doc.select("script[src]").forEach(el -> {
String absoluteURI = resolveUrl(url, el.attr("src"));
log.info("script uri: {}", absoluteURI);
if (absoluteURI != null) {
scripts.add(absoluteURI);
}
});
} catch (IOException e) {
String msg = MessageFormat.format("NonParamsIndexDetail 获取失败指标页访问失败indexCode: {0}", indexCode);
log.error(msg, e);
throw new RuntimeException(msg, e);
}
if (scripts.size() == 0) {
String msg = MessageFormat.format("NonParamsIndexDetail 获取失败未能获取基本页面内脚本链接indexCode: {0}", indexCode);
log.error(msg);
throw new RuntimeException(msg);
}
// 2. 加载 scripts试图匹配包含在 js 中的指标说明
Request.Builder scriptBuilder = new Request.Builder()
.header("Host", "appstatic.emoney.cn")
.header("Connection", "keep-alive")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "*/*")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "same-origin")
.header("Sec-Fetch-Mode", "no-cors")
.header("Sec-Fetch-Dest", "script")
.header("Accept-Encoding", "gzip, deflate")
.header("Referer", url)
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
List<NonParamsIndexDetail> valids = new ArrayList<>();
scriptLoop:
for (String scriptUrl : scripts) {
Request scriptRequest = scriptBuilder.url(scriptUrl).build();
try (Response response = client.newCall(scriptRequest).execute()) {
if (!response.isSuccessful()) {
log.warn("NonParamsIndexDetail 获取失败:读取页面内 script 请求失败, response.isSuccessful() is false, indexCode: {}, scriptUrl: {}", indexCode, scriptUrl);
continue;
}
String responseBody = response.body().string();
// 尝试匹配
Matcher m = nonParamsIndexDetailPattern.matcher(responseBody);
while (m.find()) {
String find = m.group();
Optional<String> jsonStringify = stringifyJsArray(find);
if (jsonStringify.isPresent()) {
String jsonString = jsonStringify.get();
JsonNode root = mapper.readTree(jsonString);
boolean foundAny = false;
if (root.isArray()) {
ArrayNode array = (ArrayNode) root;
for (JsonNode obj : array) {
if (obj.isObject() &&
obj.has("name") &&
obj.get("name").isTextual() &&
obj.has("nameCode") && obj.get("nameCode").isTextual() &&
obj.has("data") && obj.get("data").isArray()) {
NonParamsIndexDetail detail = mapper.treeToValue(obj, NonParamsIndexDetail.class);
valids.add(detail);
foundAny = true;
}
}
}
if (foundAny) {
break scriptLoop;
}
}
}
} catch (IOException e) {
log.warn("NonParamsIndexDetail 获取失败:读取页面内 script 请求失败, response.isSuccessful() is true, indexCode: {}, scriptUrl: {}", indexCode, scriptUrl, e);
continue;
}
}
if (valids.isEmpty()) {
String msg = MessageFormat.format(
"NonParamsIndexDetail 获取失败:未从页面 js 脚本中获取到任何可能匹配的参数说明indexCode: {}",
indexCode);
log.warn(msg);
throw new RuntimeException(msg);
}
// 直接保存在本地
Pattern numericPattern = Pattern.compile("^\\d+$");
NonParamsIndexDetail targetDetail = null;
for (NonParamsIndexDetail detail : valids) {
// 判断 nameCode 是否是合法的代码
if (!numericPattern.matcher(detail.getNameCode()).matches()) {
continue;
}
Path path = getIndexDetailPath(detail);
// 判断是否是需求的 detail
if (indexCode.toString().equals(detail.getIndexCode())) {
loadImages(detail);
targetDetail = detail;
}
// 清洗内容:凡是文本类型的内容的,都要清洗一遍,判断是否有脚本、
// 视频和图片等,有的要清除并记录,以免有泄露网址、客户端 IP 风险
detail.sanitize();
if (!Files.exists(path)) {
// 不存在则保存
saveIndexDetail(detail);
}
}
if (targetDetail == null) {
// 创建一空 target
targetDetail = new NonParamsIndexDetail();
targetDetail.setNameCode(indexCode.toString());
NonParamsIndexDetailData data = new NonParamsIndexDetailData();
List<String> items = List.of("该指标说明接口返回为空");
data.setItems(items);
targetDetail.getData().add(data);
Path path = getIndexDetailPath(targetDetail);
if (!Files.exists(path)) {
// 不存在则保存
saveIndexDetail(targetDetail);
}
}
return targetDetail;
}
/**
* 为指定 NonParamsIndexDetail 加载在线图片并转换成 base64
* <p>该方法涉及 authorization(token)/webviewUserAgent需确保 EmoneyRequestConfig 已正确注入并登录</p>
* @param detail
* @return
* @see EmoneyRequestConfig
*/
private NonParamsIndexDetail loadImages(NonParamsIndexDetail detail) {
OkHttpClient client = OkHttpClientProvider.getInstance();
for (NonParamsIndexDetailData data : detail.getData()) {
String imageUrl = data.getImage();
if (StringUtils.isNotBlank(imageUrl)) {
if (imageUrl.startsWith("data:image/") && imageUrl.contains("base64")) {
continue;
}
else if (imageUrl.toLowerCase().startsWith("http")) {
// 需要转换
URI uri;
try {
uri = new URI(imageUrl);
}
catch (URISyntaxException e) {
log.warn("未经转换的 NonParamsIndexDetailData.image {} 非合法 http(s) 地址,请检查", imageUrl, e);
continue;
}
String host = uri.getHost();
// 从网络获取
Request request = new Request.Builder()
.url(imageUrl)
.header("Host", host)
.header("Connection", "keep-alive")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "same-origin")
.header("Sec-Fetch-Mode", "no-cors")
.header("Sec-Fetch-Dest", "image")
.header("Accept-Encoding", "gzip, deflate")
.header("Referer", buildNonParamsIndexUrl(detail.getIndexCode()))
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7")
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.warn("未经转换的 NonParamsIndexDetailData.image {} 获取图片详情失败,请检查或重试", imageUrl);
continue;
}
byte[] bytes = response.body().bytes();
String contentType = response.body().contentType().toString();
String base64 = Base64.getEncoder().encodeToString(bytes);
data.setImage("data:" + contentType + ";base64," + base64);
}
catch (IOException e) {
log.warn("未经转换的 NonParamsIndexDetailData.image {} 获取图片详情失败,请检查或重试", imageUrl, e);
continue;
}
}
else {
log.warn("未知的 NonParamsIndexDetailData.image 格式 {}", imageUrl);
}
}
}
return detail;
}
/**
* 保存指标详情到本地文件
* @param <Description>
* @param des
*/
private <Description extends IndexDetail> void saveIndexDetail(Description des) {
SmartViewWriter writer = new SmartViewWriter();
String joString = writer.writeWithSmartView(des, IndexDetail.class);
Path path = getIndexDetailPath(des);
try {
Files.writeString(path, joString);
}
catch (IOException e) {
log.error("写入指标详情到 {} 失败", path.toString(), e);
}
}
/**
* 获取指标详情本地路径 Path
* @param <Description>
* @param description
* @return
*/
private <Description extends IndexDetail> Path getIndexDetailPath(Description description) {
Path path = Path.of(new StringBuilder(filePath)
.append((description instanceof NonParamsIndexDetail) ? "nonParams/": "params/")
.append(description.getIndexCode())
.append(".json").toString());
return path;
}
/**
* 根据指标代码 indexCode 获取本地文件路径
* @param indexCode
* @return
*/
private Path getIndexDetailPath(Serializable indexCode) {
boolean hasParams = hasParams(indexCode);
Path path = Path.of(new StringBuilder(filePath)
.append(!hasParams ? "nonParams/": "params/")
.append(indexCode)
.append(".json").toString());
return path;
}
/**
* 根据指标代码 indexCode 判断是否有参数
* @param indexCode
* @return
*/
private boolean hasParams(Serializable indexCode) {
JsonNode configIndOnline = indexInfoConfig.getConfigIndOnline();
JsonNode jsonNode = configIndOnline.get("indMap").get(indexCode.toString());
if (jsonNode == null) {
throw new RuntimeException("无法根据指定 indexCode: " + indexCode + " 找到对应 indMap 内容");
}
ObjectNode node = (ObjectNode)jsonNode;
return node.has("indParam");
}
/**
* 获取 NonParamsIndexDetail URL
* <p>该 Url 涉及 authorization(token),需确保 EmoneyRequestConfig 已正确注入并登录</p>
* @param indexCode
* @return
* @see EmoneyRequestConfig
*/
private String buildNonParamsIndexUrl(Serializable indexCode) {
StringBuilder urlBuilder = new StringBuilder();
urlBuilder.append("https://appstatic.emoney.cn/html/emapp/stock/note/?name=");
urlBuilder.append(indexCode.toString());
urlBuilder.append("&emoneyScaleType=0&emoneyLandMode=0&token=");
urlBuilder.append(emoneyRequestConfig.getAuthorization());
return urlBuilder.toString();
}
/**
* 根据页面解析绝对路径
*/
private static String resolveUrl(String pageUrl, String src) {
try {
URI base = URI.create(pageUrl);
return base.resolve(src).toString();
} catch (Exception e) {
log.warn("转换 URI 错误pageUrl: {}, scriptSrc: {} {}", pageUrl, src, e);
return null;
}
}
/**
* 使用 GraalVM 的 JS 引擎尝试 JSON.stringify
* @param jsArrayText
* @return
*/
private static Optional<String> stringifyJsArray(String jsArrayText) {
try {
Value result = jsContext.eval("js", "JSON.stringify(" + jsArrayText + ")");
return result.isString() ? Optional.of(result.asString()) : Optional.empty();
} catch (Exception e) {
log.warn("Failed to call JSON.stringify on {}", jsArrayText, e);
return Optional.empty();
}
}
}

View File

@@ -118,6 +118,7 @@ public class GeoIPUtil {
} catch (IOException | GeoIp2Exception e) {
log.warn("Get city response from GeoLite2-City database failed: {}", e.getMessage());
}
log.debug("ipInfo: {}", ipInfo);
return ipInfo;
}

View File

@@ -0,0 +1,134 @@
package quant.rich.emoney.util;
import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;
import java.util.Iterator;
import java.util.Random;
public class HtmlSanitizer {
public static final Random random = new Random();
public static String sanitize(String unsafeHtml) {
String nlRandom =
new StringBuilder("__NL").append(random.nextInt()).append("__").toString();
unsafeHtml = unsafeHtml.replaceAll("\n", nlRandom);
Document doc = Jsoup.parseBodyFragment(unsafeHtml);
Element body = doc.body();
NodeTraversor.traverse(new NodeVisitor() {
public void head(Node node, int depth) {
if (!(node instanceof Element)) return;
Element el = (Element) node;
String tag = el.tagName().toLowerCase();
// 清洗 <link rel="stylesheet" href="...">
if (tag.equals("link")) {
String href = el.attr("href");
if (isExternalUrl(href)) {
replaceWithNotice(el, "link[href]");
return;
}
}
// 清洗危险结构元素
switch (tag) {
case "script":
case "iframe":
case "object":
case "embed":
case "video":
case "audio":
case "source":
case "meta":
case "fencedframe":
replaceWithNotice(el, tag);
return;
}
// 清除 on* 属性
Iterator<Attribute> it = el.attributes().iterator();
while (it.hasNext()) {
Attribute attr = it.next();
if (attr.getKey().toLowerCase().startsWith("on")) {
it.remove();
}
}
// 清除 src 外链
if (el.hasAttr("src")) {
String src = el.attr("src");
if (isExternalUrl(src)) {
el.removeAttr("src");
el.appendChild(new TextNode("[已移除外链资源]"));
}
}
// 清除 style="...url(...)" 中的非 base64 URL
if (el.hasAttr("style")) {
String style = el.attr("style");
if (containsExternalUrl(style)) {
style = style.replaceAll("(?i)url\\((?!['\"]?data:)[^)]*\\)", "");
el.attr("style", style + " /* 外链背景图已清除 */");
}
}
// a 标签添加 rel="noreferrer"
if (tag.equals("a")) {
String rel = el.attr("rel");
if (!rel.toLowerCase().contains("noreferrer")) {
el.attr("rel", "noreferrer");
}
}
// style 标签内容清理 url(...)
if (tag.equals("style")) {
for (Node child : el.childNodes()) {
if (child instanceof TextNode) {
TextNode text = (TextNode) child;
String content = text.getWholeText();
// 移除非 base64 的 url(...)
String cleaned = content.replaceAll("(?i)url\\((?!['\"]?data:)[^)]*\\)", "/* 外链已清除 */");
text.text(cleaned);
}
}
}
}
public void tail(Node node, int depth) {
// nothing
}
}, body);
// SVG、MathML 类危险标签
doc.select("svg, math, foreignObject").forEach(el ->
replaceWithNotice(el, el.tagName())
);
unsafeHtml = body.html();
unsafeHtml = unsafeHtml.replaceAll(nlRandom, "\n");
return unsafeHtml;
}
private static boolean isExternalUrl(String url) {
return url != null
&& !url.trim().toLowerCase().startsWith("data:")
&& (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("//"));
}
private static boolean containsExternalUrl(String style) {
return style != null && style.toLowerCase().matches(".*url\\((?!['\"]?data:).+\\).*");
}
private static void replaceWithNotice(Element el, String tagName) {
el.replaceWith(new Element("div")
.addClass("blocked-element")
.text("" + tagName + " 元素可能包含危害安全的内容,已屏蔽。"));
}
}

View File

@@ -1,4 +1,5 @@
spring:
cache.type: simple
datasource:
postgre:
jdbc-url: jdbc:postgresql://localhost:5432/verich

View File

@@ -383,7 +383,17 @@ blockquote.layui-elem-quote {
box-shadow: 1px 1px 10px rgba(0, 0, 0, .1);
border-radius: 0;
}
.layui-layer>.layui-layer-content {
.layui-layer-indexDetail>.layui-layer-content>*:not(:last-child) {
margin-bottom: 1em;
}
.layui-layer-indexDetail>.layui-layer-content>* {
max-width: 100%;
}
.layui-layer-indexDetail>.layui-layer-content>img {
margin: auto;
display: block
}
.layui-layer-adminRight>.layui-layer-content {
overflow: visible !important;
}
.layui-anim-rl {
@@ -442,4 +452,15 @@ fieldset>legend:before {
font-weight: bold;
margin-right: .25em;
color: rgb(22, 183, 119)
}
.ipInfo {
color: #000
}
.ipInfo>dl {
max-width: 100px
}
.ipInfo .ipv6>a {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}

View File

@@ -1,14 +1,75 @@
if (!window.Helper) {window.Helper = {}}
if (!window.Helper) { window.Helper = {} }
window.Helper = {
emoneyPeriodToName: function(x) {
if (x < 10000) return `${x} 分钟`;
if (x == 10000) return '日线';
if (x == 20000) return '周线';
if (x == 30000) return '月线';
if (x == 40000) return '季线';
if (x == 50000) return '半年线';
if (x == 60000) return '年线';
},
allEmoneyPeriods: [1, 5, 15, 30, 60, 120, 10000, 20000, 30000, 40000, 50000, 60000]
emoneyPeriodToName: function(x) {
if (x < 10000) return `${x} 分钟`;
if (x == 10000) return '日线';
if (x == 20000) return '周线';
if (x == 30000) return '月线';
if (x == 40000) return '季线';
if (x == 50000) return '半年线';
if (x == 60000) return '年线';
},
allEmoneyPeriods: [1, 5, 15, 30, 60, 120, 10000, 20000, 30000, 40000, 50000, 60000],
showIndexDetailLayer: async function(obj, forceRefresh) {
// obj: {indexCode: _, indexName: _}
var layer = layui.layer;
var load = layer.load(2);
var url = !!forceRefresh ?
'/admin/v1/manage/indexInfo/forceRefreshAndGetIndexDetail?indexCode=' :
'/admin/v1/manage/indexInfo/getIndexDetail?indexCode=';
url += obj.indexCode;
var res = await (await fetch(url)).json();
if (res.ok) {
// build content
var html = [];
for (var i = 0; i < res.data.details.length; i++) {
var detail = res.data.details[i];
if (detail.type == 'TITLE') {
html.push('<h3>');
html.push(Helper.trimChars(detail.content, ':'));
html.push('</h3>');
}
else if (detail.type == 'IMAGE') {
if (detail.content.indexOf('data:image/') == 0 &&
detail.content.indexOf('base64') != -1) {
html.push('<img src="');
html.push(detail.content);
html.push('">');
}
}
else {
html.push('<p>');
html.push(detail.content.replaceAll(/\n+/g, '<br>'));
html.push('</p>');
}
}
console.log(res.data);
layer.open({
title: obj.indexName + '指标说明',
content: html.join(''),
skin: 'layui-layer-indexDetail',
area: ['520px', '320px'],
btn: ['刷新', '确定'],
btn1: function(index, layero, that) {
layer.close(index);
Helper.showIndexDetailLayer(obj, !0);
},
success: function(layero, index) {
var btns = layero.find('.layui-layer-btn>*');
btns[0].setAttribute('class', 'layui-layer-btn1');
btns[1].setAttribute('class', 'layui-layer-btn0');
}
})
}
else {
Dog.error({ msg: res && res.message || '服务器错误' });
}
layer.close(load);
},
trimChars: function (str, chars) {
const escaped = chars.split('').map(c => c.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('');
const pattern = new RegExp(`^[${escaped}]+|[${escaped}]+$`, 'g');
return str.replace(pattern, '');
}
}

View File

@@ -224,29 +224,34 @@
el.value = !!el.checked;
});
})
form.on('submit(submit)', function(data){
var field = data.field;
document.querySelectorAll('input[lay-skin="switch"]').forEach(checkbox => {
field[checkbox.getAttribute('name')] = checkbox.getAttribute('value') == 'true'
});
form.on('submit(submit)', async function(data){
$.ajax({
url: location.href,
data: JSON.stringify(field),
contentType: 'application/json',
method: 'POST',
success: function (result) {
Dog.success({
onClose: () => location.reload()
})
},
error: function (res) {
var r = res.responseJSON;
Dog.error({
msg: r && r.data,
})
}
layui.layer.confirm('若设备信息修改,会导致清空鉴权信息,确定吗?', async function() {
var load = layui.layer.load(2);
var field = data.field;
document.querySelectorAll('input[lay-skin="switch"]').forEach(checkbox => {
field[checkbox.getAttribute('name')] = checkbox.getAttribute('value') == 'true'
});
try {
var res = await (await fetch(location.href, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(field)
})).json();
if (res && res.ok) {
Dog.success({onClose: () => location.reload()});
}
else {
Dog.error({msg: res && res.data || '服务器错误'})
}
}
catch (e) {
Dog.error({msg: e})
}
layui.layer.close(load);
})
});
form.render();

View File

@@ -63,6 +63,20 @@
<dd><a th:href="@{/admin/v1/logout}">退出登录</a></dd>
</dl>
</li>
<li class="layui-nav-item ipInfo" style="float:right;margin-right: 1px" lay-unselect="">
<a id="ipThroughProxy" href="javascript:manualRefreshIp()" title="立即刷新">
IP 属地:
<span th:if="${@proxyConfig.ipInfo == null}" class="layui-badge layui-bg-cyan">加载中...</span>
<span th:if="${@proxyConfig.ipInfo != null}" class="layui-badge layui-bg-cyan">[[${@proxyConfig.ipInfo.geoString}]]</span>
</a>
<th:block th:if="${@proxyConfig.ipInfo != null}">
<dl class="layui-nav-child">
<dd class="ip"><a title="点击复制">[[${@proxyConfig.ipInfo.ip}]]</a></dd>
<dd class="ipv6" th:if="${@proxyConfig.ipInfo.ipv6 != null}"><a title="点击复制">[[${@proxyConfig.ipInfo.ipv6}]]</a></dd>
</dl>
</th:block>
</li>
</ul>
<script type="text/html" id="editUser">
<div class="layui-form" style="margin:10px 15px" id="editUserForm" lay-filter="editUserForm">
@@ -111,13 +125,47 @@
<script>
let refreshTimer = null;
document.querySelectorAll('dd.ip,dd.ipv6').forEach(el => {
el.addEventListener('click', () => {
let text = el.querySelector('a').textContent;
navigator.clipboard.writeText(text).then(() => {
Dog.success({msg: '复制成功'})
}).catch(err => {
Dog.error({msg: '复制失败'})
});
})
});
async function refreshIpThroughProxy() {
try {
let geoEl =
document.querySelector('#ipThroughProxy>span');
geoEl.textContent = '加载中...';
let res = await (await fetch('/admin/v1/config/proxy/refreshIpThroughProxy')).json();
if (res.ok) {
let ip = res.data.string || '获取代理 IP 失败';
document.getElementById('ipThroughProxy').textContent = ip;
geoEl.textContent = res.data.geoString || '获取失败';
let ipMenu = document.querySelector('.ipInfo>dl');
let genIpEL = (clazz, title) => {
let el = ipMenu.querySelector('.' + selector);
if (!el) {
el = document.createElement('dd');
let a = document.createElement('a');
a.setAttribute('title', title || '点击复制');
el.classList.add(clazz);
el.appendChild(a);
ipMenu.appendChild(el);
}
return el;
};
let ipEl = genIpEL('ip');
let ipv6El = ipMenu.querySelector('.ipv6');
ipEl.querySelector('a').textContent = res.data.ip;
if (res.data.ipv6) {
ipv6El = ipv6El || genIpEL('ipv6');
ipv6El.querySelector('a').textContent = res.data.ipv6;
}
else if (ipv6El) {
ipv6El.remove();
}
}
} catch (e) {
console.error('刷新失败:', e);
@@ -221,10 +269,6 @@
</script>
</th:block>
<div th:fragment="feet" class="layui-trans layadmin-user-login-footer">
<a id="ipThroughProxy" href="javascript:manualRefreshIp()" title="立即刷新">
[[${@proxyConfig.ipInfo == null ? '当前 IP 信息:加载中...' :
@proxyConfig.ipInfo.string}]]
</a><br>
Driven by Latte<br />
&copy;2025-[[${#dates.format(new java.util.Date().getTime(),'yyyy')}]]
<a href="#">Latte</a>

View File

@@ -67,8 +67,8 @@
cols: [ [
{field: 'indexCode', title: '指标代码'},
{field: 'indexName', title: '指标名称'},
{field: 'isCalc', title: '是否传统算法', templet: (d) => {
return d.isCalc ?
{field: 'isCalc', title: '是否在线指标', templet: (d) => {
return !d.isCalc ?
'<i class="op-green fa-solid fa-circle-check"></i>' :
'<i class="op-red fa-solid fa-circle-exclamation"></i>';
}},
@@ -102,13 +102,13 @@
}
}
})
table.on('tool(indexInfos)', (obj) => {
table.on('tool(indexInfos)', async function (obj) {
if (obj.event == 'detail') {
Helper.showIndexDetailLayer(obj.data);
}
})
})
});
</script>
<th:block th:replace="~{admin/v1/manage/indexInfo/updateForm::indexInfoUpdateForm}"></th:block>
</body>

View File

@@ -114,7 +114,32 @@
}
}
});
enableCodeMirrorResize(editor, document.getElementById("editor-wrapper"))
enableCodeMirrorResize(editor, document.getElementById("editor-wrapper"));
let submitBtn = document.querySelector('[lay-filter="submitIndexConfigFile"]');
submitBtn.addEventListener('click', async function(e) {
e.preventDefault();
let res = await (await fetch('/admin/v1/config/indexInfo', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
configIndOnlineUrl: document.querySelector('[name="configIndOnlineUrl"]').value,
configIndOnline: JSON.parse(editor.getValue())
})
})).json();
if (res.ok) {
Dog.success({
onClose: () => location.reload()
})
}
else {
Dog.error({
msg: res && res.data,
})
}
})
}
})
}
@@ -160,5 +185,6 @@
document.body.style.cursor = "default";
});
}
</script>
</th:block>

View File

@@ -25,6 +25,9 @@
<option value="">选择更新指标</option>
</select>
</div>
<div class="layui-inline">
<a style="display:none" href="#" id="showIndexDetailLayer" onclick="javascript:showIndexDetailLayer(this, event)"><i class="fa fa-solid fa-circle-question"></i></a>
</div>
</div>
</div>
<div id="params">
@@ -112,6 +115,13 @@
}
})
})
function showIndexDetailLayer(el, event) {
event.preventDefault();
Helper.showIndexDetailLayer({
indexCode: el.dataset.indexCode,
indexName: el.dataset.indexName
});
}
function openEditForm(r) {
if (r && r.ok) {
window.editLayer = layui.layer.open({
@@ -149,11 +159,21 @@
cssPath: '/admin/v1/static/css/cron.css'
});
layui.form.on('select(indexCodeFilter)', async function (obj) {
const paramsEl = document.getElementById('params');
const periodsEl = document.getElementById('periods');
paramsEl.textContent = periodsEl.textContent = '';
const paramsEl = document.getElementById('params'); // 参数选择控件
const periodsEl = document.getElementById('periods'); // 时间粒度选择控件
paramsEl.textContent = periodsEl.textContent = ''; // 清除参数和时间粒度选择控件
const dataset = obj.elem.querySelector(`[value="${obj.value}"]`).dataset;
const detailTriggerEl = document.getElementById('showIndexDetailLayer');
if (!dataset || !dataset.indInfo) {
// 未存在 dataset/.indInfo, 则去除可能存在的指标详情按钮
detailTriggerEl.style.display = 'none';
return;
}
detailTriggerEl.style.display = '';
detailTriggerEl.dataset.indexCode = obj.value;
detailTriggerEl.dataset.indexName = dataset.indName;
const indInfo =
JSON.parse(obj.elem.querySelector(`[value="${obj.value}"]`).dataset.indInfo);
JSON.parse(dataset.indInfo);
const paramTemplet = document.getElementById('paramTemplet');
const periodsTemplet = document.getElementById('periodsTemplet');
if (indInfo.indParam) {
@@ -211,6 +231,7 @@
option.textContent = `${key} ${jo.indIdNameConfig[key]}`;
option.selected = r.data.indexCode == key;
option.dataset.indInfo = JSON.stringify(indInfo);
option.dataset.indName = jo.indIdNameConfig[key];
optionsFragment.appendChild(option);
})

View File

@@ -27,7 +27,7 @@ class EmoneyAutoApplicationTests {
@Test
void contextLoads() {
EmoneyClient.loginWithAnonymous();
EmoneyClient.relogin();
CandleStickWithIndex_Request request = new CandleStickWithIndex_Request();

View File

@@ -0,0 +1,285 @@
package quant.rich;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Value;
import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import jodd.jerry.Jerry;
import lombok.extern.slf4j.Slf4j;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import quant.rich.emoney.EmoneyAutoApplication;
import quant.rich.emoney.client.EmoneyClient;
import quant.rich.emoney.client.OkHttpClientProvider;
import quant.rich.emoney.entity.config.EmoneyRequestConfig;
import quant.rich.emoney.pojo.dto.NonParamsIndexDetail;
import quant.rich.emoney.pojo.dto.NonParamsIndexDetail.NonParamsIndexDetailData;
@SpringBootTest
@ContextConfiguration(classes = EmoneyAutoApplication.class)
@RunWith(SpringJUnit4ClassRunner.class)
@Slf4j
public class EmoneyIndexScraper {
private static final ObjectMapper MAPPER = new ObjectMapper();
@Autowired
EmoneyRequestConfig emoneyRequestConfig;
static {
MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
public String buildUrl(Serializable indexCode) {
StringBuilder urlBuilder = new StringBuilder();
urlBuilder.append("https://appstatic.emoney.cn/html/emapp/stock/note/?name=");
urlBuilder.append(indexCode.toString());
urlBuilder.append("&emoneyScaleType=0&emoneyLandMode=0&token=");
urlBuilder.append(emoneyRequestConfig.getAuthorization());
return urlBuilder.toString();
}
@Test
void test() {
EmoneyClient.relogin();
String url = buildUrl(10002800);
// 1.1 按照顺序放置 Header
Request request = new Request.Builder()
.url(url)
.header("Host", "appstatic.emoney.cn")
.header("Connection", "keep-alive")
.header("Upgrade-Insecure-Requests", "1")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "none")
.header("Sec-Fetch-Mode", "navigate")
.header("Sec-Fetch-User", "?1")
.header("Sec-Fetch-Dest", "document")
.header("Accept-Encoding", "gzip, deflate")
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7")
.build();
// 1.2 请求
OkHttpClient client = OkHttpClientProvider.getInstance();
List<String> scripts = new ArrayList<>();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.error("1.2 请求指标说明基本页面失败response.isSuccessful() is false");
return;
}
String responseBody = response.body().string();
Jerry jerryDoc = Jerry.of(responseBody);
jerryDoc.s("script[src]").forEach(el -> {
String absoluteURI = resolveUrl(url, el.attr("src"));
log.info("script uri: {}", absoluteURI);
if (absoluteURI != null) {
scripts.add(absoluteURI);
}
});
} catch (IOException e) {
// TODO Auto-generated catch block
log.error("1.2 请求指标说明基本页面失败IOException", e);
e.printStackTrace();
return;
}
if (scripts.size() == 0) {
log.error("未能获取基本页面内脚本链接");
return;
}
// 2. 加载 scripts试图匹配包含在 js 中的指标说明
Pattern p = Pattern.compile("\\[(\\{(id:\\d+,?|data:\\[\\{(title:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|items:\\[(\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?)+],?|image:\".*?\",?)+\\}+\\],?|name:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|nameCode:\"\\d+\",?)+\\},?)+\\]", Pattern.DOTALL);
List<String> matchGroups = new ArrayList<>();
Request.Builder scriptBuilder = new Request.Builder()
.header("Host", "appstatic.emoney.cn")
.header("Connection", "keep-alive")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "*/*")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "same-origin")
.header("Sec-Fetch-Mode", "no-cors")
.header("Sec-Fetch-Dest", "script")
.header("Accept-Encoding", "gzip, deflate")
.header("Referer", url)
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
for (String scriptUrl : scripts) {
Request scriptRequest = scriptBuilder.url(scriptUrl).build();
try (Response response = client.newCall(scriptRequest).execute()) {
if (!response.isSuccessful()) {
log.warn("2. 读取页面内 script 请求失败, response.isSuccessful() is false, scriptUrl: {}", scriptUrl);
return;
}
String responseBody = response.body().string();
// 尝试匹配
Matcher m = p.matcher(responseBody);
while (m.find()) {
String find = m.group();
Optional<String> jsonStringify = stringifyJsArray(find);
jsonStringify.ifPresent(jsonString -> matchGroups.add(jsonString));
}
} catch (IOException e) {
// TODO Auto-generated catch block
log.warn("1.2 请求失败", e);
e.printStackTrace();
return;
}
}
// 将每个 jsonString 转换为 jsonArray进一步转换成 IndexDetail
List<NonParamsIndexDetail> valid = new ArrayList<>();
List<ArrayNode> arrayNodes = new ArrayList<>();
for (String jsonString : matchGroups) {
try {
JsonNode root = MAPPER.readTree(jsonString);
if (root.isArray()) {
ArrayNode array = (ArrayNode) root;
for (JsonNode obj : array) {
if (obj.isObject() && matches(obj)) {
NonParamsIndexDetail detail = MAPPER.treeToValue(obj, NonParamsIndexDetail.class);
valid.add(detail);
}
}
}
} catch (Exception ignored) {
log.error("试图转换匹配项出错", ignored);
}
}
// 直接保存在本地
// 遍历 valid 内合法的 object, 获取 image
Pattern numericPattern = Pattern.compile("^\\d+$");
for (NonParamsIndexDetail detail : valid) {
// 判断 nameCode 是否是合法的代码
if (!numericPattern.matcher(detail.getNameCode()).matches()) {
continue;
}
Request.Builder imageBuilder = new Request.Builder()
.header("Host", "appstatic.emoney.cn")
.header("Connection", "keep-alive")
.header("User-Agent", emoneyRequestConfig.getWebviewUserAgent())
.header("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
.header("X-Request-With", "cn.emoney.emstock")
.header("Sec-Fetch-Site", "same-origin")
.header("Sec-Fetch-Mode", "no-cors")
.header("Sec-Fetch-Dest", "image")
.header("Accept-Encoding", "gzip, deflate")
.header("Referer", buildUrl(detail.getNameCode()))
.header("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
List<NonParamsIndexDetailData> datas = detail.getData();
for (NonParamsIndexDetailData data : datas) {
if (data.getImage() != null) {
String image = data.getImage();
if (image.startsWith("data:image/") && image.contains(";base64,")) {
continue;
}
else {
image = resolveUrl(url, image);
Request imageRequest = imageBuilder.url(image).build();
try (Response response = client.newCall(imageRequest).execute()) {
if (!response.isSuccessful()) {
log.warn("获取详情图片请求失败, response.isSuccessful() is false, imageUrl: {}", image);
continue;
}
byte[] bytes = response.body().bytes();
String contentType = response.body().contentType().toString();
String base64 = Base64.getEncoder().encodeToString(bytes);
data.setImage("data:" + contentType + ";base64," + base64);
log.debug("获取图片成功");
} catch (IOException e) {
// TODO Auto-generated catch block
log.warn("获取详情图片请求失败", e);
e.printStackTrace();
return;
}
}
}
}
try {
Files.writeString(
Path.of("./conf/extra/nonParamsIndexDetail." + detail.getNameCode() + ".json"),
MAPPER.valueToTree(detail).toPrettyString());
} catch (IllegalArgumentException | IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return;
}
/**
* 判断某 jsonNode 是否为合法指标信息
* @param obj
* @return
*/
public static boolean matches(JsonNode obj) {
return obj.has("name") && obj.get("name").isTextual() &&
obj.has("nameCode") && obj.get("nameCode").isTextual() &&
obj.has("data") && obj.get("data").isArray()
;
}
/**
* 使用 GraalVM JS 尝试 JSON.stringify
* @param jsArrayText
* @return
*/
public static Optional<String> stringifyJsArray(String jsArrayText) {
try (Context context = Context.create("js")) {
Value result = context.eval("js", "JSON.stringify(" + jsArrayText + ")");
return result.isString() ? Optional.of(result.asString()) : Optional.empty();
} catch (Exception e) {
log.warn("Failed to call JSON.stringify on {}", jsArrayText, e);
return Optional.empty();
}
}
public static String resolveUrl(String pageUrl, String scriptSrc) {
try {
URI base = URI.create(pageUrl);
return base.resolve(scriptSrc).toString();
} catch (Exception e) {
log.info("转换 URI 错误pageUrl: {}, scriptSrc: {} {}", pageUrl, scriptSrc, e);
return null;
}
}
}

View File

@@ -1,12 +1,11 @@
package quant.rich;
import java.lang.reflect.Field;
import java.lang.reflect.Type;
import java.util.Map;
import java.util.function.Consumer;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import okhttp3.Request;
import quant.rich.emoney.patch.okhttp.RequestContext;
import quant.rich.emoney.patch.okhttp.PatchOkHttp;
import quant.rich.emoney.patch.okhttp.PatchOkHttpRule;
@@ -20,7 +19,11 @@ public class PatchOkHttpTest {
RequestContext context;
rule = PatchOkHttpRule.when().isHttps().build();
context = new RequestContext(Map.of(), "https", "localhost");
context = new RequestContext(
new Request.Builder()
.url("https://localhost"));
Assertions.assertTrue(rule.matches(context), "测试失败");
@@ -30,11 +33,14 @@ public class PatchOkHttpTest {
.or(b -> b.hasHeaderName("X-Protocol-Id"))
.overrideIf("User-Agent", "okhttp/3.12.2")
.build();
context = new RequestContext(Map.of(), "https", "mbs.emoney.cn");
context = new RequestContext(new Request.Builder()
.url("https://mbs.emoney.cn"));
Assertions.assertTrue(rule.matches(context), "测试失败");
context = new RequestContext(Map.of(), "https", "emapp-static.oss-cn-shanghai.aliyuncs.com");
context = new RequestContext(new Request.Builder()
.url("https://emapp-static.oss-cn-shanghai.aliyuncs.com"));
Assertions.assertTrue(rule.matches(context), "测试失败");
// 测试 Override
@@ -42,13 +48,17 @@ public class PatchOkHttpTest {
Consumer<String> consumer = str -> {
modifier[0] = str;
};
PatchOkHttp.apply(rule);
PatchOkHttp.match(context, "User-Agent", consumer);
Assertions.assertTrue("okhttp/3.12.2".equals(modifier[0]), "测试失败User-Agent 覆写失败");
modifier[0] = "";
context = new RequestContext(Map.of(), "https", "hao123.com");
context = new RequestContext(new Request.Builder()
.url("https://hao123.com"));
Assertions.assertFalse(rule.matches(context), "测试失败");
PatchOkHttp.match(context, "User-Agent", consumer);
Assertions.assertTrue("".equals(modifier[0]), "测试失败User-Agent 不覆写失败");

View File

@@ -0,0 +1,46 @@
package quant.rich;
import lombok.extern.slf4j.Slf4j;
import quant.rich.emoney.EmoneyAutoApplication;
import quant.rich.emoney.client.EmoneyClient;
import quant.rich.emoney.client.WebviewClient;
import quant.rich.emoney.client.WebviewClient.WebviewResponseWrapper;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@SpringBootTest
@ContextConfiguration(classes = EmoneyAutoApplication.class)
@RunWith(SpringJUnit4ClassRunner.class)
@Slf4j
public class RelativeEmoneyScraper {
static final Pattern nonParamsIndexDetailPattern = Pattern.compile("\\[(\\{(id:\\d+,?|data:\\[\\{(title:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|items:\\[(\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?)+],?|image:\".*?\",?)+\\}+\\],?|name:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|nameCode:\"\\d+\",?)+\\},?)+\\]", Pattern.DOTALL);
@Test
void test() throws Exception {
String js = Files.readString(Path.of("./conf/extra/indexJs.js"));
String jsArrayText;
Matcher m = nonParamsIndexDetailPattern.matcher(js);
if (m.find()) jsArrayText = m.group(); else return;
ScriptEngine jsEngine = new ScriptEngineManager().getEngineFactories().get(0).getScriptEngine();
Object result = jsEngine.eval("JSON.stringify(" + jsArrayText + ")");
if (result != null && result instanceof String resultString)
log.info(resultString);
}
}

View File

@@ -0,0 +1,109 @@
package quant.rich;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.graalvm.polyglot.*;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
public class TestIndexJsMatch {
private static final ObjectMapper MAPPER = new ObjectMapper();
public static void main(String[] args) throws IOException {
String script = Files.readString(Path.of("./conf/extra/indexJs.js"), StandardCharsets.UTF_8);
/** 一般来说如果要匹配,只会匹配出一个,但是为了保险起见还是做一个 List 来存储 */
List<ArrayNode> arrays = extractValidArrays(script);
for (ArrayNode array : arrays) {
System.out.println("✔ 匹配数组,共 " + array.size() + " 个对象");
for (JsonNode obj : array) {
System.out.println(obj.toPrettyString());
}
}
}
// 提取所有合法的 [...] 结构,支持嵌套匹配
public static List<String> extractAllArrays(String text) {
List<String> result = new ArrayList<>();
/**
* 这个正则表达式看起来复杂,实际上简单:
* 1. 欲匹配对象是一个 Array所以最外层必包含 \[...\]
* 2. 内容是若干 Object所以会是 (\{...\},?)+
* 3. 每个 Object 内可能包含以下几项:
* 3.1. id可选数字故为 (id:\d+,?)?
* 3.2. dataObject 数组, 包含:
* 3.2.1. title字符串故为 title:".*?",?
* 3.2.2. items字符串数组故为 items:\[(".*?",?)+],?
* 3.2.3. image可选字符串故为 image:".*?",?
* 3.2.4. 组合后即为 data:\[\{(title:".*?",?|items:\[(".*?",?)+],?|image:".*?",?)+\}\],?
* 3.3. name字符串故为 name:".*?",?
* 3.4. nameCode数字型字符串故为 nameCode:"\d+",?
* 3.5. 组合后即为 (\{(id:\d+,?|data:\[\{(title:".*?",?|items:\[(".*?",?)+],?|image:".*?",?)+\}+\],?|name:".*?",?|nameCode:"\d+",?)+\},?)+
* 4. 组合上述所有即为 \[(\{(id:\d+,?|data:\[\{(title:".*?",?|items:\[(".*?",?)+],?|image:".*?",?)+\}+\],?|name:".*?",?|nameCode:"\d+",?)+\},?)+\]
* 5. 将 ".*?" 替换为 "[^"\\]*(?:\\.[^"\\]*)*?" 以防止字符串中带双引号导致的异常截断image 除外
*/
Pattern p = Pattern.compile("\\[(\\{(id:\\d+,?|data:\\[\\{(title:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|items:\\[(\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?)+],?|image:\".*?\",?)+\\}+\\],?|name:\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*?\",?|nameCode:\"\\d+\",?)+\\},?)+\\]", Pattern.DOTALL);
Matcher m = p.matcher(text);
while (m.find()) {
result.add(m.group());
}
return result;
}
// 使用 GraalVM 的 JS 引擎尝试 JSON.stringify
public static Optional<String> stringifyJsArray(String jsArrayText) {
try (Context context = Context.create("js")) {
Value result = context.eval("js", "JSON.stringify(" + jsArrayText + ")");
return result.isString() ? Optional.of(result.asString()) : Optional.empty();
} catch (Exception e) {
return Optional.empty();
}
}
// 判断某个对象是否结构匹配
public static boolean matches(JsonNode obj) {
return obj.has("name") &&
obj.has("nameCode") &&
obj.has("data") &&
obj.get("data").isArray();
}
// 主处理逻辑
public static List<ArrayNode> extractValidArrays(String jsSource) {
List<ArrayNode> valid = new ArrayList<>();
for (String raw : extractAllArrays(jsSource)) {
stringifyJsArray(raw).ifPresent(json -> {
try {
JsonNode root = MAPPER.readTree(json);
if (root.isArray()) {
ArrayNode array = (ArrayNode) root;
for (JsonNode obj : array) {
if (obj.isObject() && matches(obj)) {
valid.add(array);
break;
}
}
}
} catch (Exception ignored) {
}
});
}
return valid;
}
}