69 lines
3.6 KiB
Java
69 lines
3.6 KiB
Java
import com.microsoft.playwright.*;
|
|
import com.microsoft.playwright.BrowserType.LaunchOptions;
|
|
import com.microsoft.playwright.Route.ResumeOptions;
|
|
import com.microsoft.playwright.options.HttpHeader;
|
|
import com.microsoft.playwright.options.Proxy;
|
|
import com.microsoft.playwright.options.WaitUntilState;
|
|
|
|
import java.util.*;
|
|
|
|
public class EmoneyScraper {
|
|
public static void main(String[] args) {
|
|
try (Playwright playwright = Playwright.create()) {
|
|
// 设置浏览器上下文选项，包括 User-Agent
|
|
Proxy proxy = new Proxy("http://127.0.0.1:8888");
|
|
LaunchOptions launchOptions = new BrowserType.LaunchOptions()
|
|
.setHeadless(true)
|
|
.setProxy(proxy);
|
|
Browser browser = playwright.chromium().launch(launchOptions);
|
|
BrowserContext context = browser.newContext(new Browser.NewContextOptions()
|
|
.setUserAgent("Mozilla/5.0 (Linux; Android 9; PCRT00 Build/PQ3B.190801.11191547; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.114 Mobile Safari/537.36"));
|
|
|
|
// 设置全局请求头
|
|
Map<String, String> headers = new HashMap<>();
|
|
headers.put("Upgrade-Insecure-Requests", "1");
|
|
headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
|
|
headers.put("X-Requested-With", "cn.emoney.emstock");
|
|
headers.put("Sec-Fetch-Site", "none");
|
|
headers.put("Sec-Fetch-Mode", "navigate");
|
|
headers.put("Sec-Fetch-User", "?1");
|
|
headers.put("Sec-Fetch-Dest", "document");
|
|
headers.put("Accept-Encoding", "gzip, deflate");
|
|
headers.put("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
|
|
|
|
context.setExtraHTTPHeaders(headers);
|
|
context.route("**/*", route -> {
|
|
Request request = route.request();
|
|
List<HttpHeader> requestHeaderList = request.headersArray();
|
|
Map<String, String> requestHeaders = new HashMap<>();
|
|
for (HttpHeader header : requestHeaderList) {
|
|
requestHeaders.put(header.name, header.value);
|
|
}
|
|
requestHeaders.remove("sec-ch-ua");
|
|
requestHeaders.remove("sec-ch-ua-mobile");
|
|
requestHeaders.remove("sec-ch-ua-platform");
|
|
route.resume(new ResumeOptions().setHeaders(requestHeaders));
|
|
});
|
|
|
|
Page page = context.newPage();
|
|
|
|
page.onResponse(handler -> {
|
|
String url = handler.request().url();
|
|
if (url.endsWith(".js")) {
|
|
String str = handler.request().response().text();
|
|
System.out.println(str);
|
|
}
|
|
});
|
|
|
|
String url = "https://appstatic.emoney.cn/html/emapp/stock/note/?name=10003001&emoneyScaleType=0&emoneyLandMode=0&token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImN0eSI6IkpXVCJ9.eyJ1dWQiOjEwMTA2NjAwNDcsInVpZCI6MjQ3NjY5MDIsImRpZCI6IjdmOTA3N2VjZTlmMWIxMjQ4NmZmYjRmNDhjMzdkODJhIiwidHlwIjoxLCJhY2MiOiJlbXkxNzMwOTc4Iiwic3d0IjoxLCJsZ3QiOjE3NDQ1OTY2NTY3MTgsIm5iZiI6MTc0NDU5NjY1NiwiZXhwIjoxNzQ2MzI0NjU2LCJpYXQiOjE3NDQ1OTY2NTZ9.SHLqiavrzkwtLfxaWbA6GfnF7iBqbjnv86PTOhumiGc";
|
|
|
|
page.navigate(url, new Page.NavigateOptions().setWaitUntil(WaitUntilState.NETWORKIDLE));
|
|
|
|
String html = page.content();
|
|
System.out.println(html);
|
|
|
|
browser.close();
|
|
}
|
|
}
|
|
}
|