Search in sources :

Example 1 with Page

use of com.ruiyun.jvppeteer.core.page.Page in project epic4j by huisunan.

the class MainStart method getDefaultPage.

@Override
@Retry(message = "默认页面打开失败")
public Page getDefaultPage(Browser browser) {
    List<Page> pages = browser.pages();
    Page page = CollUtil.isNotEmpty(pages) ? pages.get(0) : browser.newPage();
    Viewport viewport = new Viewport();
    viewport.setWidth(600);
    viewport.setHeight(1000);
    viewport.setHasTouch(true);
    page.setViewport(viewport);
    return page;
}
Also used : Viewport(com.ruiyun.jvppeteer.options.Viewport) Page(com.ruiyun.jvppeteer.core.page.Page) Retry(com.hsn.epic4j.aop.Retry)

Example 2 with Page

use of com.ruiyun.jvppeteer.core.page.Page in project epic4j by huisunan.

the class CrawTest method test.

@Test
@SneakyThrows
public void test() {
    BrowserFetcher.downloadIfNotExist("938248");
    String dataPath = new FileUrlResource("./data/chrome/tests").getFile().getAbsolutePath();
    LaunchOptions options = new LaunchOptionsBuilder().withArgs(Arrays.asList("--blink-settings=imagesEnabled=false", "--no-first-run", "--disable-gpu", "--no-default-browser-check", "--no-sandbox")).withHeadless(false).withUserDataDir(dataPath).withExecutablePath("").withIgnoreDefaultArgs(Collections.singletonList("--enable-automation")).build();
    Browser browser = Puppeteer.launch(options);
    Page page = browser.pages().get(0);
    // PageUtil.crawSet(page);
    // page.goTo("http://localhost:9999/");
    page.goTo("http://www.baidu.com/");
    TimeUnit.SECONDS.sleep(10);
    FileUrlResource errorDir = new FileUrlResource("data/test");
    File file = errorDir.getFile();
    log.debug("mkdir {}", file.mkdirs());
    ScreenshotOptions screenshotOptions = new ScreenshotOptions();
    screenshotOptions.setQuality(100);
    screenshotOptions.setPath(file.getAbsolutePath() + File.separator + "report.jpg");
    screenshotOptions.setType("jpeg");
    page.screenshot(screenshotOptions);
    FileUtil.writeUtf8String(page.content(), new File(file.getAbsolutePath() + File.separator + "report.html"));
    log.info("success");
}
Also used : LaunchOptions(com.ruiyun.jvppeteer.options.LaunchOptions) LaunchOptionsBuilder(com.ruiyun.jvppeteer.options.LaunchOptionsBuilder) ScreenshotOptions(com.ruiyun.jvppeteer.options.ScreenshotOptions) Page(com.ruiyun.jvppeteer.core.page.Page) FileUrlResource(org.springframework.core.io.FileUrlResource) File(java.io.File) Browser(com.ruiyun.jvppeteer.core.browser.Browser) Test(org.junit.jupiter.api.Test) SneakyThrows(lombok.SneakyThrows)

Example 3 with Page

use of com.ruiyun.jvppeteer.core.page.Page in project epic4j by huisunan.

the class BaseRunner method doStart.

public void doStart(String email, String password) {
    Browser browser = null;
    try {
        log.info("start {} work", email);
        // 用户文件路径
        String userDataPath = new FileUrlResource(epicConfig.getDataPath() + File.separator + email).getFile().getAbsolutePath();
        // 获取浏览器对象
        browser = iStart.getBrowser(userDataPath);
        // 获取默认page
        Page page = iStart.getDefaultPage(browser);
        // 加载cookie
        if (StrUtil.isNotBlank(epicConfig.getCookiePath()) && FileUtil.exist(epicConfig.getCookiePath())) {
            log.debug("load cookie");
            List<CookieParam> cookies = JSONUtil.toBean(IoUtil.read(new FileReader(epicConfig.getCookiePath())), new TypeReference<List<CookieParam>>() {
            }, false);
            page.setCookie(cookies);
        }
        // 反爬虫设置
        PageUtil.crawSet(page);
        // 打开epic主页
        page.goTo(epicConfig.getEpicUrl());
        boolean needLogin = iStart.needLogin(browser);
        log.debug("needLogin:{}", needLogin);
        if (needLogin) {
            iLogin.login(page, email, password);
        }
        List<Item> weekFreeItems = iStart.getWeekFreeItems(page);
        // 领取游戏
        List<Item> receive = iStart.receive(page, weekFreeItems);
        for (INotify notify : notifies) {
            if (notify.notifyReceive(receive)) {
                break;
            }
        }
    } catch (Exception e) {
        if (epicConfig.getErrorScreenShoot()) {
            Optional.ofNullable(browser).map(Browser::pages).filter(CollUtil::isNotEmpty).map(pages -> pages.get(0)).ifPresent(page -> {
                try {
                    FileUrlResource errorDir = new FileUrlResource("data/error");
                    log.debug("create error dir {}", errorDir.getFile().mkdirs());
                    ScreenshotOptions options = new ScreenshotOptions();
                    options.setQuality(100);
                    options.setPath(errorDir.getFile().getAbsolutePath() + File.separator + System.currentTimeMillis() + ".jpg");
                    options.setType("jpeg");
                    page.screenshot(options);
                } catch (IOException ioException) {
                    log.error("截图失败");
                }
            });
        }
        log.error("程序异常", e);
    } finally {
        if (browser != null) {
            browser.close();
        }
    }
}
Also used : DateUtil(cn.hutool.core.date.DateUtil) Item(com.hsn.epic4j.bean.Item) SneakyThrows(lombok.SneakyThrows) Autowired(org.springframework.beans.factory.annotation.Autowired) JSONUtil(cn.hutool.json.JSONUtil) PreDestroy(javax.annotation.PreDestroy) DateTime(cn.hutool.core.date.DateTime) FileUrlResource(org.springframework.core.io.FileUrlResource) ThreadPoolTaskScheduler(org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler) Browser(com.ruiyun.jvppeteer.core.browser.Browser) UserInfo(com.hsn.epic4j.bean.UserInfo) INotify(com.hsn.epic4j.notify.INotify) IoUtil(cn.hutool.core.io.IoUtil) EpicConfig(com.hsn.epic4j.config.EpicConfig) ScreenshotOptions(com.ruiyun.jvppeteer.options.ScreenshotOptions) IOException(java.io.IOException) File(java.io.File) CronTrigger(org.springframework.scheduling.support.CronTrigger) Page(com.ruiyun.jvppeteer.core.page.Page) CollUtil(cn.hutool.core.collection.CollUtil) TypeReference(cn.hutool.core.lang.TypeReference) StrUtil(cn.hutool.core.util.StrUtil) Slf4j(lombok.extern.slf4j.Slf4j) Component(org.springframework.stereotype.Component) List(java.util.List) CookieParam(com.ruiyun.jvppeteer.protocol.network.CookieParam) DateField(cn.hutool.core.date.DateField) Optional(java.util.Optional) FileUtil(cn.hutool.core.io.FileUtil) PageUtil(com.hsn.epic4j.util.PageUtil) FileReader(java.io.FileReader) INotify(com.hsn.epic4j.notify.INotify) ScreenshotOptions(com.ruiyun.jvppeteer.options.ScreenshotOptions) Page(com.ruiyun.jvppeteer.core.page.Page) IOException(java.io.IOException) IOException(java.io.IOException) CookieParam(com.ruiyun.jvppeteer.protocol.network.CookieParam) Item(com.hsn.epic4j.bean.Item) FileReader(java.io.FileReader) List(java.util.List) FileUrlResource(org.springframework.core.io.FileUrlResource) Browser(com.ruiyun.jvppeteer.core.browser.Browser)

Example 4 with Page

use of com.ruiyun.jvppeteer.core.page.Page in project epic4j by huisunan.

the class MainStart method getWeekFreeItems.

/**
 * 获取周免游戏
 */
@Retry(message = "获取周末游戏失败")
@Override
public List<Item> getWeekFreeItems(Page page) {
    Browser browser = page.browser();
    String userCountry = "CN";
    String locate = "zh-CN";
    String country = PageUtil.getJsonValue(browser, epicConfig.getUserInfoUrl(), "userInfo.country.value", String.class);
    if (StrUtil.isNotEmpty(country)) {
        userCountry = country;
        locate = locate + "-" + userCountry;
    }
    String formatUrl = StrUtil.format(epicConfig.getFreeGameUrl(), locate, userCountry, userCountry);
    log.debug(formatUrl);
    JSONObject json = PageUtil.getJson(formatUrl, browser);
    List<Item> list = new ArrayList<>();
    DateTime now = DateUtil.date();
    for (JSONObject element : json.getByPath("data.Catalog.searchStore.elements", JSONArray.class).jsonIter()) {
        if (!"ACTIVE".equals(element.getStr("status"))) {
            continue;
        }
        if (StreamSupport.stream(element.getJSONArray("categories").jsonIter().spliterator(), false).anyMatch(item -> "freegames".equals(item.getStr("path")))) {
            JSONObject promotions = element.getJSONObject("promotions");
            if (promotions == null) {
                continue;
            }
            JSONArray promotionalOffers = promotions.getJSONArray("promotionalOffers");
            if (CollUtil.isNotEmpty(promotionalOffers)) {
                if (StreamSupport.stream(promotionalOffers.jsonIter().spliterator(), false).flatMap(offerItem -> StreamSupport.stream(offerItem.getJSONArray("promotionalOffers").jsonIter().spliterator(), false)).anyMatch(offerItem -> {
                    DateTime startDate = DateUtil.parse(offerItem.getStr("startDate")).setTimeZone(TimeZone.getDefault());
                    DateTime endDate = DateUtil.parse(offerItem.getStr("endDate")).setTimeZone(TimeZone.getDefault());
                    JSONObject discountSetting = offerItem.getJSONObject("discountSetting");
                    return DateUtil.isIn(now, startDate, endDate) && "PERCENTAGE".equals(discountSetting.getStr("discountType")) && discountSetting.getInt("discountPercentage") == 0;
                })) {
                    list.add(element.toBean(Item.class));
                }
            }
        }
    }
    return list;
}
Also used : DateUtil(cn.hutool.core.date.DateUtil) Item(com.hsn.epic4j.bean.Item) java.util(java.util) SneakyThrows(lombok.SneakyThrows) Constant(com.ruiyun.jvppeteer.core.Constant) FileUtil(com.ruiyun.jvppeteer.util.FileUtil) Viewport(com.ruiyun.jvppeteer.options.Viewport) Autowired(org.springframework.beans.factory.annotation.Autowired) DateTime(cn.hutool.core.date.DateTime) StreamSupport(java.util.stream.StreamSupport) TimeException(com.hsn.epic4j.exception.TimeException) Browser(com.ruiyun.jvppeteer.core.browser.Browser) EpicConfig(com.hsn.epic4j.config.EpicConfig) Puppeteer(com.ruiyun.jvppeteer.core.Puppeteer) JSONObject(cn.hutool.json.JSONObject) LaunchOptions(com.ruiyun.jvppeteer.options.LaunchOptions) Retry(com.hsn.epic4j.aop.Retry) SelectItem(com.hsn.epic4j.bean.SelectItem) PermissionException(com.hsn.epic4j.exception.PermissionException) LaunchOptionsBuilder(com.ruiyun.jvppeteer.options.LaunchOptionsBuilder) Collectors(java.util.stream.Collectors) Page(com.ruiyun.jvppeteer.core.page.Page) CollUtil(cn.hutool.core.collection.CollUtil) StrUtil(cn.hutool.core.util.StrUtil) Slf4j(lombok.extern.slf4j.Slf4j) Component(org.springframework.stereotype.Component) BrowserFetcher(com.ruiyun.jvppeteer.core.browser.BrowserFetcher) JSONArray(cn.hutool.json.JSONArray) PageUtil(com.hsn.epic4j.util.PageUtil) ItemException(com.hsn.epic4j.exception.ItemException) Item(com.hsn.epic4j.bean.Item) SelectItem(com.hsn.epic4j.bean.SelectItem) JSONObject(cn.hutool.json.JSONObject) JSONArray(cn.hutool.json.JSONArray) DateTime(cn.hutool.core.date.DateTime) Browser(com.ruiyun.jvppeteer.core.browser.Browser) Retry(com.hsn.epic4j.aop.Retry)

Example 5 with Page

use of com.ruiyun.jvppeteer.core.page.Page in project epic4j by huisunan.

the class MainStart method receive.

@Override
@SneakyThrows
@Retry(message = "领取失败")
public List<Item> receive(Page page, List<Item> weekFreeItems) {
    if (log.isDebugEnabled()) {
        log.debug("all free items:{}", weekFreeItems.stream().map(Item::getTitle).collect(Collectors.joining(",")));
    }
    List<Item> receiveItem = new ArrayList<>();
    for (Item item : weekFreeItems) {
        String url = "";
        if (Item.BASE_GAME.equals(item.getOfferType())) {
            url = item.getProductSlug();
        } else if (Item.DLC.equals(item.getOfferType())) {
            url = item.getUrlSlug();
        }
        String itemUrl = StrUtil.format(epicConfig.getStoreUrl(), url);
        log.debug("item url:{}", itemUrl);
        page.goTo(itemUrl);
        // age limit
        PageUtil.tryClick(page, "div[data-component=PDPAgeGate] Button", itemUrl, 8, 1000);
        PageUtil.waitForTextChange(page, "div[data-component=DesktopSticky] button[data-testid=purchase-cta-button]", "Loading");
        if (isInLibrary(page)) {
            log.debug("{} had in library", item.getTitle());
            continue;
        }
        page.waitForSelector("div[data-component=WithClickTracking] button").click();
        // epic user licence check
        log.debug("user licence check");
        PageUtil.tryClick(page, "div[data-component=makePlatformUnsupportedWarningStep] button[data-component=BaseButton", itemUrl);
        PageUtil.tryClick(page, "#agree", itemUrl);
        PageUtil.tryClick(page, "div[data-component=EulaModalActions] button[data-component=BaseButton]", itemUrl);
        String purchaseUrl = PageUtil.getStrProperty(page, "#webPurchaseContainer iframe", "src");
        log.debug("purchase url :{}", purchaseUrl);
        page.goTo(purchaseUrl);
        PageUtil.tryClick(page, "#purchase-app button[class*=confirm]:not([disabled])", page.mainFrame().url(), 20, 500);
        PageUtil.tryClick(page, "#purchaseAppContainer div.payment-overlay button.payment-btn--primary", page.mainFrame().url());
        PageUtil.findSelectors(page, 10_000, true, () -> {
            throw new TimeException("time out");
        }, new SelectItem("#purchase-app div[class*=alert]", () -> {
            if (item.isDLC()) {
                // DLC情况下,在没有本体的情况下也也可以领取
                return SelectItem.SelectCallBack.CONTINUE;
            } else {
                String message = PageUtil.getStrProperty(page, "#purchase-app div[class*=alert]:not([disabled])", "textContent");
                throw new PermissionException(message);
            }
        }), new SelectItem("#talon_frame_checkout_free_prod[style*=visible]", () -> {
            // 需要验证码
            throw new PermissionException("CAPTCHA is required for unknown reasons when claiming");
        }), new SelectItem("#purchase-app > div", (p, i) -> p.$(i.getSelectors()) == null, () -> {
            // 当订单完成刷新时,该元素不存在,是订单完成后刷新到新页面
            page.goTo(itemUrl);
            PageUtil.waitForTextChange(page, "div[data-component=DesktopSticky] button[data-testid=purchase-cta-button]", "Loading");
            if (!isInLibrary(page)) {
                throw new ItemException("An item was mistakenly considered to have been claimed");
            }
            receiveItem.add(item);
            return SelectItem.SelectCallBack.END;
        }));
    }
    if (receiveItem.isEmpty()) {
        log.info("all free week games in your library:{}", weekFreeItems.stream().map(Item::getTitle).collect(Collectors.joining(",")));
    }
    return receiveItem;
}
Also used : PermissionException(com.hsn.epic4j.exception.PermissionException) DateUtil(cn.hutool.core.date.DateUtil) Item(com.hsn.epic4j.bean.Item) java.util(java.util) SneakyThrows(lombok.SneakyThrows) Constant(com.ruiyun.jvppeteer.core.Constant) FileUtil(com.ruiyun.jvppeteer.util.FileUtil) Viewport(com.ruiyun.jvppeteer.options.Viewport) Autowired(org.springframework.beans.factory.annotation.Autowired) DateTime(cn.hutool.core.date.DateTime) StreamSupport(java.util.stream.StreamSupport) TimeException(com.hsn.epic4j.exception.TimeException) Browser(com.ruiyun.jvppeteer.core.browser.Browser) EpicConfig(com.hsn.epic4j.config.EpicConfig) Puppeteer(com.ruiyun.jvppeteer.core.Puppeteer) JSONObject(cn.hutool.json.JSONObject) LaunchOptions(com.ruiyun.jvppeteer.options.LaunchOptions) Retry(com.hsn.epic4j.aop.Retry) SelectItem(com.hsn.epic4j.bean.SelectItem) PermissionException(com.hsn.epic4j.exception.PermissionException) LaunchOptionsBuilder(com.ruiyun.jvppeteer.options.LaunchOptionsBuilder) Collectors(java.util.stream.Collectors) Page(com.ruiyun.jvppeteer.core.page.Page) CollUtil(cn.hutool.core.collection.CollUtil) StrUtil(cn.hutool.core.util.StrUtil) Slf4j(lombok.extern.slf4j.Slf4j) Component(org.springframework.stereotype.Component) BrowserFetcher(com.ruiyun.jvppeteer.core.browser.BrowserFetcher) JSONArray(cn.hutool.json.JSONArray) PageUtil(com.hsn.epic4j.util.PageUtil) ItemException(com.hsn.epic4j.exception.ItemException) Item(com.hsn.epic4j.bean.Item) SelectItem(com.hsn.epic4j.bean.SelectItem) ItemException(com.hsn.epic4j.exception.ItemException) TimeException(com.hsn.epic4j.exception.TimeException) SelectItem(com.hsn.epic4j.bean.SelectItem) SneakyThrows(lombok.SneakyThrows) Retry(com.hsn.epic4j.aop.Retry)

Aggregations

Page (com.ruiyun.jvppeteer.core.page.Page)6 SneakyThrows (lombok.SneakyThrows)5 Browser (com.ruiyun.jvppeteer.core.browser.Browser)4 CollUtil (cn.hutool.core.collection.CollUtil)3 DateTime (cn.hutool.core.date.DateTime)3 DateUtil (cn.hutool.core.date.DateUtil)3 StrUtil (cn.hutool.core.util.StrUtil)3 JSONObject (cn.hutool.json.JSONObject)3 Retry (com.hsn.epic4j.aop.Retry)3 Item (com.hsn.epic4j.bean.Item)3 EpicConfig (com.hsn.epic4j.config.EpicConfig)3 PageUtil (com.hsn.epic4j.util.PageUtil)3 LaunchOptions (com.ruiyun.jvppeteer.options.LaunchOptions)3 LaunchOptionsBuilder (com.ruiyun.jvppeteer.options.LaunchOptionsBuilder)3 Viewport (com.ruiyun.jvppeteer.options.Viewport)3 Slf4j (lombok.extern.slf4j.Slf4j)3 Autowired (org.springframework.beans.factory.annotation.Autowired)3 Component (org.springframework.stereotype.Component)3 JSONArray (cn.hutool.json.JSONArray)2 SelectItem (com.hsn.epic4j.bean.SelectItem)2