Examples with WebClient com.gargoylesoftware.htmlunit.WebClient used on opensource projects

Example 66 with WebClient

use of com.gargoylesoftware.htmlunit.WebClient in project tomee by apache.

the class MoviesArquillianHtmlUnitTest method testShouldMakeSureWebappIsWorking.

@Test
public void testShouldMakeSureWebappIsWorking() throws Exception {
    final String url = "http://" + deploymentUrl.getHost() + ":" + deploymentUrl.getPort() + "/moviefun";
    WebClient webClient = new WebClient();
    HtmlPage page = webClient.getPage(url + "/setup.jsp");
    assertMoviesPresent(page);
    page = webClient.getPage(url + "/moviefun");
    assertMoviesPresent(page);
    webClient.closeAllWindows();
}

Example 67 with WebClient

use of com.gargoylesoftware.htmlunit.WebClient in project wechat by dllwh.

the class HtmlUnitHandler method crawl.

/**
 * ----------------------------------------------- [私有方法]
 */
@Override
public String crawl(String url, CrawlParameter crawlPara) {
    WebClient webClient = getWebClient(crawlPara);
    String reqtype = crawlPara.getReqmethod();
    Page page = null;
    String resource = "";
    try {
        if (StringUtils.isNotBlank(reqtype)) {
            WebRequest webRequest = new WebRequest(new URL(url));
            if ("post".equals(reqtype)) {
                webRequest.setHttpMethod(HttpMethod.POST);
            } else if ("get".equals(reqtype)) {
                webRequest.setHttpMethod(HttpMethod.GET);
            }
            Map<String, String> reqmap = crawlPara.getReqmap();
            if (MapUtils.isNotEmpty(reqmap)) {
                for (Entry<String, String> param : reqmap.entrySet()) {
                    webRequest.getRequestParameters().add(new NameValuePair(param.getKey(), param.getValue()));
                }
            }
            page = webClient.getPage(webRequest);
            resource = page.getWebResponse().getContentAsString();
        } else {
            page = webClient.getPage(url);
            resource = ((HtmlPage) page).asXml();
        }
    } catch (Exception ex) {
        ex.printStackTrace();
    }
    return resource;
}

Example 68 with WebClient

use of com.gargoylesoftware.htmlunit.WebClient in project saga by timurstrekalov.

the class WebClientFactory method newInstance.

// New version allows for creation of web drivers without the need for a config object
// which may not be readily available if the driver is created by third-party code.
public static WebClient newInstance(final BrowserVersion version) {
    final WebClient client = new WebClient(version) {

        @Override
        public WebResponse loadWebResponse(final WebRequest webRequest) throws IOException {
            return new WebResponseProxy(super.loadWebResponse(webRequest));
        }
    };
    client.setIncorrectnessListener(quietIncorrectnessListener);
    client.setJavaScriptErrorListener(loggingJsErrorListener);
    client.setHTMLParserListener(quietHtmlParserListener);
    client.setCssErrorHandler(quietCssErrorHandler);
    client.getOptions().setJavaScriptEnabled(true);
    client.setAjaxController(new NicelyResynchronizingAjaxController());
    client.getOptions().setThrowExceptionOnScriptError(false);
    client.getOptions().setThrowExceptionOnFailingStatusCode(false);
    client.getOptions().setPrintContentOnFailingStatusCode(false);
    client.setWebConnection(new HttpWebConnection(client) {

        @Override
        protected WebResponse newWebResponseInstance(final WebResponseData responseData, final long loadTime, final WebRequest request) {
            return new WebResponseProxy(super.newWebResponseInstance(responseData, loadTime, request));
        }
    });
    return client;
}

Example 69 with WebClient

use of com.gargoylesoftware.htmlunit.WebClient in project ats-framework by Axway.

the class HiddenBrowserDriver method fixHtmlUnitBehaviour.

/**
 * Fixing refresh handler to skip Refresh meta tags
 * Allowing connections to any host, regardless of whether they have valid certificates or not
 * Fixing JSESSIONID cookie value
 * Some applications expect double quotes in the beginning and at the end of the JSESSIONID cookie value
 */
private void fixHtmlUnitBehaviour() {
    Field webClientField = null;
    boolean fieldAccessibleState = false;
    try {
        TargetLocator targetLocator = webDriver.switchTo();
        webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
        fieldAccessibleState = webClientField.isAccessible();
        webClientField.setAccessible(true);
        final WebClient webClient = (WebClient) webClientField.get(targetLocator.defaultContent());
        // Allowing connections to any host, regardless of whether they have valid certificates or not
        webClient.getOptions().setUseInsecureSSL(true);
        // Set Http connection timeout (in milliseconds). The default value is 90 seconds, because in Firefox >= 16
        // the "network.http.connection-timeout" property is 90. But this value is not enough for some cases.
        // NOTE: use 0 for infinite timeout
        webClient.getOptions().setTimeout(5 * 60 * 1000);
        webClient.getOptions().setRedirectEnabled(true);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setThrowExceptionOnScriptError(true);
        webClient.getOptions().setPrintContentOnFailingStatusCode(true);
        // Hide CSS Warnings
        webClient.setCssErrorHandler(new SilentCssErrorHandler());
        // Suppress warnings like: "Expected content type ... but got ..."
        webClient.setIncorrectnessListener(new IncorrectnessListener() {

            // private final Log log = LogFactory.getLog( this.getClass() );
            @Override
            public void notify(final String message, final Object origin) {
            // log.warn( message );
            }
        });
        if (!Boolean.parseBoolean(System.getProperty(ALLOW_META_REFRESH_TAG))) {
            /*
                 * Fix for refresh meta tags eg. "<meta http-equiv="refresh" content="300">"
                 * The default refresh handler is with Thread.sleep(refreshSecondsFromMetaTag) in the main thread!!!
                     *
                     * Maybe we should check and test this handler: webClient.setRefreshHandler( new ThreadedRefreshHandler() );
                 */
            webClient.setRefreshHandler(new RefreshHandler() {

                @Override
                public void handleRefresh(Page page, URL url, int seconds) throws IOException {
                }
            });
        }
        /*
             * Fix JSessionId
             */
        // WebConnectionWrapper constructs a WebConnection object wrapping the connection of the WebClient
        // and places itself (in the constructor) as connection of the WebClient.
        new WebConnectionWrapper(webClient) {

            public WebResponse getResponse(WebRequest request) throws IOException {
                Cookie jsCookie = webClient.getCookieManager().getCookie("JSESSIONID");
                if (jsCookie != null && (!jsCookie.getValue().startsWith("\"") && !jsCookie.getValue().endsWith("\""))) {
                    Cookie newCookie = new Cookie(jsCookie.getDomain(), jsCookie.getName(), "\"" + jsCookie.getValue() + "\"", jsCookie.getPath(), jsCookie.getExpires(), jsCookie.isSecure());
                    webClient.getCookieManager().removeCookie(jsCookie);
                    webClient.getCookieManager().addCookie(newCookie);
                }
                return super.getResponse(request);
            }
        };
    } catch (Exception e) {
        throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
    } finally {
        if (webClientField != null) {
            webClientField.setAccessible(fieldAccessibleState);
        }
    }
}

Example 70 with WebClient

use of com.gargoylesoftware.htmlunit.WebClient in project ats-framework by Axway.

the class AbstractHtmlEngine method reloadFrames.

@PublicAtsApi
public void reloadFrames() {
    // real browsers reloads the frames automatically
    if (webDriver instanceof HtmlUnitDriver) {
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {
            // Retrieve current WebClient instance (with the current page) from the Selenium WebDriver
            TargetLocator targetLocator = webDriver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            WebClient webClient = (WebClient) webClientField.get(targetLocator.defaultContent());
            HtmlPage page = (HtmlPage) webClient.getCurrentWindow().getEnclosedPage();
            for (final FrameWindow frameWindow : page.getFrames()) {
                final BaseFrameElement frame = frameWindow.getFrameElement();
                // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
                if (frame.getEnclosedPage().getWebResponse().getWebRequest().getUrl() == WebClient.URL_ABOUT_BLANK) {
                    String src = frame.getSrcAttribute();
                    if (src != null && !src.isEmpty()) {
                        final URL url;
                        try {
                            url = ((HtmlPage) frame.getEnclosedPage()).getFullyQualifiedUrl(src);
                        } catch (final MalformedURLException e) {
                            String message = "Invalid src attribute of " + frame.getTagName() + ": url=[" + src + "]. Ignored.";
                            final IncorrectnessListener incorrectnessListener = webClient.getIncorrectnessListener();
                            incorrectnessListener.notify(message, this);
                            return;
                        }
                        if (isAlreadyLoadedByAncestor(url, ((HtmlPage) frame.getEnclosedPage()))) {
                            String message = "Recursive src attribute of " + frame.getTagName() + ": url=[" + src + "]. Ignored.";
                            final IncorrectnessListener incorrectnessListener = webClient.getIncorrectnessListener();
                            incorrectnessListener.notify(message, this);
                            log.info("Frame already loaded: " + frame.toString());
                            return;
                        }
                        try {
                            final WebRequest request = new WebRequest(url);
                            request.setAdditionalHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9, text/*;q=0.7, */*;q=0.5");
                            if (frameWindow.getName() == null || frameWindow.getName().isEmpty()) {
                                frameWindow.setName("frame_" + page.getFrames().indexOf(frameWindow));
                            }
                            webClient.loadWebResponseInto(webClient.loadWebResponse(request), frameWindow);
                            log.info("Frame loaded: " + frame.toString());
                        } catch (IOException e) {
                            log.error("Error when getting content for " + frame.getTagName() + " with src=" + url, e);
                        }
                    }
                } else {
                    log.info("Frame already loaded: " + frame.toString());
                }
            }
        } catch (Exception e) {
            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {
            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
}