Examples with IncorrectnessListener com.gargoylesoftware.htmlunit.IncorrectnessListener used on opensource projects

Example 1 with IncorrectnessListener

use of com.gargoylesoftware.htmlunit.IncorrectnessListener in project ats-framework by Axway.

the class HiddenBrowserDriver method fixHtmlUnitBehaviour.

/**
 * Fixing refresh handler to skip Refresh meta tags
 * Allowing connections to any host, regardless of whether they have valid certificates or not
 * Fixing JSESSIONID cookie value
 * Some applications expect double quotes in the beginning and at the end of the JSESSIONID cookie value
 */
private void fixHtmlUnitBehaviour() {
    Field webClientField = null;
    boolean fieldAccessibleState = false;
    try {
        TargetLocator targetLocator = webDriver.switchTo();
        webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
        fieldAccessibleState = webClientField.isAccessible();
        webClientField.setAccessible(true);
        final WebClient webClient = (WebClient) webClientField.get(targetLocator.defaultContent());
        // Allowing connections to any host, regardless of whether they have valid certificates or not
        webClient.getOptions().setUseInsecureSSL(true);
        // Set Http connection timeout (in milliseconds). The default value is 90 seconds, because in Firefox >= 16
        // the "network.http.connection-timeout" property is 90. But this value is not enough for some cases.
        // NOTE: use 0 for infinite timeout
        webClient.getOptions().setTimeout(5 * 60 * 1000);
        webClient.getOptions().setRedirectEnabled(true);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setThrowExceptionOnScriptError(true);
        webClient.getOptions().setPrintContentOnFailingStatusCode(true);
        // Hide CSS Warnings
        webClient.setCssErrorHandler(new SilentCssErrorHandler());
        // Suppress warnings like: "Expected content type ... but got ..."
        webClient.setIncorrectnessListener(new IncorrectnessListener() {

            // private final Log log = LogFactory.getLog( this.getClass() );
            @Override
            public void notify(final String message, final Object origin) {
            // log.warn( message );
            }
        });
        if (!Boolean.parseBoolean(System.getProperty(ALLOW_META_REFRESH_TAG))) {
            /*
                 * Fix for refresh meta tags eg. "<meta http-equiv="refresh" content="300">"
                 * The default refresh handler is with Thread.sleep(refreshSecondsFromMetaTag) in the main thread!!!
                     *
                     * Maybe we should check and test this handler: webClient.setRefreshHandler( new ThreadedRefreshHandler() );
                 */
            webClient.setRefreshHandler(new RefreshHandler() {

                @Override
                public void handleRefresh(Page page, URL url, int seconds) throws IOException {
                }
            });
        }
        /*
             * Fix JSessionId
             */
        // WebConnectionWrapper constructs a WebConnection object wrapping the connection of the WebClient
        // and places itself (in the constructor) as connection of the WebClient.
        new WebConnectionWrapper(webClient) {

            public WebResponse getResponse(WebRequest request) throws IOException {
                Cookie jsCookie = webClient.getCookieManager().getCookie("JSESSIONID");
                if (jsCookie != null && (!jsCookie.getValue().startsWith("\"") && !jsCookie.getValue().endsWith("\""))) {
                    Cookie newCookie = new Cookie(jsCookie.getDomain(), jsCookie.getName(), "\"" + jsCookie.getValue() + "\"", jsCookie.getPath(), jsCookie.getExpires(), jsCookie.isSecure());
                    webClient.getCookieManager().removeCookie(jsCookie);
                    webClient.getCookieManager().addCookie(newCookie);
                }
                return super.getResponse(request);
            }
        };
    } catch (Exception e) {
        throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
    } finally {
        if (webClientField != null) {
            webClientField.setAccessible(fieldAccessibleState);
        }
    }
}

Example 2 with IncorrectnessListener

use of com.gargoylesoftware.htmlunit.IncorrectnessListener in project ats-framework by Axway.

the class AbstractHtmlEngine method reloadFrames.

@PublicAtsApi
public void reloadFrames() {
    // real browsers reloads the frames automatically
    if (webDriver instanceof HtmlUnitDriver) {
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {
            // Retrieve current WebClient instance (with the current page) from the Selenium WebDriver
            TargetLocator targetLocator = webDriver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            WebClient webClient = (WebClient) webClientField.get(targetLocator.defaultContent());
            HtmlPage page = (HtmlPage) webClient.getCurrentWindow().getEnclosedPage();
            for (final FrameWindow frameWindow : page.getFrames()) {
                final BaseFrameElement frame = frameWindow.getFrameElement();
                // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
                if (frame.getEnclosedPage().getWebResponse().getWebRequest().getUrl() == WebClient.URL_ABOUT_BLANK) {
                    String src = frame.getSrcAttribute();
                    if (src != null && !src.isEmpty()) {
                        final URL url;
                        try {
                            url = ((HtmlPage) frame.getEnclosedPage()).getFullyQualifiedUrl(src);
                        } catch (final MalformedURLException e) {
                            String message = "Invalid src attribute of " + frame.getTagName() + ": url=[" + src + "]. Ignored.";
                            final IncorrectnessListener incorrectnessListener = webClient.getIncorrectnessListener();
                            incorrectnessListener.notify(message, this);
                            return;
                        }
                        if (isAlreadyLoadedByAncestor(url, ((HtmlPage) frame.getEnclosedPage()))) {
                            String message = "Recursive src attribute of " + frame.getTagName() + ": url=[" + src + "]. Ignored.";
                            final IncorrectnessListener incorrectnessListener = webClient.getIncorrectnessListener();
                            incorrectnessListener.notify(message, this);
                            log.info("Frame already loaded: " + frame.toString());
                            return;
                        }
                        try {
                            final WebRequest request = new WebRequest(url);
                            request.setAdditionalHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9, text/*;q=0.7, */*;q=0.5");
                            if (frameWindow.getName() == null || frameWindow.getName().isEmpty()) {
                                frameWindow.setName("frame_" + page.getFrames().indexOf(frameWindow));
                            }
                            webClient.loadWebResponseInto(webClient.loadWebResponse(request), frameWindow);
                            log.info("Frame loaded: " + frame.toString());
                        } catch (IOException e) {
                            log.error("Error when getting content for " + frame.getTagName() + " with src=" + url, e);
                        }
                    }
                } else {
                    log.info("Frame already loaded: " + frame.toString());
                }
            }
        } catch (Exception e) {
            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {
            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
}

Example 3 with IncorrectnessListener

use of com.gargoylesoftware.htmlunit.IncorrectnessListener in project yacy_grid_loader by yacy.

the class HtmlUnitLoader method initClient.

public static void initClient(String userAgent) {
    userAgentDefault = userAgent;
    if (webClient != null) {
        webClient.getCache().clear();
        webClient.close();
    }
    webClient = new WebClient(getBrowser(userAgent));
    WebClientOptions options = webClient.getOptions();
    options.setJavaScriptEnabled(true);
    options.setCssEnabled(false);
    options.setPopupBlockerEnabled(true);
    options.setRedirectEnabled(true);
    options.setThrowExceptionOnScriptError(false);
    options.setDownloadImages(false);
    options.setGeolocationEnabled(false);
    options.setPrintContentOnFailingStatusCode(false);
    // this might be a bit large, is regulated with throttling and client cache clear in short memory status
    webClient.getCache().setMaxSize(10000);
    webClient.setIncorrectnessListener(new IncorrectnessListener() {

        @Override
        public void notify(String arg0, Object arg1) {
        }
    });
    webClient.setCssErrorHandler(new ErrorHandler() {

        @Override
        public void warning(CSSParseException exception) throws CSSException {
        }

        @Override
        public void fatalError(CSSParseException exception) throws CSSException {
        }

        @Override
        public void error(CSSParseException exception) throws CSSException {
        }
    });
    webClient.setJavaScriptErrorListener(new JavaScriptErrorListener() {

        @Override
        public void timeoutError(HtmlPage arg0, long arg1, long arg2) {
        }

        @Override
        public void scriptException(HtmlPage arg0, ScriptException arg1) {
        }

        @Override
        public void malformedScriptURL(HtmlPage arg0, String arg1, MalformedURLException arg2) {
        }

        @Override
        public void loadScriptError(HtmlPage arg0, URL arg1, Exception arg2) {
        }
    });
    webClient.setHTMLParserListener(new HTMLParserListener() {

        @Override
        public void error(String message, URL url, String html, int line, int column, String key) {
        }

        @Override
        public void warning(String message, URL url, String html, int line, int column, String key) {
        }
    });
}