Examples with HTMLParserListener com.gargoylesoftware.htmlunit.html.parser.HTMLParserListener used on opensource projects

Example 1 with HTMLParserListener

use of com.gargoylesoftware.htmlunit.html.parser.HTMLParserListener in project yacy_grid_loader by yacy.

the class HtmlUnitLoader method getClient.

public static WebClient getClient(String userAgent) {
    WebClient webClient = new WebClient(getBrowser(userAgent));
    WebClientOptions options = webClient.getOptions();
    options.setJavaScriptEnabled(true);
    options.setCssEnabled(false);
    options.setPopupBlockerEnabled(true);
    options.setRedirectEnabled(true);
    options.setDownloadImages(false);
    options.setGeolocationEnabled(false);
    options.setPrintContentOnFailingStatusCode(false);
    options.setThrowExceptionOnScriptError(false);
    options.setMaxInMemory(0);
    options.setHistoryPageCacheLimit(0);
    options.setHistorySizeLimit(0);
    // ProxyConfig proxyConfig = new ProxyConfig();
    // proxyConfig.setProxyHost("127.0.0.1");
    // proxyConfig.setProxyPort(Service.getPort());
    // options.setProxyConfig(proxyConfig);
    // this might be a bit large, is regulated with throttling and client cache clear in short memory status
    webClient.getCache().setMaxSize(10000);
    webClient.setIncorrectnessListener(new IncorrectnessListener() {

        @Override
        public void notify(String arg0, Object arg1) {
        }
    });
    webClient.setCssErrorHandler(new CSSErrorHandler() {

        @Override
        public void warning(CSSParseException exception) throws CSSException {
        }

        @Override
        public void error(CSSParseException exception) throws CSSException {
        }

        @Override
        public void fatalError(CSSParseException exception) throws CSSException {
        }
    });
    webClient.setJavaScriptErrorListener(new JavaScriptErrorListener() {

        @Override
        public void timeoutError(HtmlPage arg0, long arg1, long arg2) {
        }

        @Override
        public void scriptException(HtmlPage arg0, ScriptException arg1) {
        }

        @Override
        public void malformedScriptURL(HtmlPage arg0, String arg1, MalformedURLException arg2) {
        }

        @Override
        public void loadScriptError(HtmlPage arg0, URL arg1, Exception arg2) {
        }

        @Override
        public void warn(String message, String sourceName, int line, String lineSource, int lineOffset) {
        }
    });
    webClient.setHTMLParserListener(new HTMLParserListener() {

        @Override
        public void error(String message, URL url, String html, int line, int column, String key) {
        }

        @Override
        public void warning(String message, URL url, String html, int line, int column, String key) {
        }
    });
    return webClient;
}