Examples with HttpRoutePlanner org.apache.http.conn.routing.HttpRoutePlanner used on opensource projects

Example 6 with HttpRoutePlanner

use of org.apache.http.conn.routing.HttpRoutePlanner in project fess-crawler by codelibs.

the class HcHttpClient method init.

@Override
public synchronized void init() {
    if (httpClient != null) {
        return;
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Initializing " + HcHttpClient.class.getName());
    }
    super.init();
    // robots.txt parser
    final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE, Boolean.class);
    if (robotsTxtHelper != null) {
        robotsTxtHelper.setEnabled(robotsTxtEnabled.booleanValue());
    }
    // httpclient
    final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    final Integer connectionTimeoutParam = getInitParameter(CONNECTION_TIMEOUT_PROPERTY, connectionTimeout, Integer.class);
    if (connectionTimeoutParam != null) {
        requestConfigBuilder.setConnectTimeout(connectionTimeoutParam);
    }
    final Integer soTimeoutParam = getInitParameter(SO_TIMEOUT_PROPERTY, soTimeout, Integer.class);
    if (soTimeoutParam != null) {
        requestConfigBuilder.setSocketTimeout(soTimeoutParam);
    }
    // AuthSchemeFactory
    final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create();
    @SuppressWarnings("unchecked") final Map<String, AuthSchemeProvider> factoryMap = getInitParameter(AUTH_SCHEME_PROVIDERS_PROPERTY, authSchemeProviderMap, Map.class);
    if (factoryMap != null) {
        for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) {
            authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
        }
    }
    // user agent
    userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class);
    if (StringUtil.isNotBlank(userAgent)) {
        httpClientBuilder.setUserAgent(userAgent);
    }
    final HttpRoutePlanner planner = buildRoutePlanner();
    if (planner != null) {
        httpClientBuilder.setRoutePlanner(planner);
    }
    // Authentication
    final Authentication[] siteCredentialList = getInitParameter(BASIC_AUTHENTICATIONS_PROPERTY, new Authentication[0], Authentication[].class);
    final List<Pair<FormScheme, Credentials>> formSchemeList = new ArrayList<>();
    for (final Authentication authentication : siteCredentialList) {
        final AuthScheme authScheme = authentication.getAuthScheme();
        if (authScheme instanceof FormScheme) {
            formSchemeList.add(new Pair<>((FormScheme) authScheme, authentication.getCredentials()));
        } else {
            final AuthScope authScope = authentication.getAuthScope();
            credentialsProvider.setCredentials(authScope, authentication.getCredentials());
            if (authScope.getHost() != null && authScheme != null) {
                final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort());
                authCache.put(targetHost, authScheme);
            }
        }
    }
    httpClientContext.setAuthCache(authCache);
    httpClientContext.setCredentialsProvider(credentialsProvider);
    // Request Header
    final RequestHeader[] requestHeaders = getInitParameter(REQUERT_HEADERS_PROPERTY, new RequestHeader[0], RequestHeader[].class);
    for (final RequestHeader requestHeader : requestHeaders) {
        if (requestHeader.isValid()) {
            requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue()));
        }
    }
    // do not redirect
    requestConfigBuilder.setRedirectsEnabled(getInitParameter(REDIRECTS_ENABLED, redirectsEnabled, Boolean.class));
    // cookie
    if (cookieSpec != null) {
        requestConfigBuilder.setCookieSpec(cookieSpec);
    }
    // cookie store
    httpClientBuilder.setDefaultCookieStore(cookieStore);
    if (cookieStore != null) {
        final Cookie[] cookies = getInitParameter(COOKIES_PROPERTY, new Cookie[0], Cookie[].class);
        for (final Cookie cookie : cookies) {
            cookieStore.addCookie(cookie);
        }
    }
    // cookie registry
    final Lookup<CookieSpecProvider> cookieSpecRegistry = buildCookieSpecRegistry();
    if (cookieSpecRegistry != null) {
        httpClientBuilder.setDefaultCookieSpecRegistry(cookieSpecRegistry);
    }
    // SSL
    final LayeredConnectionSocketFactory sslSocketFactory = buildSSLSocketFactory();
    if (sslSocketFactory != null) {
        httpClientBuilder.setSSLSocketFactory(sslSocketFactory);
    }
    connectionMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new HcConnectionMonitorTarget(clientConnectionManager, idleConnectionTimeout), connectionCheckInterval, true);
    final CloseableHttpClient closeableHttpClient = httpClientBuilder.setDnsResolver(dnsResolver).setConnectionManager(clientConnectionManager).setDefaultRequestConfig(requestConfigBuilder.build()).build();
    if (!httpClientPropertyMap.isEmpty()) {
        final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass());
        for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) {
            final String propertyName = entry.getKey();
            if (beanDesc.hasPropertyDesc(propertyName)) {
                final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName);
                propertyDesc.setValue(closeableHttpClient, entry.getValue());
            } else {
                logger.warn("DefaultHttpClient does not have " + propertyName + ".");
            }
        }
    }
    formSchemeList.forEach(p -> {
        final FormScheme scheme = p.getFirst();
        final Credentials credentials = p.getSecond();
        scheme.authenticate(credentials, (request, consumer) -> {
            // request header
            for (final Header header : requestHeaderList) {
                request.addHeader(header);
            }
            HttpEntity httpEntity = null;
            try {
                final HttpResponse response = closeableHttpClient.execute(request, new BasicHttpContext(httpClientContext));
                httpEntity = response.getEntity();
                consumer.accept(response, httpEntity);
            } catch (final Exception e) {
                request.abort();
                logger.warn("Failed to authenticate on " + scheme, e);
            } finally {
                EntityUtils.consumeQuietly(httpEntity);
            }
        });
    });
    httpClient = closeableHttpClient;
}

Example 7 with HttpRoutePlanner

use of org.apache.http.conn.routing.HttpRoutePlanner in project dropwizard by dropwizard.

the class HttpClientBuilderTest method usesACustomRoutePlanner.

@Test
void usesACustomRoutePlanner() throws Exception {
    final HttpRoutePlanner routePlanner = new SystemDefaultRoutePlanner(new ProxySelector() {

        @Override
        public List<Proxy> select(URI uri) {
            return Collections.singletonList(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("192.168.52.1", 8080)));
        }

        @Override
        public void connectFailed(URI uri, SocketAddress sa, IOException ioe) {
        }
    });
    final CloseableHttpClient httpClient = builder.using(configuration).using(routePlanner).createClient(apacheBuilder, connectionManager, "test").getClient();
    assertThat(httpClient).isNotNull();
    assertThat(spyHttpClientBuilderField("routePlanner", apacheBuilder)).isSameAs(routePlanner);
    assertThat(spyHttpClientField("routePlanner", httpClient)).isSameAs(routePlanner);
}

Example 8 with HttpRoutePlanner

use of org.apache.http.conn.routing.HttpRoutePlanner in project pentaho-kettle by pentaho.

the class JobEntryHTTP method getResultFromHttpSchema.

private InputStream getResultFromHttpSchema(String realUploadFile, URI uri) throws IOException, KettleException {
    HttpClient client = null;
    HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    if (!Utils.isEmpty(username)) {
        String realPassword = Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
        String realUser = environmentSubstitute(username);
        UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(realUser, realPassword != null ? realPassword : "");
        CredentialsProvider provider = new BasicCredentialsProvider();
        provider.setCredentials(AuthScope.ANY, credentials);
        clientBuilder.setDefaultCredentialsProvider(provider);
    }
    String proxyHostnameValue = environmentSubstitute(proxyHostname);
    String proxyPortValue = environmentSubstitute(proxyPort);
    String nonProxyHostsValue = environmentSubstitute(nonProxyHosts);
    if (!Utils.isEmpty(proxyHostnameValue)) {
        HttpHost proxy = new HttpHost(proxyHostnameValue, Integer.parseInt(proxyPortValue));
        clientBuilder.setProxy(proxy);
        if (!Utils.isEmpty(nonProxyHostsValue)) {
            HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy) {

                @Override
                public HttpRoute determineRoute(final HttpHost host, final HttpRequest request, final HttpContext context) throws HttpException {
                    String hostName = host.getHostName();
                    if (hostName.matches(nonProxyHostsValue)) {
                        // Return direct route
                        return new HttpRoute(host);
                    }
                    return super.determineRoute(host, request, context);
                }
            };
            clientBuilder.setRoutePlanner(routePlanner);
        }
    }
    client = clientBuilder.build();
    HttpRequestBase httpRequestBase;
    // See if we need to send a file over?
    if (!Utils.isEmpty(realUploadFile)) {
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobHTTP.Log.SendingFile", realUploadFile));
        }
        httpRequestBase = new HttpPost(uri);
        // Get content of file
        String content = new String(Files.readAllBytes(Paths.get(realUploadFile)));
        // upload data to web server
        StringEntity requestEntity = new StringEntity(content);
        requestEntity.setContentType("application/x-www-form-urlencoded");
        ((HttpPost) httpRequestBase).setEntity(requestEntity);
        if (log.isDetailed()) {
            logDetailed(BaseMessages.getString(PKG, "JobHTTP.Log.FinishedSendingFile"));
        }
    } else {
        httpRequestBase = new HttpGet(uri);
    }
    // if we have HTTP headers, add them
    if (!Utils.isEmpty(headerName)) {
        if (log.isDebug()) {
            log.logDebug(BaseMessages.getString(PKG, "JobHTTP.Log.HeadersProvided"));
        }
        for (int j = 0; j < headerName.length; j++) {
            if (!Utils.isEmpty(headerValue[j])) {
                httpRequestBase.addHeader(environmentSubstitute(headerName[j]), environmentSubstitute(headerValue[j]));
                if (log.isDebug()) {
                    log.logDebug(BaseMessages.getString(PKG, "JobHTTP.Log.HeaderSet", environmentSubstitute(headerName[j]), environmentSubstitute(headerValue[j])));
                }
            }
        }
    }
    // Get a stream for the specified URL
    HttpResponse response = null;
    if (!Utils.isEmpty(proxyHostname)) {
        HttpHost target = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
        // Create AuthCache instance
        AuthCache authCache = new BasicAuthCache();
        // Generate BASIC scheme object and add it to the local
        // auth cache
        BasicScheme basicAuth = new BasicScheme();
        authCache.put(target, basicAuth);
        // Add AuthCache to the execution context
        HttpClientContext localContext = HttpClientContext.create();
        localContext.setAuthCache(authCache);
        response = client.execute(target, httpRequestBase, localContext);
    } else {
        response = client.execute(httpRequestBase);
    }
    responseStatusCode = response.getStatusLine().getStatusCode();
    if (HttpStatus.SC_OK != responseStatusCode) {
        throw new KettleException("StatusCode: " + responseStatusCode);
    }
    if (log.isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "JobHTTP.Log.StartReadingReply"));
    }
    logBasic(BaseMessages.getString(PKG, "JobHTTP.Log.ReplayInfo", response.getEntity().getContentType(), response.getLastHeader(HttpHeaders.DATE).getValue()));
    // Read the result from the server...
    return response.getEntity().getContent();
}

Example 9 with HttpRoutePlanner

use of org.apache.http.conn.routing.HttpRoutePlanner in project vscrawler by virjar.

the class CrawlerSession method decorateRoutePlanner.

private void decorateRoutePlanner(CrawlerHttpClient crawlerHttpClient) {
    HttpRoutePlanner routePlanner = crawlerHttpClient.getRoutePlanner();
    if (!(routePlanner instanceof ProxyBindRoutPlanner)) {
        log.warn("自定义了代理发生器,vscrawler的代理功能将不会生效");
        return;
    }
    VSCrawlerRoutePlanner vsCrawlerRoutePlanner = new VSCrawlerRoutePlanner((ProxyBindRoutPlanner) routePlanner, ipPool, proxyPlanner, this);
    crawlerHttpClient.setRoutePlanner(vsCrawlerRoutePlanner);
}

Example 10 with HttpRoutePlanner

use of org.apache.http.conn.routing.HttpRoutePlanner in project java-sdk by optimizely.

the class OptimizelyFactoryTest method newDefaultInstanceWithDatafileAccessTokenAndCustomHttpClient.

@Test
public void newDefaultInstanceWithDatafileAccessTokenAndCustomHttpClient() throws Exception {
    // Add custom Proxy and Port here
    int port = 443;
    String proxyHostName = "someProxy.com";
    HttpHost proxyHost = new HttpHost(proxyHostName, port);
    HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxyHost);
    HttpClientBuilder clientBuilder = HttpClients.custom();
    clientBuilder = clientBuilder.setRoutePlanner(routePlanner);
    CloseableHttpClient httpClient = clientBuilder.build();
    String datafileString = Resources.toString(Resources.getResource("valid-project-config-v4.json"), Charsets.UTF_8);
    optimizely = OptimizelyFactory.newDefaultInstance("sdk-key", datafileString, "auth-token", httpClient);
    assertTrue(optimizely.isValid());
}