Examples with HttpClientConnectionManager org.apache.http.conn.HttpClientConnectionManager used on opensource projects

Example 26 with HttpClientConnectionManager

use of org.apache.http.conn.HttpClientConnectionManager in project cuba by cuba-platform.

the class IdpServiceLogoutCallbackInvoker method performLogoutOnServiceProviders.

public void performLogoutOnServiceProviders(String idpSessionId) {
    List<String> serviceProviderLogoutUrls = idpConfig.getServiceProviderLogoutUrls();
    asyncServiceLogoutExecutor.submit(() -> {
        HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager();
        HttpClient client = HttpClientBuilder.create().setConnectionManager(connectionManager).setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)).build();
        try {
            for (String serviceProviderLogoutUrl : serviceProviderLogoutUrls) {
                callLoggedOutServiceUrl(client, serviceProviderLogoutUrl, idpSessionId);
            }
        } catch (Throwable ex) {
            log.error("Unable to perform logout on IDP services for session {}", idpSessionId, ex);
        } finally {
            connectionManager.shutdown();
        }
    });
}

Example 27 with HttpClientConnectionManager

use of org.apache.http.conn.HttpClientConnectionManager in project nifi by apache.

the class PostHTTP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile firstFlowFile = session.get();
    if (firstFlowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final String url = context.getProperty(URL).evaluateAttributeExpressions(firstFlowFile).getValue();
    try {
        new java.net.URL(url);
    } catch (final MalformedURLException e) {
        logger.error("After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure", new Object[] { firstFlowFile, url });
        firstFlowFile = session.penalize(firstFlowFile);
        session.transfer(firstFlowFile, REL_FAILURE);
        return;
    }
    final List<FlowFile> toSend = new ArrayList<>();
    toSend.add(firstFlowFile);
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();
    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder.setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();
    final StreamThrottler throttler = throttlerRef.get();
    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    final AtomicLong bytesToSend = new AtomicLong(firstFlowFile.getSize());
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();
    final AtomicReference<String> dnHolder = new AtomicReference<>("none");
    final Config config = getConfig(url, context);
    final HttpClientConnectionManager conMan = config.getConnectionManager();
    final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    clientBuilder.setConnectionManager(conMan);
    clientBuilder.setUserAgent(userAgent);
    clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {

        @Override
        public void process(final HttpResponse response, final HttpContext httpContext) throws HttpException, IOException {
            final HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
            final ManagedHttpClientConnection conn = coreContext.getConnection(ManagedHttpClientConnection.class);
            if (!conn.isOpen()) {
                return;
            }
            final SSLSession sslSession = conn.getSSLSession();
            if (sslSession != null) {
                final Certificate[] certChain = sslSession.getPeerCertificates();
                if (certChain == null || certChain.length == 0) {
                    throw new SSLPeerUnverifiedException("No certificates found");
                }
                try {
                    final X509Certificate cert = CertificateUtils.convertAbstractX509Certificate(certChain[0]);
                    dnHolder.set(cert.getSubjectDN().getName().trim());
                } catch (CertificateException e) {
                    final String msg = "Could not extract subject DN from SSL session peer certificate";
                    logger.warn(msg);
                    throw new SSLPeerUnverifiedException(msg);
                }
            }
        }
    });
    clientBuilder.disableAutomaticRetries();
    clientBuilder.disableContentCompression();
    final String username = context.getProperty(USERNAME).getValue();
    final String password = context.getProperty(PASSWORD).getValue();
    // set the credentials if appropriate
    if (username != null) {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        if (password == null) {
            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username));
        } else {
            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
        }
        clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    }
    // Set the proxy if specified
    if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) {
        final String host = context.getProperty(PROXY_HOST).getValue();
        final int port = context.getProperty(PROXY_PORT).asInteger();
        clientBuilder.setProxy(new HttpHost(host, port));
    }
    client = clientBuilder.build();
    // determine whether or not destination accepts flowfile/gzip
    destinationAccepts = config.getDestinationAccepts();
    if (destinationAccepts == null) {
        try {
            destinationAccepts = getDestinationAcceptance(sendAsFlowFile, client, url, getLogger(), transactionId);
            config.setDestinationAccepts(destinationAccepts);
        } catch (final IOException e) {
            firstFlowFile = session.penalize(firstFlowFile);
            session.transfer(firstFlowFile, REL_FAILURE);
            logger.error("Unable to communicate with destination {} to determine whether or not it can accept " + "flowfiles/gzip; routing {} to failure due to {}", new Object[] { url, firstFlowFile, e });
            context.yield();
            return;
        }
    }
    // then we can get more flowfiles from the session up to MAX_BATCH_SIZE for the same URL
    if (sendAsFlowFile && (destinationAccepts.isFlowFileV3Accepted() || destinationAccepts.isFlowFileV2Accepted())) {
        toSend.addAll(session.get(new FlowFileFilter() {

            @Override
            public FlowFileFilterResult filter(FlowFile flowFile) {
                // if over MAX_BATCH_SIZE, then stop adding files
                if (bytesToSend.get() + flowFile.getSize() > maxBatchBytes) {
                    return FlowFileFilterResult.REJECT_AND_TERMINATE;
                }
                // check URL to see if this flowfile can be included in the batch
                final String urlToCheck = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
                if (url.equals(urlToCheck)) {
                    bytesToSend.addAndGet(flowFile.getSize());
                    return FlowFileFilterResult.ACCEPT_AND_CONTINUE;
                } else {
                    return FlowFileFilterResult.REJECT_AND_CONTINUE;
                }
            }
        }));
    }
    final HttpPost post = new HttpPost(url);
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;
    final EntityTemplate entity = new EntityTemplate(new ContentProducer() {

        @Override
        public void writeTo(final OutputStream rawOut) throws IOException {
            final OutputStream throttled = throttler == null ? rawOut : throttler.newThrottledOutputStream(rawOut);
            OutputStream wrappedOut = new BufferedOutputStream(throttled);
            if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
            }
            try (final OutputStream out = wrappedOut) {
                for (final FlowFile flowFile : toSend) {
                    session.read(flowFile, new InputStreamCallback() {

                        @Override
                        public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {
                                FlowFilePackager packager = null;
                                if (!sendAsFlowFile) {
                                    packager = null;
                                } else if (accepts.isFlowFileV3Accepted()) {
                                    packager = new FlowFilePackagerV3();
                                } else if (accepts.isFlowFileV2Accepted()) {
                                    packager = new FlowFilePackagerV2();
                                } else if (accepts.isFlowFileV1Accepted()) {
                                    packager = new FlowFilePackagerV1();
                                }
                                // formats is acceptable if sending as FlowFile.
                                if (packager == null) {
                                    StreamUtils.copy(in, out);
                                } else {
                                    final Map<String, String> flowFileAttributes;
                                    if (isDestinationLegacyNiFi) {
                                        // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path;
                                        // in order to maintain backward compatibility, we copy the filename & path to those attribute keys.
                                        flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                        flowFileAttributes.put("nf.file.name", flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                        flowFileAttributes.put("nf.file.path", flowFile.getAttribute(CoreAttributes.PATH.key()));
                                    } else {
                                        flowFileAttributes = flowFile.getAttributes();
                                    }
                                    packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize());
                                }
                            }
                        }
                    });
                }
                out.flush();
            }
        }
    }) {

        @Override
        public long getContentLength() {
            if (compressionLevel == 0 && !sendAsFlowFile && !context.getProperty(CHUNKED_ENCODING).asBoolean()) {
                return toSend.get(0).getSize();
            } else {
                return -1;
            }
        }
    };
    if (context.getProperty(CHUNKED_ENCODING).isSet()) {
        entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    }
    post.setEntity(entity);
    post.setConfig(requestConfig);
    final String contentType;
    if (sendAsFlowFile) {
        if (accepts.isFlowFileV3Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V3;
        } else if (accepts.isFlowFileV2Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V2;
        } else if (accepts.isFlowFileV1Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V1;
        } else {
            logger.error("Cannot send data to {} because the destination does not accept FlowFiles and this processor is " + "configured to deliver FlowFiles; rolling back session", new Object[] { url });
            session.rollback();
            context.yield();
            IOUtils.closeQuietly(client);
            return;
        }
    } else {
        final String contentTypeValue = context.getProperty(CONTENT_TYPE).evaluateAttributeExpressions(toSend.get(0)).getValue();
        contentType = StringUtils.isBlank(contentTypeValue) ? DEFAULT_CONTENT_TYPE : contentTypeValue;
    }
    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && toSend.size() == 1) {
        final Pattern pattern = Pattern.compile(attributeHeaderRegex);
        final Map<String, String> attributes = toSend.get(0).getAttributes();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            final String key = entry.getKey();
            if (pattern.matcher(key).matches()) {
                post.setHeader(entry.getKey(), entry.getValue());
            }
        }
    }
    post.setHeader(CONTENT_TYPE_HEADER, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
        if (sendAsFlowFile) {
            post.setHeader(GZIPPED_HEADER, "true");
        } else {
            post.setHeader(CONTENT_ENCODING_HEADER, CONTENT_ENCODING_GZIP_VALUE);
        }
    }
    // Do the actual POST
    final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";
    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        response = client.execute(post);
        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        EntityUtils.consume(response.getEntity());
        stopWatch.stop();
        uploadDataRate = stopWatch.calculateDataRate(bytesToSend.get());
        uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
        logger.error("Failed to Post {} due to {}; transferring to failure", new Object[] { flowFileDescription, e });
        context.yield();
        for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
        return;
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (final IOException e) {
                getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e });
            }
        }
    }
    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
        final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
        if (locationUriHeader != null) {
            if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
                final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
                if (holdUriHeader != null) {
                    holdUri = holdUriHeader.getValue();
                }
            }
        }
        if (holdUri == null) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI", new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
    if (holdUri == null) {
        if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}; will yield processing, " + "since the destination is temporarily unavailable", new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            context.yield();
            return;
        }
        if (responseCode >= 300) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}", new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
        logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription, url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate });
        for (final FlowFile flowFile : toSend) {
            session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true);
            session.transfer(flowFile, REL_SUCCESS);
        }
        return;
    }
    // 
    // the response indicated a Hold URI; delete the Hold.
    // 
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
        // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
        // as this really indicates that it should be whatever we posted to -- if posting directly to the
        // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
        // be posting to some other URL.
        fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
        // URL indicates the full path but not hostname or port; use the same hostname & port that we posted
        // to but use the full path indicated by the response.
        int firstSlash = url.indexOf("/", 8);
        if (firstSlash < 0) {
            firstSlash = url.length();
        }
        final String beforeSlash = url.substring(0, firstSlash);
        fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
        // Absolute URL
        fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }
    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);
    while (true) {
        try {
            final HttpResponse holdResponse = client.execute(delete);
            EntityUtils.consume(holdResponse.getEntity());
            final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
            final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
            if (holdStatusCode >= 300) {
                logger.error("Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure", new Object[] { flowFileDescription, holdStatusCode, holdReason });
                for (FlowFile flowFile : toSend) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                }
                return;
            }
            logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}", new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate });
            for (final FlowFile flowFile : toSend) {
                session.getProvenanceReporter().send(flowFile, url);
                session.transfer(flowFile, REL_SUCCESS);
            }
            return;
        } catch (final IOException e) {
            logger.warn("Failed to delete Hold that destination placed on {} due to {}", new Object[] { flowFileDescription, e });
        }
        if (!isScheduled()) {
            context.yield();
            logger.warn("Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure", new Object[] { flowFileDescription });
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
}

Example 28 with HttpClientConnectionManager

use of org.apache.http.conn.HttpClientConnectionManager in project nifi by apache.

the class GetHTTP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException {
    final ComponentLog logger = getLogger();
    final ProcessSession session = sessionFactory.createSession();
    final FlowFile incomingFlowFile = session.get();
    if (incomingFlowFile != null) {
        session.transfer(incomingFlowFile, REL_SUCCESS);
        logger.warn("found FlowFile {} in input queue; transferring to success", new Object[] { incomingFlowFile });
    }
    // get the URL
    final String url = context.getProperty(URL).evaluateAttributeExpressions().getValue();
    final URI uri;
    String source = url;
    try {
        uri = new URI(url);
        source = uri.getHost();
    } catch (final URISyntaxException swallow) {
    // this won't happen as the url has already been validated
    }
    // get the ssl context service
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    // create the connection manager
    final HttpClientConnectionManager conMan;
    if (sslContextService == null) {
        conMan = new BasicHttpClientConnectionManager();
    } else {
        final SSLContext sslContext;
        try {
            sslContext = createSSLContext(sslContextService);
        } catch (final Exception e) {
            throw new ProcessException(e);
        }
        final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
        // Also include a plain socket factory for regular http connections (especially proxies)
        final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslsf).register("http", PlainConnectionSocketFactory.getSocketFactory()).build();
        conMan = new BasicHttpClientConnectionManager(socketFactoryRegistry);
    }
    try {
        // build the request configuration
        final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
        requestConfigBuilder.setConnectionRequestTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setConnectTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setRedirectsEnabled(context.getProperty(FOLLOW_REDIRECTS).asBoolean());
        switch(context.getProperty(REDIRECT_COOKIE_POLICY).getValue()) {
            case STANDARD_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD);
                break;
            case STRICT_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD_STRICT);
                break;
            case NETSCAPE_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.NETSCAPE);
                break;
            case IGNORE_COOKIE_POLICY_STR:
                requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
                break;
            case DEFAULT_COOKIE_POLICY_STR:
            default:
                requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT);
        }
        // build the http client
        final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
        clientBuilder.setConnectionManager(conMan);
        // include the user agent
        final String userAgent = context.getProperty(USER_AGENT).getValue();
        if (userAgent != null) {
            clientBuilder.setUserAgent(userAgent);
        }
        // set the ssl context if necessary
        if (sslContextService != null) {
            clientBuilder.setSslcontext(sslContextService.createSSLContext(ClientAuth.REQUIRED));
        }
        final String username = context.getProperty(USERNAME).getValue();
        final String password = context.getProperty(PASSWORD).getValue();
        // set the credentials if appropriate
        if (username != null) {
            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            if (password == null) {
                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username));
            } else {
                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
            }
            clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }
        // Set the proxy if specified
        if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) {
            final String host = context.getProperty(PROXY_HOST).getValue();
            final int port = context.getProperty(PROXY_PORT).asInteger();
            clientBuilder.setProxy(new HttpHost(host, port));
        }
        // create request
        final HttpGet get = new HttpGet(url);
        get.setConfig(requestConfigBuilder.build());
        final StateMap beforeStateMap;
        try {
            beforeStateMap = context.getStateManager().getState(Scope.LOCAL);
            final String lastModified = beforeStateMap.get(LAST_MODIFIED + ":" + url);
            if (lastModified != null) {
                get.addHeader(HEADER_IF_MODIFIED_SINCE, parseStateValue(lastModified).getValue());
            }
            final String etag = beforeStateMap.get(ETAG + ":" + url);
            if (etag != null) {
                get.addHeader(HEADER_IF_NONE_MATCH, parseStateValue(etag).getValue());
            }
        } catch (final IOException ioe) {
            throw new ProcessException(ioe);
        }
        final String accept = context.getProperty(ACCEPT_CONTENT_TYPE).getValue();
        if (accept != null) {
            get.addHeader(HEADER_ACCEPT, accept);
        }
        // Add dynamic headers
        PropertyValue customHeaderValue;
        for (PropertyDescriptor customProperty : customHeaders) {
            customHeaderValue = context.getProperty(customProperty).evaluateAttributeExpressions();
            if (StringUtils.isNotBlank(customHeaderValue.getValue())) {
                get.addHeader(customProperty.getName(), customHeaderValue.getValue());
            }
        }
        // create the http client
        try (final CloseableHttpClient client = clientBuilder.build()) {
            // NOTE: including this inner try in order to swallow exceptions on close
            try {
                final StopWatch stopWatch = new StopWatch(true);
                final HttpResponse response = client.execute(get);
                final int statusCode = response.getStatusLine().getStatusCode();
                if (statusCode == NOT_MODIFIED) {
                    logger.info("content not retrieved because server returned HTTP Status Code {}: Not Modified", new Object[] { NOT_MODIFIED });
                    context.yield();
                    // doing a commit in case there were flow files in the input queue
                    session.commit();
                    return;
                }
                final String statusExplanation = response.getStatusLine().getReasonPhrase();
                if ((statusCode >= 300) || (statusCode == 204)) {
                    logger.error("received status code {}:{} from {}", new Object[] { statusCode, statusExplanation, url });
                    // doing a commit in case there were flow files in the input queue
                    session.commit();
                    return;
                }
                FlowFile flowFile = session.create();
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), context.getProperty(FILENAME).evaluateAttributeExpressions().getValue());
                flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", source);
                flowFile = session.importFrom(response.getEntity().getContent(), flowFile);
                final Header contentTypeHeader = response.getFirstHeader("Content-Type");
                if (contentTypeHeader != null) {
                    final String contentType = contentTypeHeader.getValue();
                    if (!contentType.trim().isEmpty()) {
                        flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), contentType.trim());
                    }
                }
                final long flowFileSize = flowFile.getSize();
                stopWatch.stop();
                final String dataRate = stopWatch.calculateDataRate(flowFileSize);
                session.getProvenanceReporter().receive(flowFile, url, stopWatch.getDuration(TimeUnit.MILLISECONDS));
                session.transfer(flowFile, REL_SUCCESS);
                logger.info("Successfully received {} from {} at a rate of {}; transferred to success", new Object[] { flowFile, url, dataRate });
                session.commit();
                updateStateMap(context, response, beforeStateMap, url);
            } catch (final IOException e) {
                context.yield();
                session.rollback();
                logger.error("Failed to retrieve file from {} due to {}; rolling back session", new Object[] { url, e.getMessage() }, e);
                throw new ProcessException(e);
            } catch (final Throwable t) {
                context.yield();
                session.rollback();
                logger.error("Failed to process due to {}; rolling back session", new Object[] { t.getMessage() }, t);
                throw t;
            }
        } catch (final IOException e) {
            logger.debug("Error closing client due to {}, continuing.", new Object[] { e.getMessage() });
        }
    } finally {
        conMan.shutdown();
    }
}

Example 29 with HttpClientConnectionManager

use of org.apache.http.conn.HttpClientConnectionManager in project pwm by pwm-project.

the class PwmHttpClient method getHttpClient.

public static HttpClient getHttpClient(final Configuration configuration, final PwmHttpClientConfiguration pwmHttpClientConfiguration) throws PwmUnrecoverableException {
    final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    clientBuilder.setUserAgent(PwmConstants.PWM_APP_NAME + " " + PwmConstants.SERVLET_VERSION);
    final boolean httpClientPromiscuousEnable = Boolean.parseBoolean(configuration.readAppProperty(AppProperty.SECURITY_HTTP_PROMISCUOUS_ENABLE));
    try {
        if (httpClientPromiscuousEnable || (pwmHttpClientConfiguration != null && pwmHttpClientConfiguration.isPromiscuous())) {
            clientBuilder.setSSLContext(promiscuousSSLContext());
            clientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
        } else if (pwmHttpClientConfiguration != null && pwmHttpClientConfiguration.getCertificates() != null) {
            final SSLContext sslContext = SSLContext.getInstance("SSL");
            final TrustManager trustManager = new X509Utils.CertMatchingTrustManager(configuration, pwmHttpClientConfiguration.getCertificates());
            sslContext.init(null, new TrustManager[] { trustManager }, new SecureRandom());
            final SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
            final Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslConnectionFactory).build();
            final HttpClientConnectionManager ccm = new BasicHttpClientConnectionManager(registry);
            clientBuilder.setSSLSocketFactory(sslConnectionFactory);
            clientBuilder.setConnectionManager(ccm);
        }
    } catch (Exception e) {
        throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN, "unexpected error creating promiscuous https client: " + e.getMessage()));
    }
    final String proxyUrl = configuration.readSettingAsString(PwmSetting.HTTP_PROXY_URL);
    if (proxyUrl != null && proxyUrl.length() > 0) {
        final URI proxyURI = URI.create(proxyUrl);
        final String host = proxyURI.getHost();
        final int port = proxyURI.getPort();
        clientBuilder.setProxy(new HttpHost(host, port));
        final String userInfo = proxyURI.getUserInfo();
        if (userInfo != null && userInfo.length() > 0) {
            final String[] parts = userInfo.split(":");
            final String username = parts[0];
            final String password = (parts.length > 1) ? parts[1] : "";
            final CredentialsProvider credsProvider = new BasicCredentialsProvider();
            credsProvider.setCredentials(new AuthScope(host, port), new UsernamePasswordCredentials(username, password));
            clientBuilder.setDefaultCredentialsProvider(credsProvider);
            clientBuilder.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy());
        }
    }
    clientBuilder.setDefaultRequestConfig(RequestConfig.copy(RequestConfig.DEFAULT).setSocketTimeout(Integer.parseInt(configuration.readAppProperty(AppProperty.HTTP_CLIENT_SOCKET_TIMEOUT_MS))).setConnectTimeout(Integer.parseInt(configuration.readAppProperty(AppProperty.HTTP_CLIENT_CONNECT_TIMEOUT_MS))).setConnectionRequestTimeout(Integer.parseInt(configuration.readAppProperty(AppProperty.HTTP_CLIENT_REQUEST_TIMEOUT_MS))).build());
    return clientBuilder.build();
}

Example 30 with HttpClientConnectionManager

use of org.apache.http.conn.HttpClientConnectionManager in project cuba by cuba-platform.

the class LogDataProvider method provide.

/**
 * You should call {@link LogDataProvider#obtainUrl()} before
 */
@Override
public InputStream provide() {
    HttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager();
    HttpClient httpClient = HttpClientBuilder.create().setConnectionManager(connectionManager).build();
    String uri = url + "?s=" + userSessionSource.getUserSession().getId();
    if (downloadFullLog) {
        uri += "&full=true";
    }
    HttpGet httpGet = new HttpGet(uri);
    try {
        HttpResponse httpResponse = httpClient.execute(httpGet);
        int httpStatus = httpResponse.getStatusLine().getStatusCode();
        if (httpStatus == HttpStatus.SC_OK) {
            HttpEntity httpEntity = httpResponse.getEntity();
            if (httpEntity != null) {
                inputStream = httpEntity.getContent();
            } else {
                log.debug("Unable to download log from " + url + "\nHttpEntity is null");
                throw new RuntimeException("Unable to download log from " + url + "\nHttpEntity is null");
            }
        } else {
            log.debug("Unable to download log from " + url + "\n" + httpResponse.getStatusLine());
            throw new RuntimeException("Unable to download log from " + url + "\n" + httpResponse.getStatusLine());
        }
    } catch (IOException e) {
        log.debug("Unable to download log from " + url + "\n" + e);
        throw new RuntimeException(e);
    }
    return inputStream;
}