Examples with RequestConfig org.apache.http.client.config.RequestConfig used on opensource projects

Example 76 with RequestConfig

use of org.apache.http.client.config.RequestConfig in project intellij-community by JetBrains.

the class EduAdaptiveStepicConnector method setTimeout.

private static void setTimeout(HttpPost request) {
    final RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(CONNECTION_TIMEOUT).setConnectTimeout(CONNECTION_TIMEOUT).setSocketTimeout(CONNECTION_TIMEOUT).build();
    request.setConfig(requestConfig);
}

Example 77 with RequestConfig

use of org.apache.http.client.config.RequestConfig in project libresonic by Libresonic.

the class PodcastService method doDownloadEpisode.

private void doDownloadEpisode(PodcastEpisode episode) {
    InputStream in = null;
    OutputStream out = null;
    if (isEpisodeDeleted(episode)) {
        LOG.info("Podcast " + episode.getUrl() + " was deleted. Aborting download.");
        return;
    }
    LOG.info("Starting to download Podcast from " + episode.getUrl());
    try (CloseableHttpClient client = HttpClients.createDefault()) {
        PodcastChannel channel = getChannel(episode.getChannelId());
        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(// 2 minutes
        2 * 60 * 1000).setSocketTimeout(// 10 minutes
        10 * 60 * 1000).build();
        HttpGet method = new HttpGet(episode.getUrl());
        method.setConfig(requestConfig);
        try (CloseableHttpResponse response = client.execute(method)) {
            in = response.getEntity().getContent();
            File file = getFile(channel, episode);
            out = new FileOutputStream(file);
            episode.setStatus(PodcastStatus.DOWNLOADING);
            episode.setBytesDownloaded(0L);
            episode.setErrorMessage(null);
            episode.setPath(file.getPath());
            podcastDao.updateEpisode(episode);
            byte[] buffer = new byte[4096];
            long bytesDownloaded = 0;
            int n;
            long nextLogCount = 30000L;
            while ((n = in.read(buffer)) != -1) {
                out.write(buffer, 0, n);
                bytesDownloaded += n;
                if (bytesDownloaded > nextLogCount) {
                    episode.setBytesDownloaded(bytesDownloaded);
                    nextLogCount += 30000L;
                    // Abort download if episode was deleted by user.
                    if (isEpisodeDeleted(episode)) {
                        break;
                    }
                    podcastDao.updateEpisode(episode);
                }
            }
            if (isEpisodeDeleted(episode)) {
                LOG.info("Podcast " + episode.getUrl() + " was deleted. Aborting download.");
                IOUtils.closeQuietly(out);
                file.delete();
            } else {
                addMediaFileIdToEpisodes(Arrays.asList(episode));
                episode.setBytesDownloaded(bytesDownloaded);
                podcastDao.updateEpisode(episode);
                LOG.info("Downloaded " + bytesDownloaded + " bytes from Podcast " + episode.getUrl());
                IOUtils.closeQuietly(out);
                updateTags(file, episode);
                episode.setStatus(PodcastStatus.COMPLETED);
                podcastDao.updateEpisode(episode);
                deleteObsoleteEpisodes(channel);
            }
        }
    } catch (Exception x) {
        LOG.warn("Failed to download Podcast from " + episode.getUrl(), x);
        episode.setStatus(PodcastStatus.ERROR);
        episode.setErrorMessage(getErrorMessage(x));
        podcastDao.updateEpisode(episode);
    } finally {
        IOUtils.closeQuietly(in);
        IOUtils.closeQuietly(out);
    }
}

Example 78 with RequestConfig

use of org.apache.http.client.config.RequestConfig in project ats-framework by Axway.

the class HttpClient method createNewHttpClientWithCredentials.

private void createNewHttpClientWithCredentials(String hostname, String userName, String password) throws FileTransferException {
    log.debug("Creating new HTTP client to host " + hostname);
    httpBuilder = HttpClientBuilder.create();
    RequestConfig.Builder requestConfig = RequestConfig.custom();
    requestConfig.setConnectTimeout(this.timeout);
    httpBuilder.setDefaultRequestConfig(requestConfig.build());
    SocketConfig.Builder socket = SocketConfig.custom();
    socket.setRcvBufSize(this.socketBufferSize);
    socket.setSndBufSize(this.socketBufferSize);
    PoolingHttpClientConnectionManager conManager = new PoolingHttpClientConnectionManager();
    // set stale connection check
    conManager.setValidateAfterInactivity(-1);
    httpBuilder.setConnectionManager(conManager);
    Object socketTimeout = customProperties.get(HTTP_HTTPS_SOCKET_READ_TIMEOUT);
    if (socketTimeout != null) {
        // TODO this could be set either with the request config
        socket.setSoTimeout(Integer.parseInt(socketTimeout.toString()));
    }
    httpBuilder.setDefaultSocketConfig(socket.build());
    if (httpClient != null) {
        // cleanup previous client
        disconnect();
    }
    if (userName != null) {
        BasicCredentialsProvider credentials = new BasicCredentialsProvider();
        credentials.setCredentials(new AuthScope(hostname, this.port), new UsernamePasswordCredentials(userName, password));
        httpBuilder.setDefaultCredentialsProvider(credentials);
    }
    httpClient = httpBuilder.build();
}

Example 79 with RequestConfig

use of org.apache.http.client.config.RequestConfig in project mastering-java by Kingminghuang.

the class CrawlerApp method main.

public static void main(String[] args) throws IOException {
    RequestConfig requestConfig = RequestConfig.custom().setProxy(new HttpHost(HttpClientCrawler.DEFAULT_PROXY, HttpClientCrawler.DEFAULT_PORT, HttpClientCrawler.DEFAULT_SCHEMA)).build();
    HttpClientCrawler.config(requestConfig);
    String html = EntityUtils.toString(HttpClientCrawler.download("https://www.zhihu.com/explore/recommendations"), "utf-8");
    Document document = Jsoup.parse(html);
    System.out.println(document.title());
    Elements elements = document.select("a.question_link");
    for (Element element : elements) {
        System.out.println(element.attr("href") + " -> " + element.text());
    }
}

Example 80 with RequestConfig

use of org.apache.http.client.config.RequestConfig in project libresonic by Libresonic.

the class CoverArtService method saveCoverArt.

private void saveCoverArt(String path, String url) throws Exception {
    InputStream input = null;
    OutputStream output = null;
    try (CloseableHttpClient client = HttpClients.createDefault()) {
        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(// 20 seconds
        20 * 1000).setSocketTimeout(// 20 seconds
        20 * 1000).build();
        HttpGet method = new HttpGet(url);
        method.setConfig(requestConfig);
        try (CloseableHttpResponse response = client.execute(method)) {
            input = response.getEntity().getContent();
            // Attempt to resolve proper suffix.
            String suffix = "jpg";
            if (url.toLowerCase().endsWith(".gif")) {
                suffix = "gif";
            } else if (url.toLowerCase().endsWith(".png")) {
                suffix = "png";
            }
            // Check permissions.
            File newCoverFile = new File(path, "cover." + suffix);
            if (!securityService.isWriteAllowed(newCoverFile)) {
                throw new Exception("Permission denied: " + StringUtil.toHtml(newCoverFile.getPath()));
            }
            // If file exists, create a backup.
            backup(newCoverFile, new File(path, "cover." + suffix + ".backup"));
            // Write file.
            output = new FileOutputStream(newCoverFile);
            IOUtils.copy(input, output);
            MediaFile dir = mediaFileService.getMediaFile(path);
            // Refresh database.
            mediaFileService.refreshMediaFile(dir);
            dir = mediaFileService.getMediaFile(dir.getId());
            // Rename existing cover files if new cover file is not the preferred.
            try {
                while (true) {
                    File coverFile = mediaFileService.getCoverArt(dir);
                    if (coverFile != null && !isMediaFile(coverFile) && !newCoverFile.equals(coverFile)) {
                        if (!coverFile.renameTo(new File(coverFile.getCanonicalPath() + ".old"))) {
                            LOG.warn("Unable to rename old image file " + coverFile);
                            break;
                        }
                        LOG.info("Renamed old image file " + coverFile);
                        // Must refresh again.
                        mediaFileService.refreshMediaFile(dir);
                        dir = mediaFileService.getMediaFile(dir.getId());
                    } else {
                        break;
                    }
                }
            } catch (Exception x) {
                LOG.warn("Failed to rename existing cover file.", x);
            }
        }
    } finally {
        IOUtils.closeQuietly(input);
        IOUtils.closeQuietly(output);
    }
}