Examples with DecompressingHttpClient org.apache.http.impl.client.DecompressingHttpClient used on opensource projects

Example 1 with DecompressingHttpClient

use of org.apache.http.impl.client.DecompressingHttpClient in project OpenRefine by OpenRefine.

the class ImportingUtilities method retrieveContentFromPostRequest.

public static void retrieveContentFromPostRequest(HttpServletRequest request, Properties parameters, File rawDataDir, JSONObject retrievalRecord, final Progress progress) throws Exception {
    JSONArray fileRecords = new JSONArray();
    JSONUtilities.safePut(retrievalRecord, "files", fileRecords);
    int clipboardCount = 0;
    int uploadCount = 0;
    int downloadCount = 0;
    int archiveCount = 0;
    // This tracks the total progress, which involves uploading data from the client
    // as well as downloading data from URLs.
    final SavingUpdate update = new SavingUpdate() {

        @Override
        public void savedMore() {
            progress.setProgress(null, calculateProgressPercent(totalExpectedSize, totalRetrievedSize));
        }

        @Override
        public boolean isCanceled() {
            return progress.isCanceled();
        }
    };
    DiskFileItemFactory fileItemFactory = new DiskFileItemFactory();
    ServletFileUpload upload = new ServletFileUpload(fileItemFactory);
    upload.setProgressListener(new ProgressListener() {

        boolean setContentLength = false;

        long lastBytesRead = 0;

        @Override
        public void update(long bytesRead, long contentLength, int itemCount) {
            if (!setContentLength) {
                // Only try to set the content length if we really know it.
                if (contentLength >= 0) {
                    update.totalExpectedSize += contentLength;
                    setContentLength = true;
                }
            }
            if (setContentLength) {
                update.totalRetrievedSize += (bytesRead - lastBytesRead);
                lastBytesRead = bytesRead;
                update.savedMore();
            }
        }
    });
    @SuppressWarnings("unchecked") List<FileItem> tempFiles = (List<FileItem>) upload.parseRequest(request);
    progress.setProgress("Uploading data ...", -1);
    parts: for (FileItem fileItem : tempFiles) {
        if (progress.isCanceled()) {
            break;
        }
        InputStream stream = fileItem.getInputStream();
        String name = fileItem.getFieldName().toLowerCase();
        if (fileItem.isFormField()) {
            if (name.equals("clipboard")) {
                String encoding = request.getCharacterEncoding();
                if (encoding == null) {
                    encoding = "UTF-8";
                }
                File file = allocateFile(rawDataDir, "clipboard.txt");
                JSONObject fileRecord = new JSONObject();
                JSONUtilities.safePut(fileRecord, "origin", "clipboard");
                JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding);
                JSONUtilities.safePut(fileRecord, "declaredMimeType", (String) null);
                JSONUtilities.safePut(fileRecord, "format", "text");
                JSONUtilities.safePut(fileRecord, "fileName", "(clipboard)");
                JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
                progress.setProgress("Uploading pasted clipboard text", calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
                JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
                JSONUtilities.append(fileRecords, fileRecord);
                clipboardCount++;
            } else if (name.equals("download")) {
                String urlString = Streams.asString(stream);
                URL url = new URL(urlString);
                JSONObject fileRecord = new JSONObject();
                JSONUtilities.safePut(fileRecord, "origin", "download");
                JSONUtilities.safePut(fileRecord, "url", urlString);
                for (UrlRewriter rewriter : ImportingManager.urlRewriters) {
                    Result result = rewriter.rewrite(urlString);
                    if (result != null) {
                        urlString = result.rewrittenUrl;
                        url = new URL(urlString);
                        JSONUtilities.safePut(fileRecord, "url", urlString);
                        JSONUtilities.safePut(fileRecord, "format", result.format);
                        if (!result.download) {
                            downloadCount++;
                            JSONUtilities.append(fileRecords, fileRecord);
                            continue parts;
                        }
                    }
                }
                if ("http".equals(url.getProtocol()) || "https".equals(url.getProtocol())) {
                    DefaultHttpClient client = new DefaultHttpClient();
                    DecompressingHttpClient httpclient = new DecompressingHttpClient(client);
                    HttpGet httpGet = new HttpGet(url.toURI());
                    httpGet.setHeader("User-Agent", RefineServlet.getUserAgent());
                    if ("https".equals(url.getProtocol())) {
                        // HTTPS only - no sending password in the clear over HTTP
                        String userinfo = url.getUserInfo();
                        if (userinfo != null) {
                            int s = userinfo.indexOf(':');
                            if (s > 0) {
                                String user = userinfo.substring(0, s);
                                String pw = userinfo.substring(s + 1, userinfo.length());
                                client.getCredentialsProvider().setCredentials(new AuthScope(url.getHost(), 443), new UsernamePasswordCredentials(user, pw));
                            }
                        }
                    }
                    HttpResponse response = httpclient.execute(httpGet);
                    try {
                        response.getStatusLine();
                        HttpEntity entity = response.getEntity();
                        if (entity == null) {
                            throw new Exception("No content found in " + url.toString());
                        }
                        InputStream stream2 = entity.getContent();
                        String encoding = null;
                        if (entity.getContentEncoding() != null) {
                            encoding = entity.getContentEncoding().getValue();
                        }
                        JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding);
                        String contentType = null;
                        if (entity.getContentType() != null) {
                            contentType = entity.getContentType().getValue();
                        }
                        JSONUtilities.safePut(fileRecord, "declaredMimeType", contentType);
                        if (saveStream(stream2, url, rawDataDir, progress, update, fileRecord, fileRecords, entity.getContentLength())) {
                            archiveCount++;
                        }
                        downloadCount++;
                        EntityUtils.consume(entity);
                    } finally {
                        httpGet.releaseConnection();
                    }
                } else {
                    // Fallback handling for non HTTP connections (only FTP?)
                    URLConnection urlConnection = url.openConnection();
                    urlConnection.setConnectTimeout(5000);
                    urlConnection.connect();
                    InputStream stream2 = urlConnection.getInputStream();
                    JSONUtilities.safePut(fileRecord, "declaredEncoding", urlConnection.getContentEncoding());
                    JSONUtilities.safePut(fileRecord, "declaredMimeType", urlConnection.getContentType());
                    try {
                        if (saveStream(stream2, url, rawDataDir, progress, update, fileRecord, fileRecords, urlConnection.getContentLength())) {
                            archiveCount++;
                        }
                        downloadCount++;
                    } finally {
                        stream2.close();
                    }
                }
            } else {
                String value = Streams.asString(stream);
                parameters.put(name, value);
            // TODO: We really want to store this on the request so it's available for everyone
            //                    request.getParameterMap().put(name, value);
            }
        } else {
            // is file content
            String fileName = fileItem.getName();
            if (fileName.length() > 0) {
                long fileSize = fileItem.getSize();
                File file = allocateFile(rawDataDir, fileName);
                JSONObject fileRecord = new JSONObject();
                JSONUtilities.safePut(fileRecord, "origin", "upload");
                JSONUtilities.safePut(fileRecord, "declaredEncoding", request.getCharacterEncoding());
                JSONUtilities.safePut(fileRecord, "declaredMimeType", fileItem.getContentType());
                JSONUtilities.safePut(fileRecord, "fileName", fileName);
                JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
                progress.setProgress("Saving file " + fileName + " locally (" + formatBytes(fileSize) + " bytes)", calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
                JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
                if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
                    archiveCount++;
                }
                uploadCount++;
            }
        }
        stream.close();
    }
    // Delete all temp files.
    for (FileItem fileItem : tempFiles) {
        fileItem.delete();
    }
    JSONUtilities.safePut(retrievalRecord, "uploadCount", uploadCount);
    JSONUtilities.safePut(retrievalRecord, "downloadCount", downloadCount);
    JSONUtilities.safePut(retrievalRecord, "clipboardCount", clipboardCount);
    JSONUtilities.safePut(retrievalRecord, "archiveCount", archiveCount);
}

Example 2 with DecompressingHttpClient

use of org.apache.http.impl.client.DecompressingHttpClient in project platformlayer by platformlayer.

the class MetricClientImpl method buildHttpClient.

private HttpClient buildHttpClient(CertificateAndKey certificateAndKey, List<String> trustKeys) {
    int port = metricBaseUrl.getPort();
    if (port == -1) {
        String scheme = metricBaseUrl.getScheme();
        if (scheme.equals("https")) {
            port = 443;
        } else if (scheme.equals("http")) {
            port = 80;
        } else {
            throw new IllegalArgumentException("Unknown scheme: " + scheme);
        }
    }
    SchemeSocketFactory schemeSocketFactory;
    try {
        KeyManager keyManager = new SimpleClientCertificateKeyManager(certificateAndKey);
        TrustManager trustManager;
        X509HostnameVerifier hostnameVerifier;
        if (trustKeys != null) {
            trustManager = new PublicKeyTrustManager(trustKeys);
            hostnameVerifier = SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;
        } else {
            trustManager = null;
            hostnameVerifier = SSLSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER;
        }
        javax.net.ssl.SSLSocketFactory sslSocketFactory = SslHelpers.buildSslSocketFactory(keyManager, trustManager);
        schemeSocketFactory = new SSLSocketFactory(sslSocketFactory, hostnameVerifier);
    } catch (GeneralSecurityException e) {
        throw new IllegalArgumentException("Error building SSL client", e);
    }
    SchemeRegistry schemeRegistry = new SchemeRegistry();
    schemeRegistry.register(new Scheme("https", port, schemeSocketFactory));
    PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(schemeRegistry);
    HttpClient httpClient = new DefaultHttpClient(connectionManager);
    httpClient = new DecompressingHttpClient(httpClient);
    return httpClient;
}

Example 3 with DecompressingHttpClient

use of org.apache.http.impl.client.DecompressingHttpClient in project nanohttpd by NanoHttpd.

the class GZipIntegrationTest method chunkedContentIsEncodedProperly.

@Test
public void chunkedContentIsEncodedProperly() throws IOException {
    InputStream data = new ByteArrayInputStream("This is a test".getBytes("UTF-8"));
    testServer.response = Response.newChunkedResponse(Status.OK, "text/plain", data);
    HttpGet request = new HttpGet("http://localhost:8192/");
    request.addHeader("Accept-encoding", "gzip");
    HttpResponse response = new DecompressingHttpClient(httpclient).execute(request);
    assertEquals("This is a test", EntityUtils.toString(response.getEntity()));
}

Example 4 with DecompressingHttpClient

use of org.apache.http.impl.client.DecompressingHttpClient in project ovirt-engine-sdk-java by oVirt.

the class ConnectionBuilder42 method createHttpClient.

/**
 * Creates HttpClient
 */
@Override
protected HttpClient createHttpClient() {
    int port = getPort();
    Credentials credentials = null;
    AuthSchemeRegistry schemeRegistry = new AuthSchemeRegistry();
    AuthScope authScope = new AuthScope(getHost(), port, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME);
    // Create credentials:
    if (user != null && user.length() > 0) {
        schemeRegistry.register(AuthPolicy.BASIC, new BasicSchemeFactory());
        credentials = new UsernamePasswordCredentials(user, password);
    } else if (kerberos) {
        schemeRegistry.register(AuthPolicy.SPNEGO, new SPNegoSchemeFactory(true));
        credentials = new Credentials() {

            @Override
            public Principal getUserPrincipal() {
                return null;
            }

            @Override
            public String getPassword() {
                return null;
            }
        };
    }
    // Create http client:
    DefaultHttpClient client = new DefaultHttpClient(new PoolingClientConnectionManager(createConnectionSocketFactoryRegistry()));
    client.setAuthSchemes(schemeRegistry);
    client.getCredentialsProvider().setCredentials(authScope, credentials);
    client.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.IGNORE_COOKIES);
    // Set request timeout:
    if (timeout != -1) {
        HttpConnectionParams.setSoTimeout(client.getParams(), timeout);
    }
    // Compress/decompress entities if compressing enabled:
    if (compress) {
        return new HttpClient42(new DecompressingHttpClient(client));
    }
    return new HttpClient42(client);
}

Example 5 with DecompressingHttpClient

use of org.apache.http.impl.client.DecompressingHttpClient in project nanohttpd by NanoHttpd.

the class GZipIntegrationTest method fixedLengthContentIsEncodedProperly.

@Test
public void fixedLengthContentIsEncodedProperly() throws IOException {
    testServer.response = Response.newFixedLengthResponse("This is a test");
    HttpGet request = new HttpGet("http://localhost:8192/");
    request.addHeader("Accept-encoding", "gzip");
    HttpResponse response = new DecompressingHttpClient(httpclient).execute(request);
    assertEquals("This is a test", EntityUtils.toString(response.getEntity()));
}