Examples with HttpGet org.apache.http.client.methods.HttpGet used on opensource projects

Example 31 with HttpGet

use of org.apache.http.client.methods.HttpGet in project frontend-maven-plugin by eirslett.

the class DefaultFileDownloader method execute.

private CloseableHttpResponse execute(String requestUrl, String userName, String password) throws IOException {
    CloseableHttpResponse response;
    Proxy proxy = proxyConfig.getProxyForUrl(requestUrl);
    if (proxy != null) {
        LOGGER.info("Downloading via proxy " + proxy.toString());
        return executeViaProxy(proxy, requestUrl);
    } else {
        LOGGER.info("No proxy was configured, downloading directly");
        if (StringUtils.isNotEmpty(userName) && StringUtils.isNotEmpty(password)) {
            LOGGER.info("Using credentials (" + userName + ") from settings.xml");
            // Auth target host
            URL aURL = new URL(requestUrl);
            HttpClientContext localContext = makeLocalContext(aURL);
            CredentialsProvider credentialsProvider = makeCredentialsProvider(aURL.getHost(), aURL.getPort(), userName, password);
            response = buildHttpClient(credentialsProvider).execute(new HttpGet(requestUrl), localContext);
        } else {
            response = buildHttpClient(null).execute(new HttpGet(requestUrl));
        }
    }
    return response;
}

Example 32 with HttpGet

use of org.apache.http.client.methods.HttpGet in project OpenRefine by OpenRefine.

the class ImportingUtilities method retrieveContentFromPostRequest.

public static void retrieveContentFromPostRequest(HttpServletRequest request, Properties parameters, File rawDataDir, JSONObject retrievalRecord, final Progress progress) throws Exception {
    JSONArray fileRecords = new JSONArray();
    JSONUtilities.safePut(retrievalRecord, "files", fileRecords);
    int clipboardCount = 0;
    int uploadCount = 0;
    int downloadCount = 0;
    int archiveCount = 0;
    // This tracks the total progress, which involves uploading data from the client
    // as well as downloading data from URLs.
    final SavingUpdate update = new SavingUpdate() {

        @Override
        public void savedMore() {
            progress.setProgress(null, calculateProgressPercent(totalExpectedSize, totalRetrievedSize));
        }

        @Override
        public boolean isCanceled() {
            return progress.isCanceled();
        }
    };
    DiskFileItemFactory fileItemFactory = new DiskFileItemFactory();
    ServletFileUpload upload = new ServletFileUpload(fileItemFactory);
    upload.setProgressListener(new ProgressListener() {

        boolean setContentLength = false;

        long lastBytesRead = 0;

        @Override
        public void update(long bytesRead, long contentLength, int itemCount) {
            if (!setContentLength) {
                // Only try to set the content length if we really know it.
                if (contentLength >= 0) {
                    update.totalExpectedSize += contentLength;
                    setContentLength = true;
                }
            }
            if (setContentLength) {
                update.totalRetrievedSize += (bytesRead - lastBytesRead);
                lastBytesRead = bytesRead;
                update.savedMore();
            }
        }
    });
    @SuppressWarnings("unchecked") List<FileItem> tempFiles = (List<FileItem>) upload.parseRequest(request);
    progress.setProgress("Uploading data ...", -1);
    parts: for (FileItem fileItem : tempFiles) {
        if (progress.isCanceled()) {
            break;
        }
        InputStream stream = fileItem.getInputStream();
        String name = fileItem.getFieldName().toLowerCase();
        if (fileItem.isFormField()) {
            if (name.equals("clipboard")) {
                String encoding = request.getCharacterEncoding();
                if (encoding == null) {
                    encoding = "UTF-8";
                }
                File file = allocateFile(rawDataDir, "clipboard.txt");
                JSONObject fileRecord = new JSONObject();
                JSONUtilities.safePut(fileRecord, "origin", "clipboard");
                JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding);
                JSONUtilities.safePut(fileRecord, "declaredMimeType", (String) null);
                JSONUtilities.safePut(fileRecord, "format", "text");
                JSONUtilities.safePut(fileRecord, "fileName", "(clipboard)");
                JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
                progress.setProgress("Uploading pasted clipboard text", calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
                JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
                JSONUtilities.append(fileRecords, fileRecord);
                clipboardCount++;
            } else if (name.equals("download")) {
                String urlString = Streams.asString(stream);
                URL url = new URL(urlString);
                JSONObject fileRecord = new JSONObject();
                JSONUtilities.safePut(fileRecord, "origin", "download");
                JSONUtilities.safePut(fileRecord, "url", urlString);
                for (UrlRewriter rewriter : ImportingManager.urlRewriters) {
                    Result result = rewriter.rewrite(urlString);
                    if (result != null) {
                        urlString = result.rewrittenUrl;
                        url = new URL(urlString);
                        JSONUtilities.safePut(fileRecord, "url", urlString);
                        JSONUtilities.safePut(fileRecord, "format", result.format);
                        if (!result.download) {
                            downloadCount++;
                            JSONUtilities.append(fileRecords, fileRecord);
                            continue parts;
                        }
                    }
                }
                if ("http".equals(url.getProtocol()) || "https".equals(url.getProtocol())) {
                    DefaultHttpClient client = new DefaultHttpClient();
                    DecompressingHttpClient httpclient = new DecompressingHttpClient(client);
                    HttpGet httpGet = new HttpGet(url.toURI());
                    httpGet.setHeader("User-Agent", RefineServlet.getUserAgent());
                    if ("https".equals(url.getProtocol())) {
                        // HTTPS only - no sending password in the clear over HTTP
                        String userinfo = url.getUserInfo();
                        if (userinfo != null) {
                            int s = userinfo.indexOf(':');
                            if (s > 0) {
                                String user = userinfo.substring(0, s);
                                String pw = userinfo.substring(s + 1, userinfo.length());
                                client.getCredentialsProvider().setCredentials(new AuthScope(url.getHost(), 443), new UsernamePasswordCredentials(user, pw));
                            }
                        }
                    }
                    HttpResponse response = httpclient.execute(httpGet);
                    try {
                        response.getStatusLine();
                        HttpEntity entity = response.getEntity();
                        if (entity == null) {
                            throw new Exception("No content found in " + url.toString());
                        }
                        InputStream stream2 = entity.getContent();
                        String encoding = null;
                        if (entity.getContentEncoding() != null) {
                            encoding = entity.getContentEncoding().getValue();
                        }
                        JSONUtilities.safePut(fileRecord, "declaredEncoding", encoding);
                        String contentType = null;
                        if (entity.getContentType() != null) {
                            contentType = entity.getContentType().getValue();
                        }
                        JSONUtilities.safePut(fileRecord, "declaredMimeType", contentType);
                        if (saveStream(stream2, url, rawDataDir, progress, update, fileRecord, fileRecords, entity.getContentLength())) {
                            archiveCount++;
                        }
                        downloadCount++;
                        EntityUtils.consume(entity);
                    } finally {
                        httpGet.releaseConnection();
                    }
                } else {
                    // Fallback handling for non HTTP connections (only FTP?)
                    URLConnection urlConnection = url.openConnection();
                    urlConnection.setConnectTimeout(5000);
                    urlConnection.connect();
                    InputStream stream2 = urlConnection.getInputStream();
                    JSONUtilities.safePut(fileRecord, "declaredEncoding", urlConnection.getContentEncoding());
                    JSONUtilities.safePut(fileRecord, "declaredMimeType", urlConnection.getContentType());
                    try {
                        if (saveStream(stream2, url, rawDataDir, progress, update, fileRecord, fileRecords, urlConnection.getContentLength())) {
                            archiveCount++;
                        }
                        downloadCount++;
                    } finally {
                        stream2.close();
                    }
                }
            } else {
                String value = Streams.asString(stream);
                parameters.put(name, value);
            // TODO: We really want to store this on the request so it's available for everyone
            //                    request.getParameterMap().put(name, value);
            }
        } else {
            // is file content
            String fileName = fileItem.getName();
            if (fileName.length() > 0) {
                long fileSize = fileItem.getSize();
                File file = allocateFile(rawDataDir, fileName);
                JSONObject fileRecord = new JSONObject();
                JSONUtilities.safePut(fileRecord, "origin", "upload");
                JSONUtilities.safePut(fileRecord, "declaredEncoding", request.getCharacterEncoding());
                JSONUtilities.safePut(fileRecord, "declaredMimeType", fileItem.getContentType());
                JSONUtilities.safePut(fileRecord, "fileName", fileName);
                JSONUtilities.safePut(fileRecord, "location", getRelativePath(file, rawDataDir));
                progress.setProgress("Saving file " + fileName + " locally (" + formatBytes(fileSize) + " bytes)", calculateProgressPercent(update.totalExpectedSize, update.totalRetrievedSize));
                JSONUtilities.safePut(fileRecord, "size", saveStreamToFile(stream, file, null));
                if (postProcessRetrievedFile(rawDataDir, file, fileRecord, fileRecords, progress)) {
                    archiveCount++;
                }
                uploadCount++;
            }
        }
        stream.close();
    }
    // Delete all temp files.
    for (FileItem fileItem : tempFiles) {
        fileItem.delete();
    }
    JSONUtilities.safePut(retrievalRecord, "uploadCount", uploadCount);
    JSONUtilities.safePut(retrievalRecord, "downloadCount", downloadCount);
    JSONUtilities.safePut(retrievalRecord, "clipboardCount", clipboardCount);
    JSONUtilities.safePut(retrievalRecord, "archiveCount", archiveCount);
}

Example 33 with HttpGet

use of org.apache.http.client.methods.HttpGet in project android_frameworks_base by ParanoidAndroid.

the class AbstractProxyTest method testExplicitNoProxyCancelsSystemProperty.

public void testExplicitNoProxyCancelsSystemProperty() throws Exception {
    server.enqueue(new MockResponse().setBody("Via the origin server!"));
    server.play();
    System.setProperty("http.proxyHost", "proxy.foo");
    System.setProperty("http.proxyPort", "8080");
    HttpClient client = newHttpClient();
    HttpGet request = new HttpGet(server.getUrl("/bar").toURI());
    request.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, ConnRouteParams.NO_HOST);
    HttpResponse response = client.execute(request);
    assertEquals("Via the origin server!", contentToString(response));
    RecordedRequest recordedRequest = server.takeRequest();
    assertEquals("GET /bar HTTP/1.1", recordedRequest.getRequestLine());
}

Example 34 with HttpGet

use of org.apache.http.client.methods.HttpGet in project android_frameworks_base by ParanoidAndroid.

the class AbstractProxyTest method testConnectToHttps.

public void testConnectToHttps() throws Exception {
    TestSSLContext testSSLContext = TestSSLContext.create();
    server.useHttps(testSSLContext.serverContext.getSocketFactory(), false);
    server.enqueue(new MockResponse().setResponseCode(200).setBody("this response comes via HTTPS"));
    server.play();
    HttpClient httpClient = newHttpClient();
    SSLSocketFactory sslSocketFactory = newSslSocketFactory(testSSLContext);
    sslSocketFactory.setHostnameVerifier(new AllowAllHostnameVerifier());
    httpClient.getConnectionManager().getSchemeRegistry().register(new Scheme("https", sslSocketFactory, server.getPort()));
    HttpResponse response = httpClient.execute(new HttpGet("https://localhost:" + server.getPort() + "/foo"));
    assertEquals("this response comes via HTTPS", contentToString(response));
    RecordedRequest request = server.takeRequest();
    assertEquals("GET /foo HTTP/1.1", request.getRequestLine());
}

Example 35 with HttpGet

use of org.apache.http.client.methods.HttpGet in project android_frameworks_base by ParanoidAndroid.

the class AbstractProxyTest method testConnectViaProxy.

/**
     * http://code.google.com/p/android/issues/detail?id=2690
     */
private void testConnectViaProxy(ProxyConfig proxyConfig) throws Exception {
    MockResponse mockResponse = new MockResponse().setResponseCode(200).setBody("this response comes via a proxy");
    server.enqueue(mockResponse);
    server.play();
    HttpClient httpProxyClient = newHttpClient();
    HttpGet request = new HttpGet("http://android.com/foo");
    proxyConfig.configure(server, httpProxyClient, request);
    HttpResponse response = httpProxyClient.execute(request);
    assertEquals("this response comes via a proxy", contentToString(response));
    RecordedRequest get = server.takeRequest();
    assertEquals("GET http://android.com/foo HTTP/1.1", get.getRequestLine());
    assertContains(get.getHeaders(), "Host: android.com");
}