Examples with Header org.apache.commons.httpclient.Header used on opensource projects

Example 76 with Header

use of org.apache.commons.httpclient.Header in project cloudstack by apache.

the class S3TemplateDownloader method download.

@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
    if (!status.equals(Status.NOT_STARTED)) {
        // Only start downloading if we haven't started yet.
        LOGGER.debug("Template download is already started, not starting again. Template: " + downloadUrl);
        return 0;
    }
    int responseCode;
    if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) {
        errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    if (!HTTPUtils.verifyResponseCode(responseCode)) {
        errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    // Headers
    Header contentLengthHeader = getMethod.getResponseHeader("Content-Length");
    Header contentTypeHeader = getMethod.getResponseHeader("Content-Type");
    // Check the contentLengthHeader and transferEncodingHeader.
    if (contentLengthHeader == null) {
        errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied";
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    } else {
        // The ContentLengthHeader is supplied, parse it's value.
        remoteSize = Long.parseLong(contentLengthHeader.getValue());
    }
    if (remoteSize > maxTemplateSizeInByte) {
        errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    InputStream inputStream;
    try {
        inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream());
    } catch (IOException e) {
        errorString = "Exception occurred while opening InputStream for template " + downloadUrl;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes");
    // Time the upload starts.
    final Date start = new Date();
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(remoteSize);
    if (contentTypeHeader.getValue() != null) {
        objectMetadata.setContentType(contentTypeHeader.getValue());
    }
    // Create the PutObjectRequest.
    PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata);
    // If reduced redundancy is enabled, set it.
    if (s3TO.getEnableRRS()) {
        putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
    }
    Upload upload = S3Utils.putObject(s3TO, putObjectRequest);
    upload.addProgressListener(new ProgressListener() {

        @Override
        public void progressChanged(ProgressEvent progressEvent) {
            // Record the amount of bytes transferred.
            totalBytes += progressEvent.getBytesTransferred();
            LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
            if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
                status = Status.IN_PROGRESS;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
                status = Status.DOWNLOAD_FINISHED;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) {
                status = Status.ABORTED;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) {
                status = Status.UNRECOVERABLE_ERROR;
            }
        }
    });
    try {
        // Wait for the upload to complete.
        upload.waitForCompletion();
    } catch (InterruptedException e) {
        // Interruption while waiting for the upload to complete.
        LOGGER.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete");
    }
    downloadTime = new Date().getTime() - start.getTime();
    if (status == Status.DOWNLOAD_FINISHED) {
        LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
    } else {
        LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
    }
    // Close input stream
    getMethod.releaseConnection();
    // Call the callback!
    if (callback != null) {
        callback.downloadComplete(status);
    }
    return totalBytes;
}

Example 77 with Header

use of org.apache.commons.httpclient.Header in project cloudstack by apache.

the class HttpTemplateDownloader method download.

@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
    switch(status) {
        case ABORTED:
        case UNRECOVERABLE_ERROR:
        case DOWNLOAD_FINISHED:
            return 0;
        default:
    }
    int bytes = 0;
    File file = new File(toFile);
    try {
        long localFileSize = 0;
        if (file.exists() && resume) {
            localFileSize = file.length();
            s_logger.info("Resuming download to file (current size)=" + localFileSize);
        }
        Date start = new Date();
        int responseCode = 0;
        if (localFileSize > 0) {
            // require partial content support for resume
            request.addRequestHeader("Range", "bytes=" + localFileSize + "-");
            if (client.executeMethod(request) != HttpStatus.SC_PARTIAL_CONTENT) {
                errorString = "HTTP Server does not support partial get";
                status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
                return 0;
            }
        } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
            status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
            errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
            //FIXME: retry?
            return 0;
        }
        Header contentLengthHeader = request.getResponseHeader("Content-Length");
        boolean chunked = false;
        long remoteSize2 = 0;
        if (contentLengthHeader == null) {
            Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
            if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
                status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
                errorString = " Failed to receive length of download ";
                //FIXME: what status do we put here? Do we retry?
                return 0;
            } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
                chunked = true;
            }
        } else {
            remoteSize2 = Long.parseLong(contentLengthHeader.getValue());
            if (remoteSize2 == 0) {
                status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
                String downloaded = "(download complete remote=" + remoteSize + "bytes)";
                errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
                downloadTime = 0;
                return 0;
            }
        }
        if (remoteSize == 0) {
            remoteSize = remoteSize2;
        }
        if (remoteSize > maxTemplateSizeInBytes) {
            s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInBytes);
            status = Status.UNRECOVERABLE_ERROR;
            errorString = "Download file size is too large";
            return 0;
        }
        if (remoteSize == 0) {
            remoteSize = maxTemplateSizeInBytes;
        }
        InputStream in = request.getResponseBodyAsStream();
        RandomAccessFile out = new RandomAccessFile(file, "rw");
        out.seek(localFileSize);
        s_logger.info("Starting download from " + getDownloadUrl() + " to " + toFile + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInBytes);
        byte[] block = new byte[CHUNK_SIZE];
        long offset = 0;
        boolean done = false;
        boolean verifiedFormat = false;
        status = TemplateDownloader.Status.IN_PROGRESS;
        while (!done && status != Status.ABORTED && offset <= remoteSize) {
            if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) {
                out.write(block, 0, bytes);
                offset += bytes;
                out.seek(offset);
                totalBytes += bytes;
                if (!verifiedFormat && (offset >= 1048576 || offset >= remoteSize)) {
                    //let's check format after we get 1MB or full file
                    String uripath = null;
                    try {
                        URI str = new URI(getDownloadUrl());
                        uripath = str.getPath();
                    } catch (URISyntaxException e) {
                        s_logger.warn("Invalid download url: " + getDownloadUrl() + ", This should not happen since we have validated the url before!!");
                    }
                    String unsupportedFormat = ImageStoreUtil.checkTemplateFormat(file.getAbsolutePath(), uripath);
                    if (unsupportedFormat == null || !unsupportedFormat.isEmpty()) {
                        try {
                            request.abort();
                            out.close();
                            in.close();
                        } catch (Exception ex) {
                            s_logger.debug("Error on http connection : " + ex.getMessage());
                        }
                        status = Status.UNRECOVERABLE_ERROR;
                        errorString = "Template content is unsupported, or mismatch between selected format and template content. Found  : " + unsupportedFormat;
                        return 0;
                    }
                    s_logger.debug("Verified format of downloading file " + file.getAbsolutePath() + " is supported");
                    verifiedFormat = true;
                }
            } else {
                done = true;
            }
        }
        out.getFD().sync();
        Date finish = new Date();
        String downloaded = "(incomplete download)";
        if (totalBytes >= remoteSize) {
            status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
            downloaded = "(download complete remote=" + remoteSize + "bytes)";
        }
        errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
        downloadTime += finish.getTime() - start.getTime();
        in.close();
        out.close();
        return totalBytes;
    } catch (HttpException hte) {
        status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
        errorString = hte.getMessage();
    } catch (IOException ioe) {
        //probably a file write error?
        status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
        errorString = ioe.getMessage();
    } finally {
        if (status == Status.UNRECOVERABLE_ERROR && file.exists() && !file.isDirectory()) {
            file.delete();
        }
        request.releaseConnection();
        if (callback != null) {
            callback.downloadComplete(status);
        }
    }
    return 0;
}

Example 78 with Header

use of org.apache.commons.httpclient.Header in project sling by apache.

the class HttpTestBase method slingServerReady.

/** Return true if able to create and retrieve a node on server */
protected boolean slingServerReady() throws Exception {
    // create a node on the server
    final String time = String.valueOf(System.currentTimeMillis());
    final String url = HTTP_BASE_URL + "/WaitForSlingStartup/" + time;
    // add some properties to the node
    final Map<String, String> props = new HashMap<String, String>();
    props.put("time", time);
    // POST, get URL of created node and get content
    String urlOfNewNode = null;
    try {
        urlOfNewNode = testClient.createNode(url, props, null, true);
        final GetMethod get = new GetMethod(urlOfNewNode + readinessCheckExtension);
        final int status = httpClient.executeMethod(get);
        if (status != 200) {
            throw new HttpStatusCodeException(200, status, "GET", urlOfNewNode);
        }
        final Header h = get.getResponseHeader("Content-Type");
        final String contentType = h == null ? "" : h.getValue();
        if (!contentType.startsWith(readinessCheckContentTypePrefix)) {
            throw new IOException("Expected Content-Type=" + readinessCheckContentTypePrefix + " but got '" + contentType + "' for URL=" + urlOfNewNode);
        }
        final String content = get.getResponseBodyAsString();
        if (!content.contains(time)) {
            throw new IOException("Content does not contain '" + time + "' (" + content + ") at URL=" + urlOfNewNode);
        }
    } finally {
        if (urlOfNewNode != null) {
            try {
                testClient.delete(urlOfNewNode);
            } catch (Exception ignore) {
            }
        }
    }
    // Also check that the WebDAV root is ready
    {
        // need the trailing slash in case the base URL is the context root
        final String webDavUrl = WEBDAV_BASE_URL + "/";
        final HttpAnyMethod options = new HttpAnyMethod("OPTIONS", webDavUrl);
        final int status = httpClient.executeMethod(options);
        if (status != 200) {
            throw new HttpStatusCodeException(200, status, "OPTIONS", webDavUrl);
        }
        // The Allow header tells us that we're talking to a WebDAV server
        final Header h = options.getResponseHeader("Allow");
        if (h == null) {
            throw new IOException("Response does not contain Allow header, at URL=" + webDavUrl);
        } else if (h.getValue() == null) {
            throw new IOException("Allow header has null value at URL=" + webDavUrl);
        } else if (!h.getValue().contains("PROPFIND")) {
            throw new IOException("Allow header (" + h.getValue() + " does not contain PROPFIND, at URL=" + webDavUrl);
        }
    }
    // And check optional additional URLs for readyness
    // Defined by system properties like
    //  launchpad.ready.1 = GET:/tmp/someUrl:200:expectedRegexpInResponse
    {
        for (int i = 0; i <= MAX_READY_URL_INDEX; i++) {
            final String propName = READY_URL_PROP_PREFIX + i;
            final String readyDef = System.getProperty(propName, "");
            final String[] parts = readyDef.split(":");
            if (parts.length == 4) {
                final String info = propName + "=" + readyDef;
                final HttpAnyMethod m = new HttpAnyMethod(parts[0], HTTP_BASE_URL + parts[1]);
                final int expectedStatus = Integer.valueOf(parts[2]);
                final int status = httpClient.executeMethod(m);
                if (expectedStatus != status) {
                    throw new IOException("Status " + status + " does not match expected value: " + info);
                }
                final String content = m.getResponseBodyAsString();
                final Pattern p = Pattern.compile("(?s).*" + parts[3] + ".*");
                if (!p.matcher(content).matches()) {
                    throw new IOException("Content does not match expected regexp:" + info + ", content=" + content);
                }
            }
        }
    }
    return true;
}

Example 79 with Header

use of org.apache.commons.httpclient.Header in project sling by apache.

the class SelectorRedirectOnLoginErrorTest method testRedirectToSelectorLoginFormAfterLoginError.

/**
     * Test SLING-2165.  Login Error should redirect back to the referrer
     * login page.
     *
     * @throws Exception
     */
public void testRedirectToSelectorLoginFormAfterLoginError() throws Exception {
    //login failure
    List<NameValuePair> params = new ArrayList<NameValuePair>();
    params.add(new NameValuePair("j_username", "___bogus___"));
    params.add(new NameValuePair("j_password", "not_a_real_user"));
    final String loginPageUrl = String.format("%s/system/sling/selector/login", HTTP_BASE_URL);
    PostMethod post = (PostMethod) assertPostStatus(HTTP_BASE_URL + "/j_security_check", HttpServletResponse.SC_MOVED_TEMPORARILY, params, null, loginPageUrl);
    final Header locationHeader = post.getResponseHeader("Location");
    String location = locationHeader.getValue();
    int queryStrStart = location.indexOf('?');
    if (queryStrStart != -1) {
        location = location.substring(0, queryStrStart);
    }
    assertEquals("Expected to remain on the selector/login page", loginPageUrl, location);
}

Example 80 with Header

use of org.apache.commons.httpclient.Header in project sling by apache.

the class SelectorRedirectOnLoginErrorTest method testRedirectToOpenIDLoginFormAfterLoginError.

/**
     * Test SLING-2165.  Login Error should redirect back to the referrer
     * login page.
     *
     * @throws Exception
     */
public void testRedirectToOpenIDLoginFormAfterLoginError() throws Exception {
    //login failure
    List<NameValuePair> params = new ArrayList<NameValuePair>();
    params.add(new NameValuePair("openid_identifier", "___bogus___"));
    final String loginPageUrl = String.format("%s/system/sling/openid/login", HTTP_BASE_URL);
    PostMethod post = (PostMethod) assertPostStatus(HTTP_BASE_URL + "/j_security_check", HttpServletResponse.SC_MOVED_TEMPORARILY, params, null, loginPageUrl);
    final Header locationHeader = post.getResponseHeader("Location");
    String location = locationHeader.getValue();
    int queryStrStart = location.indexOf('?');
    if (queryStrStart != -1) {
        location = location.substring(0, queryStrStart);
    }
    assertEquals("Expected to remain on the openid/login page", loginPageUrl, location);
}