Examples with Download com.amazonaws.services.s3.transfer.Download used on opensource projects

Example 31 with Download

use of com.amazonaws.services.s3.transfer.Download in project ice by Netflix.

the class AwsUtils method downloadFileIfChangedSince.

public static boolean downloadFileIfChangedSince(String bucketName, String bucketFileRegion, String bucketFilePrefix, File file, long milles, String accountId, String assumeRole, String externalId) {
    AmazonS3Client s3Client = AwsUtils.s3Client;
    try {
        if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
            Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
            s3Client = new AmazonS3Client(new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey(), assumedCredentials.getSessionToken()), clientConfig);
        }
        if (bucketFileRegion != null && !bucketFileRegion.isEmpty()) {
            s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com");
        }
        ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName());
        boolean download = !file.exists() || metadata.getLastModified().getTime() > milles;
        if (download) {
            return download(s3Client, bucketName, bucketFilePrefix + file.getName(), file);
        } else
            return download;
    } finally {
        if (s3Client != AwsUtils.s3Client)
            s3Client.shutdown();
    }
}

Example 32 with Download

use of com.amazonaws.services.s3.transfer.Download in project cloudstack by apache.

the class S3TemplateDownloader method download.

@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
    if (!status.equals(Status.NOT_STARTED)) {
        // Only start downloading if we haven't started yet.
        LOGGER.debug("Template download is already started, not starting again. Template: " + downloadUrl);
        return 0;
    }
    int responseCode;
    if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) {
        errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    if (!HTTPUtils.verifyResponseCode(responseCode)) {
        errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    // Headers
    Header contentLengthHeader = getMethod.getResponseHeader("Content-Length");
    Header contentTypeHeader = getMethod.getResponseHeader("Content-Type");
    // Check the contentLengthHeader and transferEncodingHeader.
    if (contentLengthHeader == null) {
        errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied";
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    } else {
        // The ContentLengthHeader is supplied, parse it's value.
        remoteSize = Long.parseLong(contentLengthHeader.getValue());
    }
    if (remoteSize > maxTemplateSizeInByte) {
        errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    InputStream inputStream;
    try {
        inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream());
    } catch (IOException e) {
        errorString = "Exception occurred while opening InputStream for template " + downloadUrl;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + toHumanReadableSize(remoteSize) + " bytes");
    // Time the upload starts.
    final Date start = new Date();
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(remoteSize);
    if (contentTypeHeader.getValue() != null) {
        objectMetadata.setContentType(contentTypeHeader.getValue());
    }
    // Create the PutObjectRequest.
    PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata);
    // If reduced redundancy is enabled, set it.
    if (s3TO.isEnableRRS()) {
        putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
    }
    Upload upload = S3Utils.putObject(s3TO, putObjectRequest);
    upload.addProgressListener(new ProgressListener() {

        @Override
        public void progressChanged(ProgressEvent progressEvent) {
            // Record the amount of bytes transferred.
            totalBytes += progressEvent.getBytesTransferred();
            LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + toHumanReadableSize(totalBytes) + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
            if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
                status = Status.IN_PROGRESS;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
                status = Status.DOWNLOAD_FINISHED;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) {
                status = Status.ABORTED;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) {
                status = Status.UNRECOVERABLE_ERROR;
            }
        }
    });
    try {
        // Wait for the upload to complete.
        upload.waitForCompletion();
    } catch (InterruptedException e) {
        // Interruption while waiting for the upload to complete.
        LOGGER.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete");
    }
    downloadTime = new Date().getTime() - start.getTime();
    if (status == Status.DOWNLOAD_FINISHED) {
        LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
    } else {
        LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + toHumanReadableSize(totalBytes) + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
    }
    // Close input stream
    getMethod.releaseConnection();
    // Call the callback!
    if (callback != null) {
        callback.downloadComplete(status);
    }
    return totalBytes;
}

Example 33 with Download

use of com.amazonaws.services.s3.transfer.Download in project carina by qaprosoft.

the class CarinaListener method updateS3AppPath.

/**
 * Method to update MOBILE_APP path in case if apk is located in s3 bucket.
 */
private static void updateS3AppPath() {
    Pattern S3_BUCKET_PATTERN = Pattern.compile("s3:\\/\\/([a-zA-Z-0-9][^\\/]*)\\/(.*)");
    // get app path to be sure that we need(do not need) to download app
    // from s3 bucket
    String mobileAppPath = Configuration.getMobileApp();
    Matcher matcher = S3_BUCKET_PATTERN.matcher(mobileAppPath);
    LOGGER.info("Analyzing if mobile app is located on S3...");
    if (matcher.find()) {
        LOGGER.info("app artifact is located on s3...");
        String bucketName = matcher.group(1);
        String key = matcher.group(2);
        Pattern pattern = Pattern.compile(key);
        // analyze if we have any pattern inside mobile_app to make extra
        // search in AWS
        int position = key.indexOf(".*");
        if (position > 0) {
            // /android/develop/dfgdfg.*/Mapmyrun.apk
            int slashPosition = key.substring(0, position).lastIndexOf("/");
            if (slashPosition > 0) {
                key = key.substring(0, slashPosition);
                S3ObjectSummary lastBuild = AmazonS3Manager.getInstance().getLatestBuildArtifact(bucketName, key, pattern);
                key = lastBuild.getKey();
            }
        } else {
            key = AmazonS3Manager.getInstance().get(bucketName, key).getKey();
        }
        LOGGER.info("next s3 app key will be used: " + key);
        // generate presign url explicitly to register link as run artifact
        // generate presigned url for nearest 3 days
        long hours = 72L * 1000 * 60 * 60;
        String presignedAppUrl = AmazonS3Manager.getInstance().generatePreSignUrl(bucketName, key, hours).toString();
        Configuration.setMobileApp(presignedAppUrl);
    }
}

Example 34 with Download

use of com.amazonaws.services.s3.transfer.Download in project carina by qaprosoft.

the class AmazonS3Manager method download.

/**
 * Method to download file from s3 to local file system
 *
 * @param bucketName AWS S3 bucket name
 * @param key (example: android/apkFolder/ApkName.apk)
 * @param file (local file name)
 * @param pollingInterval (polling interval in sec for S3 download status determination)
 */
public void download(final String bucketName, final String key, final File file, long pollingInterval) {
    LOGGER.info("App will be downloaded from s3.");
    LOGGER.info(String.format("[Bucket name: %s] [Key: %s] [File: %s]", bucketName, key, file.getAbsolutePath()));
    DefaultAWSCredentialsProviderChain credentialProviderChain = new DefaultAWSCredentialsProviderChain();
    TransferManager tx = new TransferManager(credentialProviderChain.getCredentials());
    Download appDownload = tx.download(bucketName, key, file);
    try {
        LOGGER.info("Transfer: " + appDownload.getDescription());
        LOGGER.info("	State: " + appDownload.getState());
        LOGGER.info("	Progress: ");
        // You can poll your transfer's status to check its progress
        while (!appDownload.isDone()) {
            LOGGER.info("		transferred: " + (int) (appDownload.getProgress().getPercentTransferred() + 0.5) + "%");
            CommonUtils.pause(pollingInterval);
        }
        LOGGER.info("	State: " + appDownload.getState());
    // appDownload.waitForCompletion();
    } catch (AmazonClientException e) {
        throw new RuntimeException("File wasn't downloaded from s3. See log: ".concat(e.getMessage()));
    }
// tx.shutdownNow();
}

Example 35 with Download

use of com.amazonaws.services.s3.transfer.Download in project carina by qaprosoft.

the class AmazonS3Manager method get.

/**
 * Get any file from Amazon S3 storage as S3Object.
 *
 * @param bucket
 *            - S3 Bucket name.
 * @param key
 *            - S3 storage path. Example:
 *            DEMO/TestSuiteName/TestMethodName/file.txt
 * @return S3Object
 */
public S3Object get(String bucket, String key) {
    if (bucket == null) {
        throw new RuntimeException("Bucket is null!");
    }
    if (bucket.isEmpty()) {
        throw new RuntimeException("Bucket is empty!");
    }
    if (key == null) {
        throw new RuntimeException("Key is null!");
    }
    if (key.isEmpty()) {
        throw new RuntimeException("Key is empty!");
    }
    try {
        LOGGER.info("Finding an s3object...");
        // TODO investigate possibility to add percentage of completed
        // downloading
        S3Object s3object = s3client.getObject(new GetObjectRequest(bucket, key));
        LOGGER.info("Content-Type: " + s3object.getObjectMetadata().getContentType());
        return s3object;
    /*
             * GetObjectRequest rangeObjectRequest = new GetObjectRequest(
             * bucketName, key); rangeObjectRequest.setRange(0, 10); S3Object
             * objectPortion = s3client.getObject(rangeObjectRequest);
             */
    } catch (AmazonServiceException ase) {
        LOGGER.error("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response for some reason.\n" + "Error Message:    " + ase.getMessage() + "\n" + "HTTP Status Code: " + ase.getStatusCode() + "\n" + "AWS Error Code:   " + ase.getErrorCode() + "\n" + "Error Type:       " + ase.getErrorType() + "\n" + "Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        LOGGER.error("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network.\n" + "Error Message: " + ace.getMessage());
    }
    // TODO investigate pros and cons returning null
    throw new RuntimeException("Unable to download '" + key + "' from Amazon S3 bucket '" + bucket + "'");
}