Examples with Upload com.amazonaws.services.s3.transfer.Upload used on opensource projects

Example 21 with Upload

use of com.amazonaws.services.s3.transfer.Upload in project hadoop by apache.

the class S3AFileSystem method innerCopyFromLocalFile.

/**
   * The src file is on the local disk.  Add it to FS at
   * the given dst name.
   *
   * This version doesn't need to create a temporary file to calculate the md5.
   * Sadly this doesn't seem to be used by the shell cp :(
   *
   * delSrc indicates if the source should be removed
   * @param delSrc whether to delete the src
   * @param overwrite whether to overwrite an existing file
   * @param src path
   * @param dst path
   * @throws IOException IO problem
   * @throws FileAlreadyExistsException the destination file exists and
   * overwrite==false
   * @throws AmazonClientException failure in the AWS SDK
   */
private void innerCopyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException, FileAlreadyExistsException, AmazonClientException {
    incrementStatistic(INVOCATION_COPY_FROM_LOCAL_FILE);
    final String key = pathToKey(dst);
    if (!overwrite && exists(dst)) {
        throw new FileAlreadyExistsException(dst + " already exists");
    }
    LOG.debug("Copying local file from {} to {}", src, dst);
    // Since we have a local file, we don't need to stream into a temporary file
    LocalFileSystem local = getLocal(getConf());
    File srcfile = local.pathToFile(src);
    final ObjectMetadata om = newObjectMetadata(srcfile.length());
    PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, srcfile);
    Upload up = putObject(putObjectRequest);
    ProgressableProgressListener listener = new ProgressableProgressListener(this, key, up, null);
    up.addProgressListener(listener);
    try {
        up.waitForUploadResult();
    } catch (InterruptedException e) {
        throw new InterruptedIOException("Interrupted copying " + src + " to " + dst + ", cancelling");
    }
    listener.uploadCompleted();
    // This will delete unnecessary fake parent directories
    finishedWrite(key);
    if (delSrc) {
        local.delete(src, false);
    }
}

Example 22 with Upload

use of com.amazonaws.services.s3.transfer.Upload in project hadoop by apache.

the class S3AFileSystem method putObject.

/**
   * Start a transfer-manager managed async PUT of an object,
   * incrementing the put requests and put bytes
   * counters.
   * It does not update the other counters,
   * as existing code does that as progress callbacks come in.
   * Byte length is calculated from the file length, or, if there is no
   * file, from the content length of the header.
   * Because the operation is async, any stream supplied in the request
   * must reference data (files, buffers) which stay valid until the upload
   * completes.
   * @param putObjectRequest the request
   * @return the upload initiated
   */
public Upload putObject(PutObjectRequest putObjectRequest) {
    long len;
    if (putObjectRequest.getFile() != null) {
        len = putObjectRequest.getFile().length();
    } else {
        len = putObjectRequest.getMetadata().getContentLength();
    }
    incrementPutStartStatistics(len);
    try {
        Upload upload = transfers.upload(putObjectRequest);
        incrementPutCompletedStatistics(true, len);
        return upload;
    } catch (AmazonClientException e) {
        incrementPutCompletedStatistics(false, len);
        throw e;
    }
}

Example 23 with Upload

use of com.amazonaws.services.s3.transfer.Upload in project hadoop by apache.

the class S3AOutputStream method close.

@Override
public void close() throws IOException {
    if (closed.getAndSet(true)) {
        return;
    }
    backupStream.close();
    LOG.debug("OutputStream for key '{}' closed. Now beginning upload", key);
    try {
        final ObjectMetadata om = fs.newObjectMetadata(backupFile.length());
        Upload upload = fs.putObject(fs.newPutObjectRequest(key, om, backupFile));
        ProgressableProgressListener listener = new ProgressableProgressListener(fs, key, upload, progress);
        upload.addProgressListener(listener);
        upload.waitForUploadResult();
        listener.uploadCompleted();
        // This will delete unnecessary fake parent directories
        fs.finishedWrite(key);
    } catch (InterruptedException e) {
        throw (InterruptedIOException) new InterruptedIOException(e.toString()).initCause(e);
    } catch (AmazonClientException e) {
        throw translateException("saving output", key, e);
    } finally {
        if (!backupFile.delete()) {
            LOG.warn("Could not delete temporary s3a file: {}", backupFile);
        }
        super.close();
    }
    LOG.debug("OutputStream for key '{}' upload complete", key);
}

Example 24 with Upload

use of com.amazonaws.services.s3.transfer.Upload in project deeplearning4j by deeplearning4j.

the class S3Uploader method upload.

/**
     * Upload the file to the bucket.
     * Will create the bucket if it hasn't already been created
     * @param file the file to upload
     * @param bucketName the name of the bucket
     */
public void upload(File file, String bucketName) {
    AmazonS3 client = new AmazonS3Client(creds);
    bucketName = ensureValidBucketName(bucketName);
    List<Bucket> buckets = client.listBuckets();
    for (Bucket b : buckets) if (b.getName().equals(bucketName)) {
        client.putObject(bucketName, file.getName(), file);
        return;
    }
    //bucket didn't exist: create it
    client.createBucket(bucketName);
    client.putObject(bucketName, file.getName(), file);
}

Example 25 with Upload

use of com.amazonaws.services.s3.transfer.Upload in project cloudstack by apache.

the class S3TemplateDownloader method download.

@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
    if (!status.equals(Status.NOT_STARTED)) {
        // Only start downloading if we haven't started yet.
        LOGGER.debug("Template download is already started, not starting again. Template: " + downloadUrl);
        return 0;
    }
    int responseCode;
    if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) {
        errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    if (!HTTPUtils.verifyResponseCode(responseCode)) {
        errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    // Headers
    Header contentLengthHeader = getMethod.getResponseHeader("Content-Length");
    Header contentTypeHeader = getMethod.getResponseHeader("Content-Type");
    // Check the contentLengthHeader and transferEncodingHeader.
    if (contentLengthHeader == null) {
        errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied";
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    } else {
        // The ContentLengthHeader is supplied, parse it's value.
        remoteSize = Long.parseLong(contentLengthHeader.getValue());
    }
    if (remoteSize > maxTemplateSizeInByte) {
        errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    InputStream inputStream;
    try {
        inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream());
    } catch (IOException e) {
        errorString = "Exception occurred while opening InputStream for template " + downloadUrl;
        LOGGER.warn(errorString);
        status = Status.UNRECOVERABLE_ERROR;
        return 0;
    }
    LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes");
    // Time the upload starts.
    final Date start = new Date();
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(remoteSize);
    if (contentTypeHeader.getValue() != null) {
        objectMetadata.setContentType(contentTypeHeader.getValue());
    }
    // Create the PutObjectRequest.
    PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata);
    // If reduced redundancy is enabled, set it.
    if (s3TO.getEnableRRS()) {
        putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
    }
    Upload upload = S3Utils.putObject(s3TO, putObjectRequest);
    upload.addProgressListener(new ProgressListener() {

        @Override
        public void progressChanged(ProgressEvent progressEvent) {
            // Record the amount of bytes transferred.
            totalBytes += progressEvent.getBytesTransferred();
            LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
            if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
                status = Status.IN_PROGRESS;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
                status = Status.DOWNLOAD_FINISHED;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) {
                status = Status.ABORTED;
            } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) {
                status = Status.UNRECOVERABLE_ERROR;
            }
        }
    });
    try {
        // Wait for the upload to complete.
        upload.waitForCompletion();
    } catch (InterruptedException e) {
        // Interruption while waiting for the upload to complete.
        LOGGER.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete");
    }
    downloadTime = new Date().getTime() - start.getTime();
    if (status == Status.DOWNLOAD_FINISHED) {
        LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
    } else {
        LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred  " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
    }
    // Close input stream
    getMethod.releaseConnection();
    // Call the callback!
    if (callback != null) {
        callback.downloadComplete(status);
    }
    return totalBytes;
}