Examples with CompleteMultipartUploadRequest com.amazonaws.services.s3.model.CompleteMultipartUploadRequest used on opensource projects

Example 1 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project hippo by NHS-digital-website.

the class S3ConnectorImpl method uploadFile.

public S3ObjectMetadata uploadFile(InputStream fileStream, String fileName, String contentType) {
    String objectKey = s3ObjectKeyGenerator.generateObjectKey(fileName);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentType(contentType);
    // initialise multipart upload
    InitiateMultipartUploadResult initResult = s3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, objectKey, metadata));
    // loop parts
    List<PartETag> partETags;
    try {
        partETags = uploadParts(fileStream, bucketName, objectKey, initResult.getUploadId());
    } catch (Exception ex) {
        final String errorMessage = "Failed to upload file " + objectKey;
        log.error(errorMessage, ex);
        s3.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, objectKey, initResult.getUploadId()));
        throw new RuntimeException(errorMessage, ex);
    }
    // finalise multipart upload
    s3.completeMultipartUpload(new CompleteMultipartUploadRequest(bucketName, objectKey, initResult.getUploadId(), partETags));
    // The above put request returns metadata object but it's empty,
    // hence the need for a separate call to fetch actual metadata.
    ObjectMetadata resultMetadata = s3.getObjectMetadata(bucketName, objectKey);
    return new S3ObjectMetadataImpl(resultMetadata, bucketName, objectKey);
}

Example 2 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project bender by Nextdoor.

the class S3TransportFactory method close.

@Override
public void close() {
    Exception e = null;
    for (MultiPartUpload upload : this.pendingMultiPartUploads.values()) {
        if (e == null) {
            CompleteMultipartUploadRequest req = upload.getCompleteMultipartUploadRequest();
            try {
                this.client.completeMultipartUpload(req);
            } catch (AmazonS3Exception e1) {
                logger.error("failed to complete multi-part upload for " + upload.getKey() + " parts " + upload.getPartCount(), e1);
                e = e1;
            }
        } else {
            logger.warn("aborting upload for: " + upload.getKey());
            AbortMultipartUploadRequest req = upload.getAbortMultipartUploadRequest();
            try {
                this.client.abortMultipartUpload(req);
            } catch (AmazonS3Exception e1) {
                logger.error("failed to abort multi-part upload", e1);
            }
        }
    }
    this.pendingMultiPartUploads.clear();
    if (e != null) {
        throw new RuntimeException("failed while closing transport", e);
    }
}

Example 3 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project beam by apache.

the class S3WritableByteChannel method close.

@Override
public void close() throws IOException {
    open = false;
    if (uploadBuffer.remaining() > 0) {
        flush();
    }
    CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest().withBucketName(path.getBucket()).withKey(path.getKey()).withUploadId(uploadId).withPartETags(eTags);
    try {
        amazonS3.completeMultipartUpload(request);
    } catch (AmazonClientException e) {
        throw new IOException(e);
    }
}

Example 4 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project beam by apache.

the class S3FileSystem method multipartCopy.

@VisibleForTesting
CompleteMultipartUploadResponse multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, HeadObjectResponse sourceObjectHead) throws SdkServiceException {
    CreateMultipartUploadRequest initiateUploadRequest = CreateMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).storageClass(config.getS3StorageClass()).metadata(sourceObjectHead.metadata()).serverSideEncryption(config.getSSEAlgorithm()).ssekmsKeyId(config.getSSEKMSKeyId()).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
    CreateMultipartUploadResponse createMultipartUploadResponse = s3Client.get().createMultipartUpload(initiateUploadRequest);
    final String uploadId = createMultipartUploadResponse.uploadId();
    List<CompletedPart> completedParts = new ArrayList<>();
    final long objectSize = sourceObjectHead.contentLength();
    CopyPartResult copyPartResult;
    CompletedPart completedPart;
    // without using S3FileSystem.copy in the future
    if (objectSize == 0) {
        final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(sourcePath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(1).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
        copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
        completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
        completedParts.add(completedPart);
    } else {
        long bytePosition = 0;
        Integer uploadBufferSizeBytes = config.getS3UploadBufferSizeBytes();
        // Amazon parts are 1-indexed, not zero-indexed.
        for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
            final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(destinationPath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(partNumber).copySourceRange(String.format("bytes=%s-%s", bytePosition, Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1))).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
            copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
            completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
            completedParts.add(completedPart);
            bytePosition += uploadBufferSizeBytes;
        }
    }
    CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
    CompleteMultipartUploadRequest completeUploadRequest = CompleteMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
    return s3Client.get().completeMultipartUpload(completeUploadRequest);
}

Example 5 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project aws-doc-sdk-examples by awsdocs.

the class LowLevelMultipartCopy method main.

public static void main(String[] args) throws IOException {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String sourceBucketName = "*** Source bucket name ***";
    String sourceObjectKey = "*** Source object key ***";
    String destBucketName = "*** Target bucket name ***";
    String destObjectKey = "*** Target object key ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(new ProfileCredentialsProvider()).withRegion(clientRegion).build();
        // Initiate the multipart upload.
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(destBucketName, destObjectKey);
        InitiateMultipartUploadResult initResult = s3Client.initiateMultipartUpload(initRequest);
        // Get the object size to track the end of the copy operation.
        GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(sourceBucketName, sourceObjectKey);
        ObjectMetadata metadataResult = s3Client.getObjectMetadata(metadataRequest);
        long objectSize = metadataResult.getContentLength();
        // Copy the object using 5 MB parts.
        long partSize = 5 * 1024 * 1024;
        long bytePosition = 0;
        int partNum = 1;
        List<CopyPartResult> copyResponses = new ArrayList<CopyPartResult>();
        while (bytePosition < objectSize) {
            // The last part might be smaller than partSize, so check to make sure
            // that lastByte isn't beyond the end of the object.
            long lastByte = Math.min(bytePosition + partSize - 1, objectSize - 1);
            // Copy this part.
            CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucketName).withSourceKey(sourceObjectKey).withDestinationBucketName(destBucketName).withDestinationKey(destObjectKey).withUploadId(initResult.getUploadId()).withFirstByte(bytePosition).withLastByte(lastByte).withPartNumber(partNum++);
            copyResponses.add(s3Client.copyPart(copyRequest));
            bytePosition += partSize;
        }
        // Complete the upload request to concatenate all uploaded parts and make the copied object available.
        CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(destBucketName, destObjectKey, initResult.getUploadId(), getETags(copyResponses));
        s3Client.completeMultipartUpload(completeRequest);
        System.out.println("Multipart copy complete.");
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client
        // couldn't parse the response from Amazon S3.
        e.printStackTrace();
    }
}