Examples with CopyPartRequest com.amazonaws.services.s3.model.CopyPartRequest used on opensource projects

Example 1 with CopyPartRequest

use of com.amazonaws.services.s3.model.CopyPartRequest in project beam by apache.

the class S3FileSystemTest method testMultipartCopy.

private void testMultipartCopy(S3FileSystem s3FileSystem, SSECustomerKey sseCustomerKey, long s3UploadBufferSizeBytes) {
    S3ResourceId sourcePath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/from");
    S3ResourceId destinationPath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/to");
    InitiateMultipartUploadResult initiateMultipartUploadResult = new InitiateMultipartUploadResult();
    initiateMultipartUploadResult.setUploadId("upload-id");
    String sseCustomerKeyMd5 = toMd5(sseCustomerKey);
    if (sseCustomerKeyMd5 != null) {
        initiateMultipartUploadResult.setSSECustomerKeyMd5(sseCustomerKeyMd5);
    }
    when(s3FileSystem.getAmazonS3Client().initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(initiateMultipartUploadResult);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getAmazonS3Client().initiateMultipartUpload(new InitiateMultipartUploadRequest(destinationPath.getBucket(), destinationPath.getKey())).getSSECustomerKeyMd5());
    ObjectMetadata sourceObjectMetadata = new ObjectMetadata();
    sourceObjectMetadata.setContentLength((long) (s3UploadBufferSizeBytes * 1.5));
    sourceObjectMetadata.setContentEncoding("read-seek-efficient");
    if (sseCustomerKeyMd5 != null) {
        sourceObjectMetadata.setSSECustomerKeyMd5(sseCustomerKeyMd5);
    }
    assertGetObjectMetadata(s3FileSystem, createObjectMetadataRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, sourceObjectMetadata);
    CopyPartResult copyPartResult1 = new CopyPartResult();
    copyPartResult1.setETag("etag-1");
    CopyPartResult copyPartResult2 = new CopyPartResult();
    copyPartResult1.setETag("etag-2");
    if (sseCustomerKeyMd5 != null) {
        copyPartResult1.setSSECustomerKeyMd5(sseCustomerKeyMd5);
        copyPartResult2.setSSECustomerKeyMd5(sseCustomerKeyMd5);
    }
    CopyPartRequest copyPartRequest = new CopyPartRequest();
    copyPartRequest.setSourceSSECustomerKey(sseCustomerKey);
    when(s3FileSystem.getAmazonS3Client().copyPart(any(CopyPartRequest.class))).thenReturn(copyPartResult1).thenReturn(copyPartResult2);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getAmazonS3Client().copyPart(copyPartRequest).getSSECustomerKeyMd5());
    s3FileSystem.multipartCopy(sourcePath, destinationPath, sourceObjectMetadata);
    verify(s3FileSystem.getAmazonS3Client(), times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
}

Example 2 with CopyPartRequest

use of com.amazonaws.services.s3.model.CopyPartRequest in project aws-doc-sdk-examples by awsdocs.

the class LowLevelMultipartCopy method main.

public static void main(String[] args) throws IOException {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String sourceBucketName = "*** Source bucket name ***";
    String sourceObjectKey = "*** Source object key ***";
    String destBucketName = "*** Target bucket name ***";
    String destObjectKey = "*** Target object key ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(new ProfileCredentialsProvider()).withRegion(clientRegion).build();
        // Initiate the multipart upload.
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(destBucketName, destObjectKey);
        InitiateMultipartUploadResult initResult = s3Client.initiateMultipartUpload(initRequest);
        // Get the object size to track the end of the copy operation.
        GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(sourceBucketName, sourceObjectKey);
        ObjectMetadata metadataResult = s3Client.getObjectMetadata(metadataRequest);
        long objectSize = metadataResult.getContentLength();
        // Copy the object using 5 MB parts.
        long partSize = 5 * 1024 * 1024;
        long bytePosition = 0;
        int partNum = 1;
        List<CopyPartResult> copyResponses = new ArrayList<CopyPartResult>();
        while (bytePosition < objectSize) {
            // The last part might be smaller than partSize, so check to make sure
            // that lastByte isn't beyond the end of the object.
            long lastByte = Math.min(bytePosition + partSize - 1, objectSize - 1);
            // Copy this part.
            CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucketName).withSourceKey(sourceObjectKey).withDestinationBucketName(destBucketName).withDestinationKey(destObjectKey).withUploadId(initResult.getUploadId()).withFirstByte(bytePosition).withLastByte(lastByte).withPartNumber(partNum++);
            copyResponses.add(s3Client.copyPart(copyRequest));
            bytePosition += partSize;
        }
        // Complete the upload request to concatenate all uploaded parts and make the copied object available.
        CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(destBucketName, destObjectKey, initResult.getUploadId(), getETags(copyResponses));
        s3Client.completeMultipartUpload(completeRequest);
        System.out.println("Multipart copy complete.");
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client
        // couldn't parse the response from Amazon S3.
        e.printStackTrace();
    }
}

Example 3 with CopyPartRequest

use of com.amazonaws.services.s3.model.CopyPartRequest in project beam by apache.

the class S3FileSystem method multipartCopy.

@VisibleForTesting
CompleteMultipartUploadResult multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, ObjectMetadata sourceObjectMetadata) throws AmazonClientException {
    InitiateMultipartUploadRequest initiateUploadRequest = new InitiateMultipartUploadRequest(destinationPath.getBucket(), destinationPath.getKey()).withStorageClass(config.getS3StorageClass()).withObjectMetadata(sourceObjectMetadata);
    initiateUploadRequest.setSSECustomerKey(config.getSSECustomerKey());
    InitiateMultipartUploadResult initiateUploadResult = amazonS3.get().initiateMultipartUpload(initiateUploadRequest);
    final String uploadId = initiateUploadResult.getUploadId();
    List<PartETag> eTags = new ArrayList<>();
    final long objectSize = sourceObjectMetadata.getContentLength();
    // without using S3FileSystem.copy in the future
    if (objectSize == 0) {
        final CopyPartRequest copyPartRequest = new CopyPartRequest().withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()).withDestinationBucketName(destinationPath.getBucket()).withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(1);
        copyPartRequest.setSourceSSECustomerKey(config.getSSECustomerKey());
        copyPartRequest.setDestinationSSECustomerKey(config.getSSECustomerKey());
        CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
        eTags.add(copyPartResult.getPartETag());
    } else {
        long bytePosition = 0;
        Integer uploadBufferSizeBytes = config.getS3UploadBufferSizeBytes();
        // Amazon parts are 1-indexed, not zero-indexed.
        for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
            final CopyPartRequest copyPartRequest = new CopyPartRequest().withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()).withDestinationBucketName(destinationPath.getBucket()).withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(partNumber).withFirstByte(bytePosition).withLastByte(Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1));
            copyPartRequest.setSourceSSECustomerKey(config.getSSECustomerKey());
            copyPartRequest.setDestinationSSECustomerKey(config.getSSECustomerKey());
            CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
            eTags.add(copyPartResult.getPartETag());
            bytePosition += uploadBufferSizeBytes;
        }
    }
    CompleteMultipartUploadRequest completeUploadRequest = new CompleteMultipartUploadRequest().withBucketName(destinationPath.getBucket()).withKey(destinationPath.getKey()).withUploadId(uploadId).withPartETags(eTags);
    return amazonS3.get().completeMultipartUpload(completeUploadRequest);
}