Examples with CopyPartResult com.amazonaws.services.s3.model.CopyPartResult used on opensource projects

Example 1 with CopyPartResult

use of com.amazonaws.services.s3.model.CopyPartResult in project beam by apache.

the class S3FileSystemTest method testMultipartCopy.

private void testMultipartCopy(S3FileSystem s3FileSystem, SSECustomerKey sseCustomerKey, long s3UploadBufferSizeBytes) {
    S3ResourceId sourcePath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/from");
    S3ResourceId destinationPath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/to");
    InitiateMultipartUploadResult initiateMultipartUploadResult = new InitiateMultipartUploadResult();
    initiateMultipartUploadResult.setUploadId("upload-id");
    String sseCustomerKeyMd5 = toMd5(sseCustomerKey);
    if (sseCustomerKeyMd5 != null) {
        initiateMultipartUploadResult.setSSECustomerKeyMd5(sseCustomerKeyMd5);
    }
    when(s3FileSystem.getAmazonS3Client().initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(initiateMultipartUploadResult);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getAmazonS3Client().initiateMultipartUpload(new InitiateMultipartUploadRequest(destinationPath.getBucket(), destinationPath.getKey())).getSSECustomerKeyMd5());
    ObjectMetadata sourceObjectMetadata = new ObjectMetadata();
    sourceObjectMetadata.setContentLength((long) (s3UploadBufferSizeBytes * 1.5));
    sourceObjectMetadata.setContentEncoding("read-seek-efficient");
    if (sseCustomerKeyMd5 != null) {
        sourceObjectMetadata.setSSECustomerKeyMd5(sseCustomerKeyMd5);
    }
    assertGetObjectMetadata(s3FileSystem, createObjectMetadataRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, sourceObjectMetadata);
    CopyPartResult copyPartResult1 = new CopyPartResult();
    copyPartResult1.setETag("etag-1");
    CopyPartResult copyPartResult2 = new CopyPartResult();
    copyPartResult1.setETag("etag-2");
    if (sseCustomerKeyMd5 != null) {
        copyPartResult1.setSSECustomerKeyMd5(sseCustomerKeyMd5);
        copyPartResult2.setSSECustomerKeyMd5(sseCustomerKeyMd5);
    }
    CopyPartRequest copyPartRequest = new CopyPartRequest();
    copyPartRequest.setSourceSSECustomerKey(sseCustomerKey);
    when(s3FileSystem.getAmazonS3Client().copyPart(any(CopyPartRequest.class))).thenReturn(copyPartResult1).thenReturn(copyPartResult2);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getAmazonS3Client().copyPart(copyPartRequest).getSSECustomerKeyMd5());
    s3FileSystem.multipartCopy(sourcePath, destinationPath, sourceObjectMetadata);
    verify(s3FileSystem.getAmazonS3Client(), times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
}

Example 2 with CopyPartResult

use of com.amazonaws.services.s3.model.CopyPartResult in project beam by apache.

the class S3FileSystem method multipartCopy.

@VisibleForTesting
CompleteMultipartUploadResponse multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, HeadObjectResponse sourceObjectHead) throws SdkServiceException {
    CreateMultipartUploadRequest initiateUploadRequest = CreateMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).storageClass(config.getS3StorageClass()).metadata(sourceObjectHead.metadata()).serverSideEncryption(config.getSSEAlgorithm()).ssekmsKeyId(config.getSSEKMSKeyId()).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
    CreateMultipartUploadResponse createMultipartUploadResponse = s3Client.get().createMultipartUpload(initiateUploadRequest);
    final String uploadId = createMultipartUploadResponse.uploadId();
    List<CompletedPart> completedParts = new ArrayList<>();
    final long objectSize = sourceObjectHead.contentLength();
    CopyPartResult copyPartResult;
    CompletedPart completedPart;
    // without using S3FileSystem.copy in the future
    if (objectSize == 0) {
        final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(sourcePath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(1).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
        copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
        completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
        completedParts.add(completedPart);
    } else {
        long bytePosition = 0;
        Integer uploadBufferSizeBytes = config.getS3UploadBufferSizeBytes();
        // Amazon parts are 1-indexed, not zero-indexed.
        for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
            final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(destinationPath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(partNumber).copySourceRange(String.format("bytes=%s-%s", bytePosition, Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1))).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
            copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
            completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
            completedParts.add(completedPart);
            bytePosition += uploadBufferSizeBytes;
        }
    }
    CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
    CompleteMultipartUploadRequest completeUploadRequest = CompleteMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
    return s3Client.get().completeMultipartUpload(completeUploadRequest);
}

Example 3 with CopyPartResult

use of com.amazonaws.services.s3.model.CopyPartResult in project beam by apache.

the class S3FileSystemTest method testMultipartCopy.

private void testMultipartCopy(S3FileSystem s3FileSystem, SSECustomerKey sseCustomerKey, long s3UploadBufferSizeBytes) throws IOException {
    S3ResourceId sourcePath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/from");
    S3ResourceId destinationPath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/to");
    CreateMultipartUploadResponse.Builder builder = CreateMultipartUploadResponse.builder().uploadId("upload-id");
    String sseCustomerKeyMd5 = toMd5(sseCustomerKey);
    if (sseCustomerKeyMd5 != null) {
        builder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    CreateMultipartUploadResponse createMultipartUploadResponse = builder.build();
    when(s3FileSystem.getS3Client().createMultipartUpload(any(CreateMultipartUploadRequest.class))).thenReturn(createMultipartUploadResponse);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getS3Client().createMultipartUpload(CreateMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).build()).sseCustomerKeyMD5());
    HeadObjectResponse.Builder headObjectResponseBuilder = HeadObjectResponse.builder().contentLength((long) (s3UploadBufferSizeBytes * 1.5)).contentEncoding("read-seek-efficient");
    if (sseCustomerKeyMd5 != null) {
        headObjectResponseBuilder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    HeadObjectResponse headObjectResponse = headObjectResponseBuilder.build();
    assertGetObjectHead(s3FileSystem, createObjectHeadRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, headObjectResponse);
    CopyPartResult copyPartResult1 = CopyPartResult.builder().eTag("etag-1").build();
    CopyPartResult copyPartResult2 = CopyPartResult.builder().eTag("etag-2").build();
    UploadPartCopyResponse.Builder uploadPartCopyResponseBuilder1 = UploadPartCopyResponse.builder().copyPartResult(copyPartResult1);
    UploadPartCopyResponse.Builder uploadPartCopyResponseBuilder2 = UploadPartCopyResponse.builder().copyPartResult(copyPartResult2);
    if (sseCustomerKeyMd5 != null) {
        uploadPartCopyResponseBuilder1.sseCustomerKeyMD5(sseCustomerKeyMd5);
        uploadPartCopyResponseBuilder2.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    UploadPartCopyResponse uploadPartCopyResponse1 = uploadPartCopyResponseBuilder1.build();
    UploadPartCopyResponse uploadPartCopyResponse2 = uploadPartCopyResponseBuilder2.build();
    UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().sseCustomerKey(sseCustomerKey.getKey()).build();
    when(s3FileSystem.getS3Client().uploadPartCopy(any(UploadPartCopyRequest.class))).thenReturn(uploadPartCopyResponse1).thenReturn(uploadPartCopyResponse2);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getS3Client().uploadPartCopy(uploadPartCopyRequest).sseCustomerKeyMD5());
    s3FileSystem.multipartCopy(sourcePath, destinationPath, headObjectResponse);
    verify(s3FileSystem.getS3Client(), times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
}

Example 4 with CopyPartResult

use of com.amazonaws.services.s3.model.CopyPartResult in project aws-doc-sdk-examples by awsdocs.

the class LowLevelMultipartCopy method main.

public static void main(String[] args) throws IOException {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String sourceBucketName = "*** Source bucket name ***";
    String sourceObjectKey = "*** Source object key ***";
    String destBucketName = "*** Target bucket name ***";
    String destObjectKey = "*** Target object key ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(new ProfileCredentialsProvider()).withRegion(clientRegion).build();
        // Initiate the multipart upload.
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(destBucketName, destObjectKey);
        InitiateMultipartUploadResult initResult = s3Client.initiateMultipartUpload(initRequest);
        // Get the object size to track the end of the copy operation.
        GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(sourceBucketName, sourceObjectKey);
        ObjectMetadata metadataResult = s3Client.getObjectMetadata(metadataRequest);
        long objectSize = metadataResult.getContentLength();
        // Copy the object using 5 MB parts.
        long partSize = 5 * 1024 * 1024;
        long bytePosition = 0;
        int partNum = 1;
        List<CopyPartResult> copyResponses = new ArrayList<CopyPartResult>();
        while (bytePosition < objectSize) {
            // The last part might be smaller than partSize, so check to make sure
            // that lastByte isn't beyond the end of the object.
            long lastByte = Math.min(bytePosition + partSize - 1, objectSize - 1);
            // Copy this part.
            CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucketName).withSourceKey(sourceObjectKey).withDestinationBucketName(destBucketName).withDestinationKey(destObjectKey).withUploadId(initResult.getUploadId()).withFirstByte(bytePosition).withLastByte(lastByte).withPartNumber(partNum++);
            copyResponses.add(s3Client.copyPart(copyRequest));
            bytePosition += partSize;
        }
        // Complete the upload request to concatenate all uploaded parts and make the copied object available.
        CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(destBucketName, destObjectKey, initResult.getUploadId(), getETags(copyResponses));
        s3Client.completeMultipartUpload(completeRequest);
        System.out.println("Multipart copy complete.");
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client
        // couldn't parse the response from Amazon S3.
        e.printStackTrace();
    }
}

Example 5 with CopyPartResult

use of com.amazonaws.services.s3.model.CopyPartResult in project beam by apache.

the class S3FileSystem method multipartCopy.

@VisibleForTesting
CompleteMultipartUploadResult multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, ObjectMetadata sourceObjectMetadata) throws AmazonClientException {
    InitiateMultipartUploadRequest initiateUploadRequest = new InitiateMultipartUploadRequest(destinationPath.getBucket(), destinationPath.getKey()).withStorageClass(config.getS3StorageClass()).withObjectMetadata(sourceObjectMetadata);
    initiateUploadRequest.setSSECustomerKey(config.getSSECustomerKey());
    InitiateMultipartUploadResult initiateUploadResult = amazonS3.get().initiateMultipartUpload(initiateUploadRequest);
    final String uploadId = initiateUploadResult.getUploadId();
    List<PartETag> eTags = new ArrayList<>();
    final long objectSize = sourceObjectMetadata.getContentLength();
    // without using S3FileSystem.copy in the future
    if (objectSize == 0) {
        final CopyPartRequest copyPartRequest = new CopyPartRequest().withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()).withDestinationBucketName(destinationPath.getBucket()).withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(1);
        copyPartRequest.setSourceSSECustomerKey(config.getSSECustomerKey());
        copyPartRequest.setDestinationSSECustomerKey(config.getSSECustomerKey());
        CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
        eTags.add(copyPartResult.getPartETag());
    } else {
        long bytePosition = 0;
        Integer uploadBufferSizeBytes = config.getS3UploadBufferSizeBytes();
        // Amazon parts are 1-indexed, not zero-indexed.
        for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
            final CopyPartRequest copyPartRequest = new CopyPartRequest().withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()).withDestinationBucketName(destinationPath.getBucket()).withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(partNumber).withFirstByte(bytePosition).withLastByte(Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1));
            copyPartRequest.setSourceSSECustomerKey(config.getSSECustomerKey());
            copyPartRequest.setDestinationSSECustomerKey(config.getSSECustomerKey());
            CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
            eTags.add(copyPartResult.getPartETag());
            bytePosition += uploadBufferSizeBytes;
        }
    }
    CompleteMultipartUploadRequest completeUploadRequest = new CompleteMultipartUploadRequest().withBucketName(destinationPath.getBucket()).withKey(destinationPath.getKey()).withUploadId(uploadId).withPartETags(eTags);
    return amazonS3.get().completeMultipartUpload(completeUploadRequest);
}