Examples with CreateMultipartUploadResponse software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse used on opensource projects

Example 1 with CreateMultipartUploadResponse

use of software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse in project beam by apache.

the class S3FileSystem method multipartCopy.

@VisibleForTesting
CompleteMultipartUploadResponse multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, HeadObjectResponse sourceObjectHead) throws SdkServiceException {
    CreateMultipartUploadRequest initiateUploadRequest = CreateMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).storageClass(config.getS3StorageClass()).metadata(sourceObjectHead.metadata()).serverSideEncryption(config.getSSEAlgorithm()).ssekmsKeyId(config.getSSEKMSKeyId()).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
    CreateMultipartUploadResponse createMultipartUploadResponse = s3Client.get().createMultipartUpload(initiateUploadRequest);
    final String uploadId = createMultipartUploadResponse.uploadId();
    List<CompletedPart> completedParts = new ArrayList<>();
    final long objectSize = sourceObjectHead.contentLength();
    CopyPartResult copyPartResult;
    CompletedPart completedPart;
    // without using S3FileSystem.copy in the future
    if (objectSize == 0) {
        final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(sourcePath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(1).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
        copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
        completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
        completedParts.add(completedPart);
    } else {
        long bytePosition = 0;
        Integer uploadBufferSizeBytes = config.getS3UploadBufferSizeBytes();
        // Amazon parts are 1-indexed, not zero-indexed.
        for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
            final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(destinationPath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(partNumber).copySourceRange(String.format("bytes=%s-%s", bytePosition, Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1))).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
            copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
            completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
            completedParts.add(completedPart);
            bytePosition += uploadBufferSizeBytes;
        }
    }
    CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
    CompleteMultipartUploadRequest completeUploadRequest = CompleteMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
    return s3Client.get().completeMultipartUpload(completeUploadRequest);
}

Example 2 with CreateMultipartUploadResponse

use of software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse in project beam by apache.

the class S3FileSystemTest method testMultipartCopy.

private void testMultipartCopy(S3FileSystem s3FileSystem, SSECustomerKey sseCustomerKey, long s3UploadBufferSizeBytes) throws IOException {
    S3ResourceId sourcePath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/from");
    S3ResourceId destinationPath = S3ResourceId.fromUri(s3FileSystem.getScheme() + "://bucket/to");
    CreateMultipartUploadResponse.Builder builder = CreateMultipartUploadResponse.builder().uploadId("upload-id");
    String sseCustomerKeyMd5 = toMd5(sseCustomerKey);
    if (sseCustomerKeyMd5 != null) {
        builder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    CreateMultipartUploadResponse createMultipartUploadResponse = builder.build();
    when(s3FileSystem.getS3Client().createMultipartUpload(any(CreateMultipartUploadRequest.class))).thenReturn(createMultipartUploadResponse);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getS3Client().createMultipartUpload(CreateMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).build()).sseCustomerKeyMD5());
    HeadObjectResponse.Builder headObjectResponseBuilder = HeadObjectResponse.builder().contentLength((long) (s3UploadBufferSizeBytes * 1.5)).contentEncoding("read-seek-efficient");
    if (sseCustomerKeyMd5 != null) {
        headObjectResponseBuilder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    HeadObjectResponse headObjectResponse = headObjectResponseBuilder.build();
    assertGetObjectHead(s3FileSystem, createObjectHeadRequest(sourcePath, sseCustomerKey), sseCustomerKeyMd5, headObjectResponse);
    CopyPartResult copyPartResult1 = CopyPartResult.builder().eTag("etag-1").build();
    CopyPartResult copyPartResult2 = CopyPartResult.builder().eTag("etag-2").build();
    UploadPartCopyResponse.Builder uploadPartCopyResponseBuilder1 = UploadPartCopyResponse.builder().copyPartResult(copyPartResult1);
    UploadPartCopyResponse.Builder uploadPartCopyResponseBuilder2 = UploadPartCopyResponse.builder().copyPartResult(copyPartResult2);
    if (sseCustomerKeyMd5 != null) {
        uploadPartCopyResponseBuilder1.sseCustomerKeyMD5(sseCustomerKeyMd5);
        uploadPartCopyResponseBuilder2.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    UploadPartCopyResponse uploadPartCopyResponse1 = uploadPartCopyResponseBuilder1.build();
    UploadPartCopyResponse uploadPartCopyResponse2 = uploadPartCopyResponseBuilder2.build();
    UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().sseCustomerKey(sseCustomerKey.getKey()).build();
    when(s3FileSystem.getS3Client().uploadPartCopy(any(UploadPartCopyRequest.class))).thenReturn(uploadPartCopyResponse1).thenReturn(uploadPartCopyResponse2);
    assertEquals(sseCustomerKeyMd5, s3FileSystem.getS3Client().uploadPartCopy(uploadPartCopyRequest).sseCustomerKeyMD5());
    s3FileSystem.multipartCopy(sourcePath, destinationPath, headObjectResponse);
    verify(s3FileSystem.getS3Client(), times(1)).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
}

Example 3 with CreateMultipartUploadResponse

use of software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse in project aws-doc-sdk-examples by awsdocs.

the class S3ObjectOperations method multipartUpload.

/**
 * Upload an object in parts
 */
private static void multipartUpload(String bucketName, String key) throws IOException {
    int mB = 1024 * 1024;
    // snippet-start:[s3.java2.s3_object_operations.upload_multi_part]
    // First create a multipart upload and get the upload id
    CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(bucketName).key(key).build();
    CreateMultipartUploadResponse response = s3.createMultipartUpload(createMultipartUploadRequest);
    String uploadId = response.uploadId();
    System.out.println(uploadId);
    // Upload all the different parts of the object
    UploadPartRequest uploadPartRequest1 = UploadPartRequest.builder().bucket(bucketName).key(key).uploadId(uploadId).partNumber(1).build();
    String etag1 = s3.uploadPart(uploadPartRequest1, RequestBody.fromByteBuffer(getRandomByteBuffer(5 * mB))).eTag();
    CompletedPart part1 = CompletedPart.builder().partNumber(1).eTag(etag1).build();
    UploadPartRequest uploadPartRequest2 = UploadPartRequest.builder().bucket(bucketName).key(key).uploadId(uploadId).partNumber(2).build();
    String etag2 = s3.uploadPart(uploadPartRequest2, RequestBody.fromByteBuffer(getRandomByteBuffer(3 * mB))).eTag();
    CompletedPart part2 = CompletedPart.builder().partNumber(2).eTag(etag2).build();
    // Finally call completeMultipartUpload operation to tell S3 to merge all uploaded
    // parts and finish the multipart operation.
    CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(part1, part2).build();
    CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder().bucket(bucketName).key(key).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
    s3.completeMultipartUpload(completeMultipartUploadRequest);
// snippet-end:[s3.java2.s3_object_operations.upload_multi_part]
}

Example 4 with CreateMultipartUploadResponse

use of software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse in project beam by apache.

the class S3WritableByteChannelTest method write.

private void write(S3Client mockS3Client, Supplier channelSupplier, S3ResourceId path, String sseAlgorithmStr, String sseCustomerKeyMd5, String ssekmsKeyId, long s3UploadBufferSizeBytes, boolean bucketKeyEnabled, boolean writeReadOnlyBuffer) throws IOException {
    CreateMultipartUploadResponse.Builder builder = CreateMultipartUploadResponse.builder().uploadId("upload-id");
    ServerSideEncryption sseAlgorithm = ServerSideEncryption.fromValue(sseAlgorithmStr);
    if (sseAlgorithm != null) {
        builder.serverSideEncryption(sseAlgorithm);
    }
    if (sseCustomerKeyMd5 != null) {
        builder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    if (ssekmsKeyId != null) {
        sseAlgorithm = ServerSideEncryption.AWS_KMS;
        builder.serverSideEncryption(sseAlgorithm);
    }
    builder.bucketKeyEnabled(bucketKeyEnabled);
    CreateMultipartUploadResponse createMultipartUploadResponse = builder.build();
    doReturn(createMultipartUploadResponse).when(mockS3Client).createMultipartUpload(any(CreateMultipartUploadRequest.class));
    CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(path.getBucket()).key(path.getKey()).build();
    CreateMultipartUploadResponse mockCreateMultipartUploadResponse1 = mockS3Client.createMultipartUpload(createMultipartUploadRequest);
    assertEquals(sseAlgorithm, mockCreateMultipartUploadResponse1.serverSideEncryption());
    assertEquals(sseCustomerKeyMd5, mockCreateMultipartUploadResponse1.sseCustomerKeyMD5());
    assertEquals(bucketKeyEnabled, mockCreateMultipartUploadResponse1.bucketKeyEnabled());
    UploadPartResponse.Builder uploadPartResponseBuilder = UploadPartResponse.builder().eTag("etag");
    if (sseCustomerKeyMd5 != null) {
        uploadPartResponseBuilder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    UploadPartResponse response = uploadPartResponseBuilder.build();
    doReturn(response).when(mockS3Client).uploadPart(any(UploadPartRequest.class), any(RequestBody.class));
    UploadPartResponse mockUploadPartResult = mockS3Client.uploadPart(UploadPartRequest.builder().build(), RequestBody.empty());
    assertEquals(sseCustomerKeyMd5, mockUploadPartResult.sseCustomerKeyMD5());
    S3WritableByteChannel channel = channelSupplier.get();
    int contentSize = 34_078_720;
    ByteBuffer uploadContent = ByteBuffer.allocate((int) (contentSize * 2.5));
    for (int i = 0; i < contentSize; i++) {
        uploadContent.put((byte) 0xff);
    }
    uploadContent.flip();
    int uploadedSize = channel.write(writeReadOnlyBuffer ? uploadContent.asReadOnlyBuffer() : uploadContent);
    assertEquals(contentSize, uploadedSize);
    CompleteMultipartUploadResponse completeMultipartUploadResponse = CompleteMultipartUploadResponse.builder().build();
    doReturn(completeMultipartUploadResponse).when(mockS3Client).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
    channel.close();
    int partQuantity = (int) Math.ceil((double) contentSize / s3UploadBufferSizeBytes) + 1;
    verify(mockS3Client, times(2)).createMultipartUpload((CreateMultipartUploadRequest) isNotNull());
    verify(mockS3Client, times(partQuantity)).uploadPart((UploadPartRequest) isNotNull(), any(RequestBody.class));
    verify(mockS3Client, times(1)).completeMultipartUpload((CompleteMultipartUploadRequest) notNull());
    verifyNoMoreInteractions(mockS3Client);
}