Examples with CompleteMultipartUploadRequest com.amazonaws.services.s3.model.CompleteMultipartUploadRequest used on opensource projects

Example 21 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project beam by apache.

the class S3FileSystem method multipartCopy.

@VisibleForTesting
CompleteMultipartUploadResult multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, ObjectMetadata sourceObjectMetadata) throws AmazonClientException {
    InitiateMultipartUploadRequest initiateUploadRequest = new InitiateMultipartUploadRequest(destinationPath.getBucket(), destinationPath.getKey()).withStorageClass(config.getS3StorageClass()).withObjectMetadata(sourceObjectMetadata);
    initiateUploadRequest.setSSECustomerKey(config.getSSECustomerKey());
    InitiateMultipartUploadResult initiateUploadResult = amazonS3.get().initiateMultipartUpload(initiateUploadRequest);
    final String uploadId = initiateUploadResult.getUploadId();
    List<PartETag> eTags = new ArrayList<>();
    final long objectSize = sourceObjectMetadata.getContentLength();
    // without using S3FileSystem.copy in the future
    if (objectSize == 0) {
        final CopyPartRequest copyPartRequest = new CopyPartRequest().withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()).withDestinationBucketName(destinationPath.getBucket()).withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(1);
        copyPartRequest.setSourceSSECustomerKey(config.getSSECustomerKey());
        copyPartRequest.setDestinationSSECustomerKey(config.getSSECustomerKey());
        CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
        eTags.add(copyPartResult.getPartETag());
    } else {
        long bytePosition = 0;
        Integer uploadBufferSizeBytes = config.getS3UploadBufferSizeBytes();
        // Amazon parts are 1-indexed, not zero-indexed.
        for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
            final CopyPartRequest copyPartRequest = new CopyPartRequest().withSourceBucketName(sourcePath.getBucket()).withSourceKey(sourcePath.getKey()).withDestinationBucketName(destinationPath.getBucket()).withDestinationKey(destinationPath.getKey()).withUploadId(uploadId).withPartNumber(partNumber).withFirstByte(bytePosition).withLastByte(Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1));
            copyPartRequest.setSourceSSECustomerKey(config.getSSECustomerKey());
            copyPartRequest.setDestinationSSECustomerKey(config.getSSECustomerKey());
            CopyPartResult copyPartResult = amazonS3.get().copyPart(copyPartRequest);
            eTags.add(copyPartResult.getPartETag());
            bytePosition += uploadBufferSizeBytes;
        }
    }
    CompleteMultipartUploadRequest completeUploadRequest = new CompleteMultipartUploadRequest().withBucketName(destinationPath.getBucket()).withKey(destinationPath.getKey()).withUploadId(uploadId).withPartETags(eTags);
    return amazonS3.get().completeMultipartUpload(completeUploadRequest);
}

Example 22 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project apex-malhar by apache.

the class S3FileMerger method verifyAndEmitFileMerge.

/**
 * Send the CompleteMultipartUploadRequest to S3 if all the blocks of a file are uploaded into S3.
 * @param keyName file to upload into S3
 */
private void verifyAndEmitFileMerge(String keyName) {
    if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) {
        return;
    }
    S3InitiateFileUploadOperator.UploadFileMetadata uploadFileMetadata = fileMetadatas.get(keyName);
    List<PartETag> partETags = uploadParts.get(keyName);
    if (partETags == null || uploadFileMetadata == null || uploadFileMetadata.getFileMetadata().getNumberOfBlocks() != partETags.size()) {
        return;
    }
    if (partETags.size() <= 1) {
        uploadedFiles.add(keyName);
        LOG.debug("Uploaded file {} successfully", keyName);
        return;
    }
    CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName, uploadFileMetadata.getUploadId(), partETags);
    CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(compRequest);
    if (result.getETag() != null) {
        uploadedFiles.add(keyName);
        LOG.debug("Uploaded file {} successfully", keyName);
    }
}

Example 23 with CompleteMultipartUploadRequest

use of com.amazonaws.services.s3.model.CompleteMultipartUploadRequest in project crate by crate.

the class S3BlobStoreContainerTests method testExecuteMultipartUpload.

@Test
public void testExecuteMultipartUpload() throws IOException {
    final String bucketName = randomAlphaOfLengthBetween(1, 10);
    final String blobName = randomAlphaOfLengthBetween(1, 10);
    final BlobPath blobPath = new BlobPath();
    if (randomBoolean()) {
        IntStream.of(randomIntBetween(1, 5)).forEach(value -> blobPath.add("path_" + value));
    }
    final long blobSize = ByteSizeUnit.GB.toBytes(randomIntBetween(1, 128));
    final long bufferSize = ByteSizeUnit.MB.toBytes(randomIntBetween(5, 1024));
    final S3BlobStore blobStore = mock(S3BlobStore.class);
    when(blobStore.bucket()).thenReturn(bucketName);
    when(blobStore.bufferSizeInBytes()).thenReturn(bufferSize);
    final boolean serverSideEncryption = randomBoolean();
    when(blobStore.serverSideEncryption()).thenReturn(serverSideEncryption);
    final StorageClass storageClass = randomFrom(StorageClass.values());
    when(blobStore.getStorageClass()).thenReturn(storageClass);
    final CannedAccessControlList cannedAccessControlList = randomBoolean() ? randomFrom(CannedAccessControlList.values()) : null;
    if (cannedAccessControlList != null) {
        when(blobStore.getCannedACL()).thenReturn(cannedAccessControlList);
    }
    final AmazonS3 client = mock(AmazonS3.class);
    final AmazonS3Reference clientReference = new AmazonS3Reference(client);
    when(blobStore.clientReference()).thenReturn(clientReference);
    final ArgumentCaptor<InitiateMultipartUploadRequest> initArgCaptor = ArgumentCaptor.forClass(InitiateMultipartUploadRequest.class);
    final InitiateMultipartUploadResult initResult = new InitiateMultipartUploadResult();
    initResult.setUploadId(randomAlphaOfLength(10));
    when(client.initiateMultipartUpload(initArgCaptor.capture())).thenReturn(initResult);
    final ArgumentCaptor<UploadPartRequest> uploadArgCaptor = ArgumentCaptor.forClass(UploadPartRequest.class);
    final List<String> expectedEtags = new ArrayList<>();
    final long partSize = Math.min(bufferSize, blobSize);
    long totalBytes = 0;
    do {
        expectedEtags.add(randomAlphaOfLength(50));
        totalBytes += partSize;
    } while (totalBytes < blobSize);
    when(client.uploadPart(uploadArgCaptor.capture())).thenAnswer(invocationOnMock -> {
        final UploadPartRequest request = (UploadPartRequest) invocationOnMock.getArguments()[0];
        final UploadPartResult response = new UploadPartResult();
        response.setPartNumber(request.getPartNumber());
        response.setETag(expectedEtags.get(request.getPartNumber() - 1));
        return response;
    });
    final ArgumentCaptor<CompleteMultipartUploadRequest> compArgCaptor = ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class);
    when(client.completeMultipartUpload(compArgCaptor.capture())).thenReturn(new CompleteMultipartUploadResult());
    final ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[0]);
    final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
    blobContainer.executeMultipartUpload(blobStore, blobName, inputStream, blobSize);
    final InitiateMultipartUploadRequest initRequest = initArgCaptor.getValue();
    assertEquals(bucketName, initRequest.getBucketName());
    assertEquals(blobPath.buildAsString() + blobName, initRequest.getKey());
    assertEquals(storageClass, initRequest.getStorageClass());
    assertEquals(cannedAccessControlList, initRequest.getCannedACL());
    if (serverSideEncryption) {
        assertEquals(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION, initRequest.getObjectMetadata().getSSEAlgorithm());
    }
    final Tuple<Long, Long> numberOfParts = S3BlobContainer.numberOfMultiparts(blobSize, bufferSize);
    final List<UploadPartRequest> uploadRequests = uploadArgCaptor.getAllValues();
    assertEquals(numberOfParts.v1().intValue(), uploadRequests.size());
    for (int i = 0; i < uploadRequests.size(); i++) {
        final UploadPartRequest uploadRequest = uploadRequests.get(i);
        assertEquals(bucketName, uploadRequest.getBucketName());
        assertEquals(blobPath.buildAsString() + blobName, uploadRequest.getKey());
        assertEquals(initResult.getUploadId(), uploadRequest.getUploadId());
        assertEquals(i + 1, uploadRequest.getPartNumber());
        assertEquals(inputStream, uploadRequest.getInputStream());
        if (i == (uploadRequests.size() - 1)) {
            assertTrue(uploadRequest.isLastPart());
            assertEquals(numberOfParts.v2().longValue(), uploadRequest.getPartSize());
        } else {
            assertFalse(uploadRequest.isLastPart());
            assertEquals(bufferSize, uploadRequest.getPartSize());
        }
    }
    final CompleteMultipartUploadRequest compRequest = compArgCaptor.getValue();
    assertEquals(bucketName, compRequest.getBucketName());
    assertEquals(blobPath.buildAsString() + blobName, compRequest.getKey());
    assertEquals(initResult.getUploadId(), compRequest.getUploadId());
    final List<String> actualETags = compRequest.getPartETags().stream().map(PartETag::getETag).collect(Collectors.toList());
    assertEquals(expectedEtags, actualETags);
}