Examples with CompleteMultipartUploadRequest software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest used on opensource projects

Example 6 with CompleteMultipartUploadRequest

use of software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest in project gradle by gradle.

the class S3Client method putMultiPartObject.

private void putMultiPartObject(InputStream inputStream, Long contentLength, URI destination) {
    try {
        S3RegionalResource s3RegionalResource = new S3RegionalResource(destination);
        String bucketName = s3RegionalResource.getBucketName();
        String s3BucketKey = s3RegionalResource.getKey();
        configureClient(s3RegionalResource);
        List<PartETag> partETags = new ArrayList<>();
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, s3BucketKey).withCannedACL(CannedAccessControlList.BucketOwnerFullControl);
        InitiateMultipartUploadResult initResponse = amazonS3Client.initiateMultipartUpload(initRequest);
        try {
            long filePosition = 0;
            long partSize = s3ConnectionProperties.getPartSize();
            LOGGER.debug("Attempting to put resource:[{}] into s3 bucket [{}]", s3BucketKey, bucketName);
            for (int partNumber = 1; filePosition < contentLength; partNumber++) {
                partSize = Math.min(partSize, contentLength - filePosition);
                UploadPartRequest uploadPartRequest = new UploadPartRequest().withBucketName(bucketName).withKey(s3BucketKey).withUploadId(initResponse.getUploadId()).withPartNumber(partNumber).withPartSize(partSize).withInputStream(inputStream);
                partETags.add(amazonS3Client.uploadPart(uploadPartRequest).getPartETag());
                filePosition += partSize;
            }
            CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(bucketName, s3BucketKey, initResponse.getUploadId(), partETags);
            amazonS3Client.completeMultipartUpload(completeRequest);
        } catch (AmazonClientException e) {
            amazonS3Client.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, s3BucketKey, initResponse.getUploadId()));
            throw e;
        }
    } catch (AmazonClientException e) {
        throw ResourceExceptions.putFailed(destination, e);
    }
}

Example 7 with CompleteMultipartUploadRequest

use of software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest in project hippo by NHS-digital-website.

the class S3SdkConnectorTest method uploadsFileAsS3Resource_usingMultipartRequest.

@Test
public void uploadsFileAsS3Resource_usingMultipartRequest() throws Exception {
    // given
    final String contentType = newRandomString();
    final String uploadId = newRandomString();
    final String s3ObjectUrl = "https://" + bucketName + "/" + objectKey;
    final byte[] fullChunk = newRandomByteArray(BUFFER_SIZE);
    final byte[] partialChunk = newRandomByteArray(1024);
    final byte[][] expectedChunks = { fullChunk, partialChunk };
    final long contentLength = fullChunk.length + partialChunk.length;
    given(s3ObjectKeyGenerator.generateObjectKey(fileName)).willReturn(objectKey);
    final InitiateMultipartUploadResult result = mock(InitiateMultipartUploadResult.class);
    given(result.getUploadId()).willReturn(uploadId);
    given(s3.initiateMultipartUpload(any())).willReturn(result);
    final PartETag firstPartETag = mock(PartETag.class);
    final PartETag finalPartETag = mock(PartETag.class);
    final UploadPartResult uploadPartResult = mock(UploadPartResult.class);
    given(uploadPartResult.getPartETag()).willReturn(firstPartETag, finalPartETag);
    given(s3.uploadPart(any(UploadPartRequest.class))).willReturn(uploadPartResult);
    final ObjectMetadata resultMetadata = mock(ObjectMetadata.class);
    given(resultMetadata.getContentType()).willReturn(contentType);
    given(resultMetadata.getContentLength()).willReturn(contentLength);
    given(s3.getObjectMetadata(bucketName, objectKey)).willReturn(resultMetadata);
    final InputStream uploadedFileInputStream = new InputStreamStub(expectedChunks);
    // when
    final S3ObjectMetadata actualMetadata = s3Connector.uploadFile(uploadedFileInputStream, fileName, contentType);
    // then
    // assert upload request - initiation
    final ArgumentCaptor<InitiateMultipartUploadRequest> initiateRequestArgCaptor = ArgumentCaptor.forClass(InitiateMultipartUploadRequest.class);
    then(s3).should().initiateMultipartUpload(initiateRequestArgCaptor.capture());
    final InitiateMultipartUploadRequest initRequest = initiateRequestArgCaptor.getValue();
    assertThat("Upload to expected S3 bucket is initiated.", initRequest.getBucketName(), is(bucketName));
    assertThat("Upload is initiated with expected object key.", initRequest.getKey(), is(objectKey));
    // assert upload request - both chunks (full and partial) of the upload itself
    final ArgumentCaptor<UploadPartRequest> uploadPartRequestArgCaptor = ArgumentCaptor.forClass(UploadPartRequest.class);
    then(s3).should(times(2)).uploadPart(uploadPartRequestArgCaptor.capture());
    final List<UploadPartRequest> actualUploadRequests = uploadPartRequestArgCaptor.getAllValues();
    for (int i = 0; i < expectedChunks.length; i++) {
        final byte[] expectedChunk = expectedChunks[i];
        final UploadPartRequest uploadPartRequest = actualUploadRequests.get(i);
        assertThat("Chunk " + i + " uploaded to correct bucket", uploadPartRequest.getBucketName(), is(bucketName));
        assertThat("Chunk " + i + " uploaded with correct object key", uploadPartRequest.getKey(), is(objectKey));
        assertThat("Chunk " + i + " uploaded with correct upload id", uploadPartRequest.getUploadId(), is(uploadId));
        assertThat("Chunk " + i + " uploaded with correct part number", uploadPartRequest.getPartNumber(), is(i + 1));
        assertThat("Chunk " + i + " uploaded with correct part size", uploadPartRequest.getPartSize(), is(Long.valueOf(expectedChunk.length)));
        final byte[] actualChunk = new byte[expectedChunk.length];
        try (final InputStream uploadPartRequestInputStream = uploadPartRequest.getInputStream()) {
            IOUtils.read(uploadPartRequestInputStream, actualChunk, 0, actualChunk.length);
        }
        assertThat("Chunk " + i + " uploaded with correct content", actualChunk, is(expectedChunk));
    }
    // assert upload request - finalisation
    final ArgumentCaptor<CompleteMultipartUploadRequest> completeRequestArgCaptor = ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class);
    then(s3).should().completeMultipartUpload(completeRequestArgCaptor.capture());
    final CompleteMultipartUploadRequest completeMultipartUploadRequest = completeRequestArgCaptor.getValue();
    assertThat("Complete upload request with correct bucket name", completeMultipartUploadRequest.getBucketName(), is(bucketName));
    assertThat("Complete upload request with correct object key", completeMultipartUploadRequest.getKey(), is(objectKey));
    assertThat("Complete upload request with correct upload id", completeMultipartUploadRequest.getUploadId(), is(uploadId));
    assertThat("Complete upload request with correct upload id", completeMultipartUploadRequest.getPartETags(), is(asList(firstPartETag, finalPartETag)));
    // assert upload response metadata
    then(s3).should().getObjectMetadata(bucketName, objectKey);
    assertNewResourceMetadata(contentType, s3ObjectUrl, contentLength, objectKey, fileName, actualMetadata);
}

Example 8 with CompleteMultipartUploadRequest

use of software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest in project hippo by NHS-digital-website.

the class S3SdkConnector method uploadFile.

@Override
public S3ObjectMetadata uploadFile(InputStream fileStream, String fileName, String contentType) {
    String objectKey = s3ObjectKeyGenerator.generateObjectKey(fileName);
    reportAction("Uploading S3 resource {}", objectKey);
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentType(contentType);
    // initialise multipart upload
    InitiateMultipartUploadResult initResult = s3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, objectKey, metadata));
    // loop parts
    List<PartETag> partETags;
    try {
        partETags = uploadParts(fileStream, bucketName, objectKey, initResult.getUploadId());
    } catch (Exception ex) {
        final String errorMessage = "Failed to upload file " + objectKey;
        log.error(errorMessage, ex);
        s3.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, objectKey, initResult.getUploadId()));
        throw new RuntimeException(errorMessage, ex);
    }
    // finalise multipart upload
    s3.completeMultipartUpload(new CompleteMultipartUploadRequest(bucketName, objectKey, initResult.getUploadId(), partETags));
    // The above put request returns metadata object but it's empty,
    // hence the need for a separate call to fetch actual metadata.
    ObjectMetadata resultMetadata = s3.getObjectMetadata(bucketName, objectKey);
    reportAction("S3 resource uploaded {}", objectKey);
    return new S3ObjectMetadataImpl(resultMetadata, bucketName, objectKey);
}

Example 9 with CompleteMultipartUploadRequest

use of software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest in project bender by Nextdoor.

the class S3TransportFactory method close.

@Override
public void close() {
    Exception e = null;
    for (MultiPartUpload upload : this.pendingMultiPartUploads.values()) {
        CompleteMultipartUploadRequest req = upload.getCompleteMultipartUploadRequest();
        try {
            this.client.completeMultipartUpload(req);
        } catch (AmazonS3Exception ex) {
            String errorMessage = String.format("failed to complete multi-part upload for key %s, " + "uploadId %s, and partCount %s. All remaining uploads were aborted as a result.", upload.getKey(), upload.getPartCount(), upload.getUploadId());
            logger.error(errorMessage, ex);
            abortAllMultiPartUploads();
            e = ex;
            break;
        }
    }
    this.pendingMultiPartUploads.clear();
    if (e != null) {
        throw new RuntimeException(e);
    }
}

Example 10 with CompleteMultipartUploadRequest

use of software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest in project data-transfer-project by google.

the class BackblazeDataTransferClient method uploadFileUsingMultipartUpload.

private String uploadFileUsingMultipartUpload(String fileKey, File file, long contentLength) throws IOException, AwsServiceException, SdkClientException {
    List<CompletedPart> completedParts = new ArrayList<>();
    CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(bucketName).key(fileKey).build();
    CreateMultipartUploadResponse createMultipartUploadResponse = s3Client.createMultipartUpload(createMultipartUploadRequest);
    long filePosition = 0;
    try (InputStream fileInputStream = new FileInputStream(file)) {
        for (int i = 1; filePosition < contentLength; i++) {
            // Because the last part could be smaller than others, adjust the part size as needed
            long partSize = Math.min(partSizeForMultiPartUpload, (contentLength - filePosition));
            UploadPartRequest uploadRequest = UploadPartRequest.builder().bucket(bucketName).key(fileKey).uploadId(createMultipartUploadResponse.uploadId()).partNumber(i).build();
            RequestBody requestBody = RequestBody.fromInputStream(fileInputStream, partSize);
            UploadPartResponse uploadPartResponse = s3Client.uploadPart(uploadRequest, requestBody);
            completedParts.add(CompletedPart.builder().partNumber(i).eTag(uploadPartResponse.eTag()).build());
            filePosition += partSize;
        }
    }
    CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder().bucket(bucketName).key(fileKey).uploadId(createMultipartUploadResponse.uploadId()).multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()).build();
    CompleteMultipartUploadResponse completeMultipartUploadResponse = s3Client.completeMultipartUpload(completeMultipartUploadRequest);
    return completeMultipartUploadResponse.versionId();
}