Examples with MultipartUpload software.amazon.awssdk.services.s3.model.MultipartUpload used on opensource projects

Example 16 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project data-transfer-project by google.

the class BackblazeDataTransferClient method uploadFileUsingMultipartUpload.

private String uploadFileUsingMultipartUpload(String fileKey, File file, long contentLength) throws IOException, AwsServiceException, SdkClientException {
    List<CompletedPart> completedParts = new ArrayList<>();
    CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(bucketName).key(fileKey).build();
    CreateMultipartUploadResponse createMultipartUploadResponse = s3Client.createMultipartUpload(createMultipartUploadRequest);
    long filePosition = 0;
    try (InputStream fileInputStream = new FileInputStream(file)) {
        for (int i = 1; filePosition < contentLength; i++) {
            // Because the last part could be smaller than others, adjust the part size as needed
            long partSize = Math.min(partSizeForMultiPartUpload, (contentLength - filePosition));
            UploadPartRequest uploadRequest = UploadPartRequest.builder().bucket(bucketName).key(fileKey).uploadId(createMultipartUploadResponse.uploadId()).partNumber(i).build();
            RequestBody requestBody = RequestBody.fromInputStream(fileInputStream, partSize);
            UploadPartResponse uploadPartResponse = s3Client.uploadPart(uploadRequest, requestBody);
            completedParts.add(CompletedPart.builder().partNumber(i).eTag(uploadPartResponse.eTag()).build());
            filePosition += partSize;
        }
    }
    CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder().bucket(bucketName).key(fileKey).uploadId(createMultipartUploadResponse.uploadId()).multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()).build();
    CompleteMultipartUploadResponse completeMultipartUploadResponse = s3Client.completeMultipartUpload(completeMultipartUploadRequest);
    return completeMultipartUploadResponse.versionId();
}

Example 17 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project nifi by apache.

the class ITPutS3Object method testStateRemove.

@Test
public void testStateRemove() throws IOException {
    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);
    final String bucket = runner.getProcessContext().getProperty(PutS3Object.BUCKET).getValue();
    final String key = runner.getProcessContext().getProperty(PutS3Object.KEY).getValue();
    final String cacheKey = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-sr";
    final List<MultipartUpload> uploadList = new ArrayList<>();
    final MultipartUpload upload1 = new MultipartUpload();
    upload1.setKey(key);
    upload1.setUploadId("1234");
    uploadList.add(upload1);
    final MultipartUploadListing uploadListing = new MultipartUploadListing();
    uploadListing.setMultipartUploads(uploadList);
    final MockAmazonS3Client mockClient = new MockAmazonS3Client();
    mockClient.setListing(uploadListing);
    /*
         * store state, retrieve and validate, remove and validate
         */
    PutS3Object.MultipartState stateOrig = new PutS3Object.MultipartState();
    stateOrig.setUploadId("1234");
    stateOrig.setContentLength(1234L);
    processor.persistLocalState(cacheKey, stateOrig);
    PutS3Object.MultipartState state1 = processor.getLocalStateIfInS3(mockClient, bucket, cacheKey);
    Assert.assertEquals("1234", state1.getUploadId());
    Assert.assertEquals(1234L, state1.getContentLength().longValue());
    processor.persistLocalState(cacheKey, null);
    PutS3Object.MultipartState state2 = processor.getLocalStateIfInS3(mockClient, bucket, cacheKey);
    Assert.assertNull(state2);
}

Example 18 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project herd by FINRAOS.

the class S3DaoImpl method abortMultipartUploads.

@Override
public int abortMultipartUploads(S3FileTransferRequestParamsDto params, Date thresholdDate) {
    // Create an Amazon S3 client.
    AmazonS3Client s3Client = getAmazonS3(params);
    int abortedMultipartUploadsCount = 0;
    try {
        // List upload markers. Null implies initial list request.
        String uploadIdMarker = null;
        String keyMarker = null;
        boolean truncated;
        do {
            // Create the list multipart request, optionally using the last markers.
            ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(params.getS3BucketName());
            request.setUploadIdMarker(uploadIdMarker);
            request.setKeyMarker(keyMarker);
            // Request the multipart upload listing.
            MultipartUploadListing uploadListing = s3Operations.listMultipartUploads(TransferManager.appendSingleObjectUserAgent(request), s3Client);
            for (MultipartUpload upload : uploadListing.getMultipartUploads()) {
                if (upload.getInitiated().compareTo(thresholdDate) < 0) {
                    // Abort the upload.
                    s3Operations.abortMultipartUpload(TransferManager.appendSingleObjectUserAgent(new AbortMultipartUploadRequest(params.getS3BucketName(), upload.getKey(), upload.getUploadId())), s3Client);
                    // Log the information about the aborted multipart upload.
                    LOGGER.info("Aborted S3 multipart upload. s3Key=\"{}\" s3BucketName=\"{}\" s3MultipartUploadInitiatedDate=\"{}\"", upload.getKey(), params.getS3BucketName(), upload.getInitiated());
                    // Increment the counter.
                    abortedMultipartUploadsCount++;
                }
            }
            // Determine whether there are more uploads to list.
            truncated = uploadListing.isTruncated();
            if (truncated) {
                // Record the list markers.
                uploadIdMarker = uploadListing.getNextUploadIdMarker();
                keyMarker = uploadListing.getNextKeyMarker();
            }
        } while (truncated);
    } finally {
        // Shutdown the Amazon S3 client instance to release resources.
        s3Client.shutdown();
    }
    return abortedMultipartUploadsCount;
}

Example 19 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project aws-doc-sdk-examples by awsdocs.

the class ListMultipartUploads method listUploads.

// snippet-start:[s3.java2.list_multi_uploads.main]
public static void listUploads(S3Client s3, String bucketName) {
    try {
        ListMultipartUploadsRequest listMultipartUploadsRequest = ListMultipartUploadsRequest.builder().bucket(bucketName).build();
        ListMultipartUploadsResponse response = s3.listMultipartUploads(listMultipartUploadsRequest);
        List<MultipartUpload> uploads = response.uploads();
        for (MultipartUpload upload : uploads) {
            System.out.println("Upload in progress: Key = \"" + upload.key() + "\", id = " + upload.uploadId());
        }
    } catch (S3Exception e) {
        System.err.println(e.getMessage());
        System.exit(1);
    }
}

Example 20 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project aws-doc-sdk-examples by awsdocs.

the class S3ObjectOperations method multipartUpload.

/**
 * Upload an object in parts
 */
private static void multipartUpload(String bucketName, String key) throws IOException {
    int mB = 1024 * 1024;
    // snippet-start:[s3.java2.s3_object_operations.upload_multi_part]
    // First create a multipart upload and get the upload id
    CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(bucketName).key(key).build();
    CreateMultipartUploadResponse response = s3.createMultipartUpload(createMultipartUploadRequest);
    String uploadId = response.uploadId();
    System.out.println(uploadId);
    // Upload all the different parts of the object
    UploadPartRequest uploadPartRequest1 = UploadPartRequest.builder().bucket(bucketName).key(key).uploadId(uploadId).partNumber(1).build();
    String etag1 = s3.uploadPart(uploadPartRequest1, RequestBody.fromByteBuffer(getRandomByteBuffer(5 * mB))).eTag();
    CompletedPart part1 = CompletedPart.builder().partNumber(1).eTag(etag1).build();
    UploadPartRequest uploadPartRequest2 = UploadPartRequest.builder().bucket(bucketName).key(key).uploadId(uploadId).partNumber(2).build();
    String etag2 = s3.uploadPart(uploadPartRequest2, RequestBody.fromByteBuffer(getRandomByteBuffer(3 * mB))).eTag();
    CompletedPart part2 = CompletedPart.builder().partNumber(2).eTag(etag2).build();
    // Finally call completeMultipartUpload operation to tell S3 to merge all uploaded
    // parts and finish the multipart operation.
    CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(part1, part2).build();
    CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder().bucket(bucketName).key(key).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
    s3.completeMultipartUpload(completeMultipartUploadRequest);
// snippet-end:[s3.java2.s3_object_operations.upload_multi_part]
}