Examples with MultipartUpload software.amazon.awssdk.services.s3.model.MultipartUpload used on opensource projects

Example 11 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project aws-doc-sdk-examples by awsdocs.

the class S3ObjectOperations method main.

public static void main(String[] args) throws IOException {
    final String USAGE = "\n" + "Usage:\n" + "    <bucketName> <key>\n\n" + "Where:\n" + "    bucketName - the Amazon S3 bucket to create.\n\n" + "    key - the key to use.\n\n";
    if (args.length != 2) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucketName = args[0];
    String key = args[1];
    // snippet-start:[s3.java2.s3_object_operations.upload]
    Region region = Region.US_WEST_2;
    s3 = S3Client.builder().region(region).build();
    createBucket(s3, bucketName, region);
    PutObjectRequest objectRequest = PutObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.putObject(objectRequest, RequestBody.fromByteBuffer(getRandomByteBuffer(10_000)));
    // snippet-end:[s3.java2.s3_object_operations.upload]
    // Multipart upload example
    String multipartKey = "multiPartKey";
    multipartUpload(bucketName, multipartKey);
    // snippet-start:[s3.java2.s3_object_operations.pagination]
    ListObjectsV2Request listObjectsReqManual = ListObjectsV2Request.builder().bucket(bucketName).maxKeys(1).build();
    boolean done = false;
    while (!done) {
        ListObjectsV2Response listObjResponse = s3.listObjectsV2(listObjectsReqManual);
        for (S3Object content : listObjResponse.contents()) {
            System.out.println(content.key());
        }
        if (listObjResponse.nextContinuationToken() == null) {
            done = true;
        }
        listObjectsReqManual = listObjectsReqManual.toBuilder().continuationToken(listObjResponse.nextContinuationToken()).build();
    }
    // snippet-end:[s3.java2.s3_object_operations.pagination]
    // snippet-start:[s3.java2.s3_object_operations.iterative]
    ListObjectsV2Request listReq = ListObjectsV2Request.builder().bucket(bucketName).maxKeys(1).build();
    ListObjectsV2Iterable listRes = s3.listObjectsV2Paginator(listReq);
    // Process response pages
    listRes.stream().flatMap(r -> r.contents().stream()).forEach(content -> System.out.println(" Key: " + content.key() + " size = " + content.size()));
    // snippet-end:[s3.java2.s3_object_operations.iterative]
    // snippet-start:[s3.java2.s3_object_operations.stream]
    // Helper method to work with paginated collection of items directly
    listRes.contents().stream().forEach(content -> System.out.println(" Key: " + content.key() + " size = " + content.size()));
    // snippet-start:[s3.java2.s3_object_operations.forloop]
    for (S3Object content : listRes.contents()) {
        System.out.println(" Key: " + content.key() + " size = " + content.size());
    }
    // snippet-end:[s3.java2.s3_object_operations.forloop]
    // snippet-start:[s3.java2.s3_object_operations.download]
    GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.getObject(getObjectRequest);
    // snippet-end:[s3.java2.s3_object_operations.download]
    // snippet-start:[s3.java2.s3_object_operations.delete]
    DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName).key(key).build();
    s3.deleteObject(deleteObjectRequest);
    // snippet-end:[s3.java2.s3_object_operations.delete]
    // Delete an object
    deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName).key(multipartKey).build();
    s3.deleteObject(deleteObjectRequest);
    deleteBucket(s3, bucketName);
    System.out.println("Done");
}

Example 12 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project aws-doc-sdk-examples by awsdocs.

the class S3Scenario method main.

public static void main(String[] args) throws IOException {
    final String usage = "\n" + "Usage:\n" + "    <bucketName> <key> <objectPath> <savePath> <toBucket>\n\n" + "Where:\n" + "    bucketName - the Amazon S3 bucket to create.\n\n" + "    key - the key to use.\n\n" + "    objectPath - the path where the file is located (for example, C:/AWS/book2.pdf). " + "    savePath - the path where the file is saved after it's downloaded (for example, C:/AWS/book2.pdf). " + "    toBucket - an Amazon S3 bucket to where an object is copied to (for example, C:/AWS/book2.pdf). ";
    if (args.length != 5) {
        System.out.println(usage);
        System.exit(1);
    }
    String bucketName = args[0];
    String key = args[1];
    String objectPath = args[2];
    String savePath = args[3];
    String toBucket = args[4];
    Region region = Region.US_EAST_1;
    S3Client s3 = S3Client.builder().region(region).build();
    // Create an Amazon S3 bucket.
    createBucket(s3, bucketName);
    // Update a local file to the Amazon S3 bucket.
    uploadLocalFile(s3, bucketName, key, objectPath);
    // Download the object to another local file.
    getObjectBytes(s3, bucketName, key, savePath);
    // Perform a multipart upload.
    String multipartKey = "multiPartKey";
    multipartUpload(s3, toBucket, multipartKey);
    // List all objects located in the Amazon S3 bucket.
    // Show 2 ways
    listAllObjects(s3, bucketName);
    anotherListExample(s3, bucketName);
    // Copy the object to another Amazon S3 bucket
    copyBucketObject(s3, bucketName, key, toBucket);
    // Delete the object from the Amazon S3 bucket.
    deleteObjectFromBucket(s3, bucketName, key);
    // Delete the Amazon S3 bucket
    deleteBucket(s3, bucketName);
    System.out.println("All Amazon S3 operations were successfully performed");
    s3.close();
}

Example 13 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project aws-doc-sdk-examples by awsdocs.

the class AbortMultipartUpload method abortUploads.

// snippet-start:[s3.java2.abort_upload.main]
public static void abortUploads(S3Client s3, String bucketName, String accountId) {
    try {
        ListMultipartUploadsRequest listMultipartUploadsRequest = ListMultipartUploadsRequest.builder().bucket(bucketName).build();
        ListMultipartUploadsResponse response = s3.listMultipartUploads(listMultipartUploadsRequest);
        List<MultipartUpload> uploads = response.uploads();
        AbortMultipartUploadRequest abortMultipartUploadRequest = null;
        for (MultipartUpload upload : uploads) {
            abortMultipartUploadRequest = AbortMultipartUploadRequest.builder().bucket(bucketName).key(upload.key()).expectedBucketOwner(accountId).uploadId(upload.uploadId()).build();
            s3.abortMultipartUpload(abortMultipartUploadRequest);
        }
    } catch (S3Exception e) {
        System.err.println(e.getMessage());
        System.exit(1);
    }
}

Example 14 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project aws-doc-sdk-examples by awsdocs.

the class ListMultipartUploads method main.

public static void main(String[] args) {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String bucketName = "*** Bucket name ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(new ProfileCredentialsProvider()).withRegion(clientRegion).build();
        // Retrieve a list of all in-progress multipart uploads.
        ListMultipartUploadsRequest allMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName);
        MultipartUploadListing multipartUploadListing = s3Client.listMultipartUploads(allMultipartUploadsRequest);
        List<MultipartUpload> uploads = multipartUploadListing.getMultipartUploads();
        // Display information about all in-progress multipart uploads.
        System.out.println(uploads.size() + " multipart upload(s) in progress.");
        for (MultipartUpload u : uploads) {
            System.out.println("Upload in progress: Key = \"" + u.getKey() + "\", id = " + u.getUploadId());
        }
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client
        // couldn't parse the response from Amazon S3.
        e.printStackTrace();
    }
}

Example 15 with MultipartUpload

use of software.amazon.awssdk.services.s3.model.MultipartUpload in project pravega by pravega.

the class S3ChunkStorage method doConcat.

@Override
public int doConcat(ConcatArgument[] chunks) throws ChunkStorageException {
    int totalBytesConcatenated = 0;
    String targetPath = getObjectPath(chunks[0].getName());
    String uploadId = null;
    boolean isCompleted = false;
    try {
        int partNumber = 1;
        val response = client.createMultipartUpload(CreateMultipartUploadRequest.builder().bucket(config.getBucket()).key(targetPath).build());
        uploadId = response.uploadId();
        // check whether the target exists
        if (!checkExists(chunks[0].getName())) {
            throw new ChunkNotFoundException(chunks[0].getName(), "doConcat - Target segment does not exist");
        }
        CompletedPart[] completedParts = new CompletedPart[chunks.length];
        // Copy the parts
        for (int i = 0; i < chunks.length; i++) {
            if (0 != chunks[i].getLength()) {
                val sourceHandle = chunks[i];
                long objectSize = client.headObject(HeadObjectRequest.builder().bucket(this.config.getBucket()).key(getObjectPath(sourceHandle.getName())).build()).contentLength();
                Preconditions.checkState(objectSize >= chunks[i].getLength(), "Length of object should be equal or greater. Length on LTS={} provided={}", objectSize, chunks[i].getLength());
                UploadPartCopyRequest copyRequest = UploadPartCopyRequest.builder().destinationBucket(config.getBucket()).destinationKey(targetPath).sourceBucket(config.getBucket()).sourceKey(getObjectPath(sourceHandle.getName())).uploadId(uploadId).partNumber(partNumber).copySourceRange(getRangeWithLength(0, chunks[i].getLength())).build();
                val copyResult = client.uploadPartCopy(copyRequest);
                val eTag = copyResult.copyPartResult().eTag();
                completedParts[i] = CompletedPart.builder().partNumber(partNumber).eTag(eTag).build();
                partNumber++;
                totalBytesConcatenated += chunks[i].getLength();
            }
        }
        // Close the upload
        CompletedMultipartUpload completedRequest = CompletedMultipartUpload.builder().parts(completedParts).build();
        client.completeMultipartUpload(CompleteMultipartUploadRequest.builder().bucket(config.getBucket()).key(targetPath).multipartUpload(completedRequest).uploadId(uploadId).build());
        isCompleted = true;
    } catch (RuntimeException e) {
        // Error message is REC_CATCH_EXCEPTION: Exception is caught when Exception is not thrown
        throw convertException(chunks[0].getName(), "doConcat", e);
    } catch (Exception e) {
        throw convertException(chunks[0].getName(), "doConcat", e);
    } finally {
        if (!isCompleted && null != uploadId) {
            try {
                client.abortMultipartUpload(AbortMultipartUploadRequest.builder().bucket(config.getBucket()).key(targetPath).uploadId(uploadId).build());
            } catch (Exception e) {
                throw convertException(chunks[0].getName(), "doConcat", e);
            }
        }
    }
    return totalBytesConcatenated;
}