Examples with UploadResult com.amazonaws.services.s3.transfer.model.UploadResult used on opensource projects

Example 1 with UploadResult

use of com.amazonaws.services.s3.transfer.model.UploadResult in project aws-doc-sdk-examples by awsdocs.

the class LowLevelMultipartUpload method main.

public static void main(String[] args) throws IOException {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String bucketName = "*** Bucket name ***";
    String keyName = "*** Key name ***";
    String filePath = "*** Path to file to upload ***";
    File file = new File(filePath);
    long contentLength = file.length();
    // Set part size to 5 MB.
    long partSize = 5 * 1024 * 1024;
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(clientRegion).withCredentials(new ProfileCredentialsProvider()).build();
        // Create a list of ETag objects. You retrieve ETags for each object part uploaded,
        // then, after each individual part has been uploaded, pass the list of ETags to
        // the request to complete the upload.
        List<PartETag> partETags = new ArrayList<PartETag>();
        // Initiate the multipart upload.
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName);
        InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
        // Upload the file parts.
        long filePosition = 0;
        for (int i = 1; filePosition < contentLength; i++) {
            // Because the last part could be less than 5 MB, adjust the part size as needed.
            partSize = Math.min(partSize, (contentLength - filePosition));
            // Create the request to upload a part.
            UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucketName).withKey(keyName).withUploadId(initResponse.getUploadId()).withPartNumber(i).withFileOffset(filePosition).withFile(file).withPartSize(partSize);
            // Upload the part and add the response's ETag to our list.
            UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest);
            partETags.add(uploadResult.getPartETag());
            filePosition += partSize;
        }
        // Complete the multipart upload.
        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName, initResponse.getUploadId(), partETags);
        s3Client.completeMultipartUpload(compRequest);
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client
        // couldn't parse the response from Amazon S3.
        e.printStackTrace();
    }
}

Example 2 with UploadResult

use of com.amazonaws.services.s3.transfer.model.UploadResult in project alluxio by Alluxio.

the class S3ALowLevelOutputStreamTest method mockS3ClientAndExecutor.

/**
 * Mocks the S3 client and executor.
 */
private void mockS3ClientAndExecutor() throws Exception {
    mMockS3Client = PowerMockito.mock(AmazonS3.class);
    InitiateMultipartUploadResult initResult = new InitiateMultipartUploadResult();
    when(mMockS3Client.initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(initResult);
    initResult.setUploadId(UPLOAD_ID);
    when(mMockS3Client.uploadPart(any(UploadPartRequest.class))).thenAnswer((InvocationOnMock invocation) -> {
        Object[] args = invocation.getArguments();
        UploadPartResult uploadResult = new UploadPartResult();
        uploadResult.setPartNumber(((UploadPartRequest) args[0]).getPartNumber());
        return uploadResult;
    });
    when(mMockS3Client.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))).thenReturn(new CompleteMultipartUploadResult());
    mMockTag = (ListenableFuture<PartETag>) PowerMockito.mock(ListenableFuture.class);
    when(mMockTag.get()).thenReturn(new PartETag(1, "someTag"));
    mMockExecutor = Mockito.mock(ListeningExecutorService.class);
    when(mMockExecutor.submit(any(Callable.class))).thenReturn(mMockTag);
}

Example 3 with UploadResult

use of com.amazonaws.services.s3.transfer.model.UploadResult in project bender by Nextdoor.

the class S3TransporterTest method getMockClient.

private AmazonS3Client getMockClient() {
    AmazonS3Client mockClient = spy(AmazonS3Client.class);
    UploadPartResult uploadResult = new UploadPartResult();
    uploadResult.setETag("foo");
    doReturn(uploadResult).when(mockClient).uploadPart(any(UploadPartRequest.class));
    InitiateMultipartUploadResult initUploadResult = new InitiateMultipartUploadResult();
    initUploadResult.setUploadId("123");
    doReturn(initUploadResult).when(mockClient).initiateMultipartUpload(any(InitiateMultipartUploadRequest.class));
    return mockClient;
}

Example 4 with UploadResult

use of com.amazonaws.services.s3.transfer.model.UploadResult in project camel by apache.

the class S3Producer method processMultiPart.

public void processMultiPart(final Exchange exchange) throws Exception {
    File filePayload = null;
    Object obj = exchange.getIn().getMandatoryBody();
    // Need to check if the message body is WrappedFile
    if (obj instanceof WrappedFile) {
        obj = ((WrappedFile<?>) obj).getFile();
    }
    if (obj instanceof File) {
        filePayload = (File) obj;
    } else {
        throw new InvalidArgumentException("aws-s3: MultiPart upload requires a File input.");
    }
    ObjectMetadata objectMetadata = determineMetadata(exchange);
    if (objectMetadata.getContentLength() == 0) {
        objectMetadata.setContentLength(filePayload.length());
    }
    final String keyName = determineKey(exchange);
    final InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(getConfiguration().getBucketName(), keyName, objectMetadata);
    String storageClass = determineStorageClass(exchange);
    if (storageClass != null) {
        initRequest.setStorageClass(StorageClass.fromValue(storageClass));
    }
    String cannedAcl = exchange.getIn().getHeader(S3Constants.CANNED_ACL, String.class);
    if (cannedAcl != null) {
        CannedAccessControlList objectAcl = CannedAccessControlList.valueOf(cannedAcl);
        initRequest.setCannedACL(objectAcl);
    }
    AccessControlList acl = exchange.getIn().getHeader(S3Constants.ACL, AccessControlList.class);
    if (acl != null) {
        // note: if cannedacl and acl are both specified the last one will be used. refer to
        // PutObjectRequest#setAccessControlList for more details
        initRequest.setAccessControlList(acl);
    }
    LOG.trace("Initiating multipart upload [{}] from exchange [{}]...", initRequest, exchange);
    final InitiateMultipartUploadResult initResponse = getEndpoint().getS3Client().initiateMultipartUpload(initRequest);
    final long contentLength = objectMetadata.getContentLength();
    final List<PartETag> partETags = new ArrayList<PartETag>();
    long partSize = getConfiguration().getPartSize();
    CompleteMultipartUploadResult uploadResult = null;
    long filePosition = 0;
    try {
        for (int part = 1; filePosition < contentLength; part++) {
            partSize = Math.min(partSize, contentLength - filePosition);
            UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(getConfiguration().getBucketName()).withKey(keyName).withUploadId(initResponse.getUploadId()).withPartNumber(part).withFileOffset(filePosition).withFile(filePayload).withPartSize(partSize);
            LOG.trace("Uploading part [{}] for {}", part, keyName);
            partETags.add(getEndpoint().getS3Client().uploadPart(uploadRequest).getPartETag());
            filePosition += partSize;
        }
        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(getConfiguration().getBucketName(), keyName, initResponse.getUploadId(), partETags);
        uploadResult = getEndpoint().getS3Client().completeMultipartUpload(compRequest);
    } catch (Exception e) {
        getEndpoint().getS3Client().abortMultipartUpload(new AbortMultipartUploadRequest(getConfiguration().getBucketName(), keyName, initResponse.getUploadId()));
        throw e;
    }
    Message message = getMessageForResponse(exchange);
    message.setHeader(S3Constants.E_TAG, uploadResult.getETag());
    if (uploadResult.getVersionId() != null) {
        message.setHeader(S3Constants.VERSION_ID, uploadResult.getVersionId());
    }
    if (getConfiguration().isDeleteAfterWrite() && filePayload != null) {
        FileUtil.deleteFile(filePayload);
    }
}

Example 5 with UploadResult

use of com.amazonaws.services.s3.transfer.model.UploadResult in project sandbox by irof.

the class S3MultipartUploadTest method 高レベルAPIでマルチパートアップロードする.

@Test
public void 高レベルAPIでマルチパートアップロードする() throws Exception {
    TransferManager transferManager = new TransferManager();
    TransferManagerConfiguration configuration = new TransferManagerConfiguration();
    // パートのサイズ: 5MB-5GB, default: 5MB
    configuration.setMinimumUploadPartSize(5 * Constants.MB);
    // 閾値: デフォルト16MB
    configuration.setMultipartUploadThreshold(5 * Constants.MB);
    transferManager.setConfiguration(configuration);
    // アップロードするデータを雑に用意（閾値より大きいの）
    byte[] bytes = new byte[5 * Constants.MB + 1];
    // 渡すものは通常のputObjectと同じ
    ObjectMetadata meta = new ObjectMetadata();
    meta.setContentLength(bytes.length);
    PutObjectRequest request = new PutObjectRequest("irof-sandbox", "S3MultipartUpload/hi-level-api.dat", new ByteArrayInputStream(bytes), meta);
    // アップロード実行
    Upload upload = transferManager.upload(request);
    // 完了まで待つ
    UploadResult result = upload.waitForUploadResult();
    // ETagは末尾に "-2" がつく
    logger.info(result.getETag());
    assertTrue(result.getETag().endsWith("-2"));
}