Examples with UploadPartRequest software.amazon.awssdk.services.s3.model.UploadPartRequest used on opensource projects

Example 51 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project bender by Nextdoor.

the class S3TransporterTest method testAmazonClientException.

@Test(expected = TransportException.class)
public void testAmazonClientException() throws TransportException, IllegalStateException, IOException {
    /*
     * Create mock client, requets, and replies
     */
    AmazonS3Client mockClient = mock(AmazonS3Client.class);
    UploadPartResult uploadResult = new UploadPartResult();
    uploadResult.setETag("foo");
    doThrow(new AmazonClientException("expected")).when(mockClient).uploadPart(any(UploadPartRequest.class));
    InitiateMultipartUploadResult initUploadResult = new InitiateMultipartUploadResult();
    initUploadResult.setUploadId("123");
    doReturn(initUploadResult).when(mockClient).initiateMultipartUpload(any(InitiateMultipartUploadRequest.class));
    /*
     * Fill buffer with mock data
     */
    S3TransportBuffer buffer = new S3TransportBuffer(1000, false, new S3TransportSerializer());
    InternalEvent mockIevent = mock(InternalEvent.class);
    doReturn("foo").when(mockIevent).getSerialized();
    /*
     * Create transport
     */
    Map<String, MultiPartUpload> multiPartUploads = new HashMap<String, MultiPartUpload>(0);
    S3Transport transport = new S3Transport(mockClient, "bucket", "basepath", false, multiPartUploads);
    /*
     * Do actual test
     */
    buffer.add(mockIevent);
    LinkedHashMap<String, String> partitions = new LinkedHashMap<String, String>();
    partitions.put(S3Transport.FILENAME_KEY, "a_filename");
    ArgumentCaptor<UploadPartRequest> argument = ArgumentCaptor.forClass(UploadPartRequest.class);
    try {
        transport.sendBatch(buffer, partitions, new TestContext());
    } catch (Exception e) {
        assertEquals(e.getCause().getClass(), AmazonClientException.class);
        throw e;
    }
}

Example 52 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project bender by Nextdoor.

the class S3TransporterTest method testGzFilename.

@Test
public void testGzFilename() throws TransportException, IllegalStateException, IOException {
    /*
     * Create mock client, requests, and replies
     */
    AmazonS3Client mockClient = getMockClient();
    /*
     * Fill buffer with mock data
     */
    S3TransportBuffer buffer = new S3TransportBuffer(1000, true, new S3TransportSerializer());
    InternalEvent mockIevent = mock(InternalEvent.class);
    doReturn("foo").when(mockIevent).getSerialized();
    /*
     * Create transport
     */
    Map<String, MultiPartUpload> multiPartUploads = new HashMap<String, MultiPartUpload>(0);
    S3Transport transport = new S3Transport(mockClient, "bucket", "basepath/", true, multiPartUploads);
    /*
     * Do actual test
     */
    buffer.add(mockIevent);
    LinkedHashMap<String, String> partitions = new LinkedHashMap<String, String>();
    partitions.put(S3Transport.FILENAME_KEY, "a_filename.gz");
    ArgumentCaptor<UploadPartRequest> argument = ArgumentCaptor.forClass(UploadPartRequest.class);
    transport.sendBatch(buffer, partitions, new TestContext());
    verify(mockClient).uploadPart(argument.capture());
    /*
     * Check results
     */
    assertEquals("basepath/a_filename.bz2", argument.getValue().getKey());
}

Example 53 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project Singularity by HubSpot.

the class SingularityS3Uploader method multipartUpload.

private void multipartUpload(AmazonS3 s3Client, String key, File file, ObjectMetadata objectMetadata, Optional<StorageClass> maybeStorageClass) throws Exception {
    List<PartETag> partETags = new ArrayList<>();
    InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, key, objectMetadata);
    maybeStorageClass.ifPresent(initRequest::setStorageClass);
    InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
    long contentLength = file.length();
    long partSize = configuration.getUploadPartSize();
    try {
        long filePosition = 0;
        for (int i = 1; filePosition < contentLength; i++) {
            partSize = Math.min(partSize, (contentLength - filePosition));
            UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(bucketName).withKey(key).withUploadId(initResponse.getUploadId()).withPartNumber(i).withFileOffset(filePosition).withFile(file).withPartSize(partSize);
            partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
            filePosition += partSize;
        }
        CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(bucketName, key, initResponse.getUploadId(), partETags);
        s3Client.completeMultipartUpload(completeRequest);
    } catch (Exception e) {
        s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, key, initResponse.getUploadId()));
        throw new RuntimeException(e);
    }
}

Example 54 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project nexus-public by sonatype.

the class ProducerConsumerUploader method submitPartUploads.

private List<PartETag> submitPartUploads(final InputStream input, final String bucket, final String key, final String uploadId, final AmazonS3 s3) throws IOException, InterruptedException {
    BlockingQueue<PartETag> tags = new LinkedBlockingQueue<>();
    ChunkReader parallelReader = new ChunkReader(input, readChunk);
    Optional<Chunk> optionalChunk;
    int chunkCount = 0;
    while ((optionalChunk = parallelReader.readChunk(chunkSize)).isPresent()) {
        Chunk chunk = optionalChunk.get();
        chunkCount++;
        UploadPartRequest request = buildRequest(bucket, key, uploadId, chunk);
        waitingRequests.put(new UploadBundle(s3, request, tags));
    }
    List<PartETag> partETags = new ArrayList<>(chunkCount);
    for (int idx = 0; idx < chunkCount; idx++) {
        PartETag partETag = tags.take();
        if (partETag == POISON_TAG) {
            throw new CancellationException("Part upload failed");
        } else {
            partETags.add(partETag);
        }
    }
    return partETags;
}

Example 55 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project aws-sdk-java-v2 by aws.

the class S3PresignerIntegrationTest method completeMultipartUpload_CanBePresigned.

@Test
public void completeMultipartUpload_CanBePresigned() throws IOException {
    String objectKey = generateRandomObjectKey();
    S3TestUtils.addCleanupTask(S3PresignerIntegrationTest.class, () -> client.deleteObject(r -> r.bucket(testBucket).key(objectKey)));
    CreateMultipartUploadResponse create = client.createMultipartUpload(createMultipartUploadRequest(objectKey));
    S3TestUtils.addCleanupTask(S3PresignerIntegrationTest.class, () -> client.abortMultipartUpload(abortMultipartUploadRequest(objectKey, create.uploadId())));
    UploadPartResponse uploadPartResponse = client.uploadPart(uploadPartRequest(objectKey, create), RequestBody.fromString(testObjectContent));
    String etag = uploadPartResponse.eTag();
    PresignedCompleteMultipartUploadRequest presignedRequest = presigner.presignCompleteMultipartUpload(r -> r.signatureDuration(Duration.ofDays(1)).completeMultipartUploadRequest(createMultipartUploadRequest(objectKey, create, etag)));
    assertThat(execute(presignedRequest, presignedRequest.signedPayload().get().asUtf8String()).httpResponse().isSuccessful()).isTrue();
    String content = client.getObjectAsBytes(r -> r.bucket(testBucket).key(objectKey)).asUtf8String();
    assertThat(content).isEqualTo(testObjectContent);
}