Examples with UploadPartRequest software.amazon.awssdk.services.s3.model.UploadPartRequest used on opensource projects

Example 1 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project elasticsearch by elastic.

the class DefaultS3OutputStream method doUploadMultipart.

protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is, int length, boolean lastPart) throws AmazonS3Exception {
    UploadPartRequest request = new UploadPartRequest().withBucketName(bucketName).withKey(blobName).withUploadId(uploadId).withPartNumber(multipartChunks).withInputStream(is).withPartSize(length).withLastPart(lastPart);
    UploadPartResult response = blobStore.client().uploadPart(request);
    return response.getPartETag();
}

Example 2 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project hippo by NHS-digital-website.

the class S3ConnectorImpl method uploadParts.

private List<PartETag> uploadParts(final InputStream fileStream, final String bucketName, final String objectKey, final String uploadId) throws IOException {
    final List<PartETag> partETags = new ArrayList<>();
    int lastBytesReadCount = 0;
    int processedBytesCount = 0;
    int currentPartNumber = 0;
    while (lastBytesReadCount >= 0) {
        final byte[] buffer = new byte[BUFFER_SIZE];
        lastBytesReadCount = fileStream.read(buffer, 0, BUFFER_SIZE);
        if (lastBytesReadCount >= 0) {
            processedBytesCount += lastBytesReadCount;
            currentPartNumber++;
            try (ByteArrayInputStream partStream = new ByteArrayInputStream(buffer, 0, lastBytesReadCount)) {
                UploadPartRequest req = new UploadPartRequest().withBucketName(bucketName).withKey(objectKey).withUploadId(uploadId).withInputStream(partStream).withPartNumber(currentPartNumber).withPartSize(lastBytesReadCount);
                partETags.add(s3.uploadPart(req).getPartETag());
            } catch (Exception ex) {
                log.error("Error processing upload part {} for object {} with upload id {}", currentPartNumber, objectKey, uploadId);
                throw ex;
            }
        }
    }
    log.info("Uploaded {} bytes in {} parts for {}", processedBytesCount, currentPartNumber, objectKey);
    return partETags;
}

Example 3 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project bender by Nextdoor.

the class S3TransporterTest method testUnpartitioned.

@Test
public void testUnpartitioned() throws TransportException, IllegalStateException, IOException {
    /*
     * Create mock client, requets, and replies
     */
    AmazonS3Client mockClient = getMockClient();
    /*
     * Fill buffer with mock data
     */
    S3TransportBuffer buffer = new S3TransportBuffer(1000, false, new S3TransportSerializer());
    InternalEvent mockIevent = mock(InternalEvent.class);
    doReturn("foo").when(mockIevent).getSerialized();
    /*
     * Create transport
     */
    Map<String, MultiPartUpload> multiPartUploads = new HashMap<String, MultiPartUpload>(0);
    S3Transport transport = new S3Transport(mockClient, "bucket", "basepath", false, multiPartUploads);
    /*
     * Do actual test
     */
    buffer.add(mockIevent);
    LinkedHashMap<String, String> partitions = new LinkedHashMap<String, String>();
    partitions.put(S3Transport.FILENAME_KEY, "a_filename");
    ArgumentCaptor<UploadPartRequest> argument = ArgumentCaptor.forClass(UploadPartRequest.class);
    transport.sendBatch(buffer, partitions, new TestContext());
    verify(mockClient).uploadPart(argument.capture());
    /*
     * Check results
     */
    assertEquals("bucket", argument.getValue().getBucketName());
    assertEquals("basepath/a_filename", argument.getValue().getKey());
    assertEquals(1, argument.getValue().getPartNumber());
    // foo\n
    assertEquals(4, argument.getValue().getPartSize());
    assertEquals("123", argument.getValue().getUploadId());
}

Example 4 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project bender by Nextdoor.

the class S3TransporterTest method testCompressed.

@Test
public void testCompressed() throws TransportException, IllegalStateException, IOException {
    /*
     * Create mock client, requets, and replies
     */
    AmazonS3Client mockClient = getMockClient();
    /*
     * Capture the InputStream into a ByteArrayOutputStream before the Transport thread closes the
     * InputStream and makes it unavailable for reading.
     */
    ByteArrayOutputStream captured = new ByteArrayOutputStream();
    Answer answer = new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            UploadPartRequest req = invocation.getArgumentAt(0, UploadPartRequest.class);
            captured.write(req.getInputStream());
            return new UploadPartResult();
        }
    };
    Mockito.doAnswer(answer).when(mockClient).uploadPart(any(UploadPartRequest.class));
    /*
     * Fill buffer with mock data
     */
    S3TransportBuffer buffer = new S3TransportBuffer(1000, false, new S3TransportSerializer());
    InternalEvent mockIevent = mock(InternalEvent.class);
    doReturn("foo").when(mockIevent).getSerialized();
    /*
     * Create transport
     */
    Map<String, MultiPartUpload> multiPartUploads = new HashMap<String, MultiPartUpload>(0);
    S3Transport transport = new S3Transport(mockClient, "bucket", "basepath", true, multiPartUploads);
    /*
     * Do actual test
     */
    buffer.add(mockIevent);
    LinkedHashMap<String, String> partitions = new LinkedHashMap<String, String>();
    partitions.put(S3Transport.FILENAME_KEY, "a_filename");
    ArgumentCaptor<UploadPartRequest> argument = ArgumentCaptor.forClass(UploadPartRequest.class);
    buffer.close();
    transport.sendBatch(buffer, partitions, new TestContext());
    verify(mockClient).uploadPart(argument.capture());
    /*
     * Check results
     */
    assertEquals("bucket", argument.getValue().getBucketName());
    assertEquals("basepath/a_filename.bz2", argument.getValue().getKey());
    assertEquals(1, argument.getValue().getPartNumber());
    assertEquals(40, argument.getValue().getPartSize());
    assertEquals("123", argument.getValue().getUploadId());
    /*
     * Convert the actual InputStream from the client into a ByteArrayOutputStream which can be read
     * and verified.
     */
    byte[] actualBytes = captured.toByteArray();
    byte[] expectedBytes = { 66, 90, 104, 57, 49, 65, 89, 38, 83, 89, 118, -10, -77, -27, 0, 0, 0, -63, 0, 0, 16, 1, 0, -96, 0, 48, -52, 12, -62, 12, 46, -28, -118, 112, -95, 32, -19, -19, 103, -54 };
    assertArrayEquals(expectedBytes, actualBytes);
}

Example 5 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project bender by Nextdoor.

the class S3TransporterTest method testPartitioned.

@Test
public void testPartitioned() throws TransportException, IllegalStateException, IOException {
    /*
     * Create mock client, requets, and replies
     */
    AmazonS3Client mockClient = getMockClient();
    /*
     * Fill buffer with mock data
     */
    S3TransportBuffer buffer = new S3TransportBuffer(1000, false, new S3TransportSerializer());
    InternalEvent mockIevent = mock(InternalEvent.class);
    doReturn("foo").when(mockIevent).getSerialized();
    /*
     * Create transport
     */
    Map<String, MultiPartUpload> multiPartUploads = new HashMap<String, MultiPartUpload>(0);
    S3Transport transport = new S3Transport(mockClient, "bucket", "basepath", false, multiPartUploads);
    /*
     * Do actual test
     */
    buffer.add(mockIevent);
    LinkedHashMap<String, String> partitions = new LinkedHashMap<String, String>();
    partitions.put(S3Transport.FILENAME_KEY, "a_filename");
    partitions.put("day", "01");
    partitions.put("hour", "23");
    ArgumentCaptor<UploadPartRequest> argument = ArgumentCaptor.forClass(UploadPartRequest.class);
    transport.sendBatch(buffer, partitions, new TestContext());
    verify(mockClient).uploadPart(argument.capture());
    /*
     * Check results
     */
    assertEquals("bucket", argument.getValue().getBucketName());
    assertEquals("basepath/day=01/hour=23/a_filename", argument.getValue().getKey());
    assertEquals(1, argument.getValue().getPartNumber());
    // foo\n
    assertEquals(4, argument.getValue().getPartSize());
    assertEquals("123", argument.getValue().getUploadId());
}