Examples with UploadPartRequest software.amazon.awssdk.services.s3.model.UploadPartRequest used on opensource projects

Example 31 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project aws-sdk-android by aws-amplify.

the class ServerSideEncryptionWithCustomerKeyIntegrationTest method testMultipartUpload.

@Test
public void testMultipartUpload() throws IOException {
    SSECustomerKey serverSideEncryptionKey = new SSECustomerKey(secretKey);
    InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(BUCKET_NAME, KEY).withSSECustomerKey(serverSideEncryptionKey);
    InitiateMultipartUploadResult initResult = s3.initiateMultipartUpload(initRequest);
    assertEquals(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION, initResult.getSSECustomerAlgorithm());
    assertNotNull(initResult.getSSECustomerKeyMd5());
    assertNull(initResult.getSSEAlgorithm());
    String uploadId = initResult.getUploadId();
    List<PartETag> partETags = new ArrayList<PartETag>(PART_NUMBER);
    long offset = 0;
    for (int part = 1; part <= PART_NUMBER; part++) {
        UploadPartRequest uploadPartRequest = new UploadPartRequest().withUploadId(uploadId).withBucketName(BUCKET_NAME).withKey(KEY).withFile(file_multipartUpload).withPartSize(PART_SIZE).withFileOffset(offset).withPartNumber(part).withLastPart(part == PART_NUMBER).withSSECustomerKey(serverSideEncryptionKey);
        UploadPartResult uploadPartResult = s3.uploadPart(uploadPartRequest);
        partETags.add(uploadPartResult.getPartETag());
        assertEquals(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION, uploadPartResult.getSSECustomerAlgorithm());
        assertNotNull(uploadPartResult.getSSECustomerKeyMd5());
        assertNull(uploadPartResult.getSSEAlgorithm());
        offset += PART_SIZE;
    }
    CompleteMultipartUploadResult completeResult = s3.completeMultipartUpload(new CompleteMultipartUploadRequest(BUCKET_NAME, KEY, uploadId, partETags));
    assertEquals(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION, completeResult.getSSECustomerAlgorithm());
    // CompleteMultipartUpload does not return the key hash
    assertNull(completeResult.getSSEAlgorithm());
    // Verify the content of the uploaded object
    GetObjectRequest request = new GetObjectRequest(BUCKET_NAME, KEY).withSSECustomerKey(serverSideEncryptionKey);
    File destination = createTempFile(BUCKET_NAME, KEY);
    s3.getObject(request, destination);
    assertFileEqualsFile(file_multipartUpload, destination);
}

Example 32 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project jmix by jmix-framework.

the class AwsFileStorage method saveStream.

@Override
public FileRef saveStream(String fileName, InputStream inputStream) {
    String fileKey = createFileKey(fileName);
    int s3ChunkSizeBytes = this.chunkSize * 1024;
    try (BufferedInputStream bos = new BufferedInputStream(inputStream, s3ChunkSizeBytes)) {
        S3Client s3Client = s3ClientReference.get();
        int totalSizeBytes = bos.available();
        if (totalSizeBytes == 0) {
            s3Client.putObject(PutObjectRequest.builder().bucket(bucket).key(fileKey).build(), RequestBody.empty());
            return new FileRef(getStorageName(), fileKey, fileName);
        }
        CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(bucket).key(fileKey).build();
        CreateMultipartUploadResponse response = s3Client.createMultipartUpload(createMultipartUploadRequest);
        List<CompletedPart> completedParts = new ArrayList<>();
        for (int partNumber = 1, readBytes = 0; readBytes != totalSizeBytes; partNumber++) {
            byte[] chunkBytes = new byte[Math.min(totalSizeBytes - readBytes, s3ChunkSizeBytes)];
            readBytes += bos.read(chunkBytes);
            UploadPartRequest uploadPartRequest = UploadPartRequest.builder().bucket(bucket).key(fileKey).uploadId(response.uploadId()).partNumber(partNumber).build();
            String eTag = s3Client.uploadPart(uploadPartRequest, RequestBody.fromBytes(chunkBytes)).eTag();
            CompletedPart part = CompletedPart.builder().partNumber(partNumber).eTag(eTag).build();
            completedParts.add(part);
        }
        CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
        CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder().bucket(bucket).key(fileKey).uploadId(response.uploadId()).multipartUpload(completedMultipartUpload).build();
        s3Client.completeMultipartUpload(completeMultipartUploadRequest);
        return new FileRef(getStorageName(), fileKey, fileName);
    } catch (IOException | SdkException e) {
        log.error("Error saving file to S3 storage", e);
        String message = String.format("Could not save file %s.", fileName);
        throw new FileStorageException(FileStorageException.Type.IO_EXCEPTION, message);
    }
}

Example 33 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project trino by trinodb.

the class TestTrinoS3FileSystem method testStreamingUpload.

@Test
public void testStreamingUpload() throws Exception {
    Configuration config = new Configuration(false);
    config.set(S3_STREAMING_UPLOAD_ENABLED, "true");
    config.set(S3_STREAMING_UPLOAD_PART_SIZE, "10");
    try (TrinoS3FileSystem fs = new TrinoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        String expectedBucketName = "test-bucket";
        fs.initialize(new URI("s3n://" + expectedBucketName + "/"), config);
        fs.setS3Client(s3);
        try (FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test"))) {
            stream.write('a');
            stream.write("foo".repeat(2).getBytes(US_ASCII));
            stream.write("bar".repeat(3).getBytes(US_ASCII));
            stream.write("orange".repeat(4).getBytes(US_ASCII), 6, 12);
        }
        List<UploadPartRequest> parts = s3.getUploadParts();
        assertThat(parts).size().isEqualTo(3);
        InputStream concatInputStream = parts.stream().map(UploadPartRequest::getInputStream).reduce(new ByteArrayInputStream(new byte[0]), SequenceInputStream::new);
        String data = new String(toByteArray(concatInputStream), US_ASCII);
        assertEquals(data, "afoofoobarbarbarorangeorange");
    }
}

Example 34 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project formkiq-core by formkiq.

the class S3Service method presignPostUrl.

/**
 * Generate a S3 Signed Url for creating an object using POST request.
 *
 * @param bucket {@link String}
 * @param key {@link String}
 * @param duration {@link Duration}
 * @param contentLength {@link Optional} {@link Long}
 * @return {@link URL}
 */
public URL presignPostUrl(final String bucket, final String key, final Duration duration, final Optional<Long> contentLength) {
    try (S3Presigner signer = this.builder.buildPresigner()) {
        UploadPartRequest.Builder uploadBuilder = UploadPartRequest.builder().bucket(bucket).key(key);
        if (contentLength.isPresent()) {
            uploadBuilder = uploadBuilder.contentLength(contentLength.get());
        }
        UploadPartPresignRequest prereq = UploadPartPresignRequest.builder().signatureDuration(duration).uploadPartRequest(uploadBuilder.build()).build();
        PresignedUploadPartRequest req = signer.presignUploadPart(prereq);
        return req.url();
    }
}

Example 35 with UploadPartRequest

use of software.amazon.awssdk.services.s3.model.UploadPartRequest in project android-simpl3r by jgilfelt.

the class Uploader method start.

/**
 * Initiate a multipart file upload to Amazon S3
 *
 * @return the URL of a successfully uploaded file
 */
public String start() {
    // initialize
    List<PartETag> partETags = new ArrayList<PartETag>();
    final long contentLength = file.length();
    long filePosition = 0;
    int startPartNumber = 1;
    userInterrupted = false;
    userAborted = false;
    bytesUploaded = 0;
    // check if we can resume an incomplete download
    String uploadId = getCachedUploadId();
    if (uploadId != null) {
        // we can resume the download
        Log.i(TAG, "resuming upload for " + uploadId);
        // get the cached etags
        List<PartETag> cachedEtags = getCachedPartEtags();
        partETags.addAll(cachedEtags);
        // calculate the start position for resume
        startPartNumber = cachedEtags.size() + 1;
        filePosition = (startPartNumber - 1) * partSize;
        bytesUploaded = filePosition;
        Log.i(TAG, "resuming at part " + startPartNumber + " position " + filePosition);
    } else {
        // initiate a new multi part upload
        Log.i(TAG, "initiating new upload");
        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(s3bucketName, s3key);
        configureInitiateRequest(initRequest);
        InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
        uploadId = initResponse.getUploadId();
    }
    final AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest(s3bucketName, s3key, uploadId);
    for (int k = startPartNumber; filePosition < contentLength; k++) {
        long thisPartSize = Math.min(partSize, (contentLength - filePosition));
        Log.i(TAG, "starting file part " + k + " with size " + thisPartSize);
        UploadPartRequest uploadRequest = new UploadPartRequest().withBucketName(s3bucketName).withKey(s3key).withUploadId(uploadId).withPartNumber(k).withFileOffset(filePosition).withFile(file).withPartSize(thisPartSize);
        ProgressListener s3progressListener = new ProgressListener() {

            public void progressChanged(ProgressEvent progressEvent) {
                // TODO calling shutdown too brute force?
                if (userInterrupted) {
                    s3Client.shutdown();
                    throw new UploadIterruptedException("User interrupted");
                } else if (userAborted) {
                    // aborted requests cannot be resumed, so clear any cached etags
                    clearProgressCache();
                    s3Client.abortMultipartUpload(abortRequest);
                    s3Client.shutdown();
                }
                bytesUploaded += progressEvent.getBytesTransfered();
                // Log.d(TAG, "bytesUploaded=" + bytesUploaded);
                // broadcast progress
                float fpercent = ((bytesUploaded * 100) / contentLength);
                int percent = Math.round(fpercent);
                if (progressListener != null) {
                    progressListener.progressChanged(progressEvent, bytesUploaded, percent);
                }
            }
        };
        uploadRequest.setProgressListener(s3progressListener);
        UploadPartResult result = s3Client.uploadPart(uploadRequest);
        partETags.add(result.getPartETag());
        // cache the part progress for this upload
        if (k == 1) {
            initProgressCache(uploadId);
        }
        // store part etag
        cachePartEtag(result);
        filePosition += thisPartSize;
    }
    CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(s3bucketName, s3key, uploadId, partETags);
    CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(compRequest);
    bytesUploaded = 0;
    Log.i(TAG, "upload complete for " + uploadId);
    clearProgressCache();
    return result.getLocation();
}