Examples with ServerSideEncryption software.amazon.awssdk.services.s3.model.ServerSideEncryption used on opensource projects

Example 1 with ServerSideEncryption

use of software.amazon.awssdk.services.s3.model.ServerSideEncryption in project uploader by smoketurner.

the class Uploader method upload.

/**
 * Upload a batch to S3
 *
 * @param batch
 *            Batch to upload
 */
public void upload(@Nonnull final Batch batch) {
    batchSize.update(batch.size());
    batchCount.update(batch.getCount());
    final ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String>builder().put("count", String.valueOf(batch.getCount()));
    batch.getCustomerId().ifPresent(id -> builder.put("customer_id", id));
    final Map<String, String> metadata = builder.build();
    final String key;
    if (configuration.getPrefix().isPresent()) {
        key = configuration.getPrefix().get() + "/" + batch.getKey();
    } else {
        key = batch.getKey();
    }
    LOGGER.debug("Customer: {}, S3 key: {}", batch.getCustomerId().orElse(null), key);
    final PutObjectRequest request = PutObjectRequest.builder().bucket(configuration.getBucketName()).key(key).metadata(metadata).contentLength(batch.size()).contentType(MediaType.TEXT_PLAIN).contentEncoding("gzip").serverSideEncryption(ServerSideEncryption.AES256).build();
    final long start = currentTimeProvider.get();
    final CompletableFuture<PutObjectResponse> future = s3.putObject(request, new BatchRequestProvider(batch));
    future.whenComplete((resp, err) -> {
        if (resp != null) {
            final long took = currentTimeProvider.get() - start;
            uploadTime.update(took, TimeUnit.NANOSECONDS);
            successCounter.inc();
            LOGGER.info("Finished uploading \"{}\" ({} events, {} bytes) in {}ms", key, batch.getCount(), batch.size(), (took / NANOS_IN_MILLIS));
        } else {
            failedCounter.inc();
            LOGGER.error(String.format("Failed to upload \"%s\"", key), err);
        }
    });
}

Example 2 with ServerSideEncryption

use of software.amazon.awssdk.services.s3.model.ServerSideEncryption in project beam by apache.

the class S3FileSystem method multipartCopy.

@VisibleForTesting
CompleteMultipartUploadResponse multipartCopy(S3ResourceId sourcePath, S3ResourceId destinationPath, HeadObjectResponse sourceObjectHead) throws SdkServiceException {
    CreateMultipartUploadRequest initiateUploadRequest = CreateMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).storageClass(config.getS3StorageClass()).metadata(sourceObjectHead.metadata()).serverSideEncryption(config.getSSEAlgorithm()).ssekmsKeyId(config.getSSEKMSKeyId()).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
    CreateMultipartUploadResponse createMultipartUploadResponse = s3Client.get().createMultipartUpload(initiateUploadRequest);
    final String uploadId = createMultipartUploadResponse.uploadId();
    List<CompletedPart> completedParts = new ArrayList<>();
    final long objectSize = sourceObjectHead.contentLength();
    CopyPartResult copyPartResult;
    CompletedPart completedPart;
    // without using S3FileSystem.copy in the future
    if (objectSize == 0) {
        final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(sourcePath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(1).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
        copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
        completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
        completedParts.add(completedPart);
    } else {
        long bytePosition = 0;
        Integer uploadBufferSizeBytes = config.getS3UploadBufferSizeBytes();
        // Amazon parts are 1-indexed, not zero-indexed.
        for (int partNumber = 1; bytePosition < objectSize; partNumber++) {
            final UploadPartCopyRequest uploadPartCopyRequest = UploadPartCopyRequest.builder().bucket(sourcePath.getBucket()).key(sourcePath.getKey()).copySource(destinationPath.getBucket() + "/" + sourcePath.getKey()).uploadId(uploadId).partNumber(partNumber).copySourceRange(String.format("bytes=%s-%s", bytePosition, Math.min(objectSize - 1, bytePosition + uploadBufferSizeBytes - 1))).sseCustomerKey(config.getSSECustomerKey().getKey()).sseCustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).copySourceSSECustomerKey(config.getSSECustomerKey().getKey()).copySourceSSECustomerAlgorithm(config.getSSECustomerKey().getAlgorithm()).build();
            copyPartResult = s3Client.get().uploadPartCopy(uploadPartCopyRequest).copyPartResult();
            completedPart = CompletedPart.builder().partNumber(1).eTag(copyPartResult.eTag()).build();
            completedParts.add(completedPart);
            bytePosition += uploadBufferSizeBytes;
        }
    }
    CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
    CompleteMultipartUploadRequest completeUploadRequest = CompleteMultipartUploadRequest.builder().bucket(destinationPath.getBucket()).key(destinationPath.getKey()).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
    return s3Client.get().completeMultipartUpload(completeUploadRequest);
}

Example 3 with ServerSideEncryption

use of software.amazon.awssdk.services.s3.model.ServerSideEncryption in project beam by apache.

the class S3WritableByteChannelTest method write.

private void write(S3Client mockS3Client, Supplier channelSupplier, S3ResourceId path, String sseAlgorithmStr, String sseCustomerKeyMd5, String ssekmsKeyId, long s3UploadBufferSizeBytes, boolean bucketKeyEnabled, boolean writeReadOnlyBuffer) throws IOException {
    CreateMultipartUploadResponse.Builder builder = CreateMultipartUploadResponse.builder().uploadId("upload-id");
    ServerSideEncryption sseAlgorithm = ServerSideEncryption.fromValue(sseAlgorithmStr);
    if (sseAlgorithm != null) {
        builder.serverSideEncryption(sseAlgorithm);
    }
    if (sseCustomerKeyMd5 != null) {
        builder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    if (ssekmsKeyId != null) {
        sseAlgorithm = ServerSideEncryption.AWS_KMS;
        builder.serverSideEncryption(sseAlgorithm);
    }
    builder.bucketKeyEnabled(bucketKeyEnabled);
    CreateMultipartUploadResponse createMultipartUploadResponse = builder.build();
    doReturn(createMultipartUploadResponse).when(mockS3Client).createMultipartUpload(any(CreateMultipartUploadRequest.class));
    CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(path.getBucket()).key(path.getKey()).build();
    CreateMultipartUploadResponse mockCreateMultipartUploadResponse1 = mockS3Client.createMultipartUpload(createMultipartUploadRequest);
    assertEquals(sseAlgorithm, mockCreateMultipartUploadResponse1.serverSideEncryption());
    assertEquals(sseCustomerKeyMd5, mockCreateMultipartUploadResponse1.sseCustomerKeyMD5());
    assertEquals(bucketKeyEnabled, mockCreateMultipartUploadResponse1.bucketKeyEnabled());
    UploadPartResponse.Builder uploadPartResponseBuilder = UploadPartResponse.builder().eTag("etag");
    if (sseCustomerKeyMd5 != null) {
        uploadPartResponseBuilder.sseCustomerKeyMD5(sseCustomerKeyMd5);
    }
    UploadPartResponse response = uploadPartResponseBuilder.build();
    doReturn(response).when(mockS3Client).uploadPart(any(UploadPartRequest.class), any(RequestBody.class));
    UploadPartResponse mockUploadPartResult = mockS3Client.uploadPart(UploadPartRequest.builder().build(), RequestBody.empty());
    assertEquals(sseCustomerKeyMd5, mockUploadPartResult.sseCustomerKeyMD5());
    S3WritableByteChannel channel = channelSupplier.get();
    int contentSize = 34_078_720;
    ByteBuffer uploadContent = ByteBuffer.allocate((int) (contentSize * 2.5));
    for (int i = 0; i < contentSize; i++) {
        uploadContent.put((byte) 0xff);
    }
    uploadContent.flip();
    int uploadedSize = channel.write(writeReadOnlyBuffer ? uploadContent.asReadOnlyBuffer() : uploadContent);
    assertEquals(contentSize, uploadedSize);
    CompleteMultipartUploadResponse completeMultipartUploadResponse = CompleteMultipartUploadResponse.builder().build();
    doReturn(completeMultipartUploadResponse).when(mockS3Client).completeMultipartUpload(any(CompleteMultipartUploadRequest.class));
    channel.close();
    int partQuantity = (int) Math.ceil((double) contentSize / s3UploadBufferSizeBytes) + 1;
    verify(mockS3Client, times(2)).createMultipartUpload((CreateMultipartUploadRequest) isNotNull());
    verify(mockS3Client, times(partQuantity)).uploadPart((UploadPartRequest) isNotNull(), any(RequestBody.class));
    verify(mockS3Client, times(1)).completeMultipartUpload((CompleteMultipartUploadRequest) notNull());
    verifyNoMoreInteractions(mockS3Client);
}