Examples with SSEAwsKeyManagementParams com.amazonaws.services.s3.model.SSEAwsKeyManagementParams used on opensource projects

Example 1 with SSEAwsKeyManagementParams

use of com.amazonaws.services.s3.model.SSEAwsKeyManagementParams in project qpp-conversion-tool by CMSgov.

the class StorageServiceImpl method store.

/**
 * Stores the {@link InputStream} as an object in the S3 bucket.
 *
 * @param keyName The requested key name for the object.
 * @param inStream The {@link InputStream} to write out to an object in S3.
 * @param size The size of the {@link InputStream}.
 * @return A {@link CompletableFuture} that will eventually contain the S3 object key.
 */
@Override
public CompletableFuture<String> store(String keyName, Supplier<InputStream> inStream, long size) {
    final String bucketName = environment.getProperty(Constants.BUCKET_NAME_ENV_VARIABLE);
    final String kmsKey = environment.getProperty(Constants.KMS_KEY_ENV_VARIABLE);
    if (Strings.isNullOrEmpty(bucketName) || Strings.isNullOrEmpty(kmsKey)) {
        API_LOG.warn("No bucket name is specified or no KMS key specified.");
        return CompletableFuture.completedFuture("");
    }
    ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
    s3ObjectMetadata.setContentLength(size);
    Supplier<PutObjectRequest> putObjectRequest = () -> new PutObjectRequest(bucketName, keyName, inStream.get(), s3ObjectMetadata).withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsKey));
    API_LOG.info("Writing object {} to S3 bucket {}", keyName, bucketName);
    return actOnItem(putObjectRequest);
}

Example 2 with SSEAwsKeyManagementParams

use of com.amazonaws.services.s3.model.SSEAwsKeyManagementParams in project Singularity by HubSpot.

the class SingularityS3Uploader method uploadSingle.

protected void uploadSingle(int sequence, Path file) throws Exception {
    Retryer<Boolean> retryer = RetryerBuilder.<Boolean>newBuilder().retryIfExceptionOfType(AmazonS3Exception.class).retryIfRuntimeException().withWaitStrategy(WaitStrategies.fixedWait(configuration.getRetryWaitMs(), TimeUnit.MILLISECONDS)).withStopStrategy(StopStrategies.stopAfterAttempt(configuration.getRetryCount())).build();
    retryer.call(() -> {
        final long start = System.currentTimeMillis();
        final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname);
        long fileSizeBytes = Files.size(file);
        LOG.info("{} Uploading {} to {}/{} (size {})", logIdentifier, file, bucketName, key, fileSizeBytes);
        try {
            ObjectMetadata objectMetadata = new ObjectMetadata();
            UploaderFileAttributes fileAttributes = getFileAttributes(file);
            if (fileAttributes.getStartTime().isPresent()) {
                objectMetadata.addUserMetadata(SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get().toString());
                LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get());
            }
            if (fileAttributes.getEndTime().isPresent()) {
                objectMetadata.addUserMetadata(SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get().toString());
                LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get());
            }
            for (SingularityS3UploaderContentHeaders contentHeaders : configuration.getS3ContentHeaders()) {
                if (file.toString().endsWith(contentHeaders.getFilenameEndsWith())) {
                    LOG.debug("{} Using content headers {} for file {}", logIdentifier, contentHeaders, file);
                    if (contentHeaders.getContentType().isPresent()) {
                        objectMetadata.setContentType(contentHeaders.getContentType().get());
                    }
                    if (contentHeaders.getContentEncoding().isPresent()) {
                        objectMetadata.setContentEncoding(contentHeaders.getContentEncoding().get());
                    }
                    break;
                }
            }
            Optional<StorageClass> maybeStorageClass = Optional.absent();
            if (shouldApplyStorageClass(fileSizeBytes, uploadMetadata.getS3StorageClass())) {
                LOG.debug("{} adding storage class {} to {}", logIdentifier, uploadMetadata.getS3StorageClass().get(), file);
                maybeStorageClass = Optional.of(StorageClass.fromValue(uploadMetadata.getS3StorageClass().get()));
            }
            LOG.debug("Uploading object with metadata {}", objectMetadata);
            if (fileSizeBytes > configuration.getMaxSingleUploadSizeBytes()) {
                multipartUpload(key, file.toFile(), objectMetadata, maybeStorageClass);
            } else {
                PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, file.toFile()).withMetadata(objectMetadata);
                if (maybeStorageClass.isPresent()) {
                    putObjectRequest.setStorageClass(maybeStorageClass.get());
                }
                if (uploadMetadata.getEncryptionKey().isPresent()) {
                    putObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(uploadMetadata.getEncryptionKey().get()));
                }
                s3Client.putObject(putObjectRequest);
            }
        } catch (AmazonS3Exception se) {
            LOG.warn("{} Couldn't upload {} due to {} - {}", logIdentifier, file, se.getErrorCode(), se.getErrorMessage(), se);
            throw se;
        } catch (Exception e) {
            LOG.warn("Exception uploading {}", file, e);
            throw e;
        }
        LOG.info("{} Uploaded {} in {}", logIdentifier, key, JavaUtils.duration(start));
        return true;
    });
}

Example 3 with SSEAwsKeyManagementParams

use of com.amazonaws.services.s3.model.SSEAwsKeyManagementParams in project herd by FINRAOS.

the class S3DaoImpl method copyFile.

@Override
public S3FileTransferResultsDto copyFile(final S3FileCopyRequestParamsDto params) throws InterruptedException {
    LOGGER.info("Copying S3 object... sourceS3Key=\"{}\" sourceS3BucketName=\"{}\" targetS3Key=\"{}\" targetS3BucketName=\"{}\"", params.getSourceObjectKey(), params.getSourceBucketName(), params.getTargetObjectKey(), params.getTargetBucketName());
    // Perform the copy.
    S3FileTransferResultsDto results = performTransfer(params, new Transferer() {

        @Override
        public Transfer performTransfer(TransferManager transferManager) {
            // Create a copy request.
            CopyObjectRequest copyObjectRequest = new CopyObjectRequest(params.getSourceBucketName(), params.getSourceObjectKey(), params.getTargetBucketName(), params.getTargetObjectKey());
            // If KMS Key ID is specified, set the AWS Key Management System parameters to be used to encrypt the object.
            if (StringUtils.isNotBlank(params.getKmsKeyId())) {
                copyObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(params.getKmsKeyId()));
            } else // Otherwise, specify the server-side encryption algorithm for encrypting the object using AWS-managed keys.
            {
                ObjectMetadata metadata = new ObjectMetadata();
                metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
                copyObjectRequest.setNewObjectMetadata(metadata);
            }
            return s3Operations.copyFile(copyObjectRequest, transferManager);
        }
    });
    LOGGER.info("Copied S3 object. sourceS3Key=\"{}\" sourceS3BucketName=\"{}\" targetS3Key=\"{}\" targetS3BucketName=\"{}\" " + "totalBytesTransferred={} transferDuration=\"{}\"", params.getSourceObjectKey(), params.getSourceBucketName(), params.getTargetObjectKey(), params.getTargetBucketName(), results.getTotalBytesTransferred(), HerdDateUtils.formatDuration(results.getDurationMillis()));
    logOverallTransferRate(results);
    return results;
}

Example 4 with SSEAwsKeyManagementParams

use of com.amazonaws.services.s3.model.SSEAwsKeyManagementParams in project components by Talend.

the class S3OutputWriter method close.

/**
 * not sure the method is called one or two times, it depend on the platform
 */
@Override
public Result close() throws IOException {
    if (closed) {
        return result;
    }
    closed = true;
    try {
        if (writer != null) {
            writer.flush();
            writer.close();
        }
        S3DatasetProperties data_set = properties.getDatasetProperties();
        PutObjectRequest request = new PutObjectRequest(data_set.bucket.getValue(), data_set.object.getValue(), data_file);
        Boolean serverSideEnc = data_set.encryptDataAtRest.getValue();
        if (serverSideEnc != null && serverSideEnc) {
            request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(data_set.kmsForDataAtRest.getValue()));
        }
        s3_client.putObject(request);
    } finally {
        writer = null;
        data_file.delete();
        if (s3_client != null) {
            s3_client.shutdown();
            s3_client = null;
        }
    }
    result.successCount = result.totalCount;
    return result;
}