use of com.hubspot.singularity.s3uploader.config.SingularityS3UploaderContentHeaders in project Singularity by HubSpot.
the class SingularityGCSUploader method uploadSingle.
@Override
protected void uploadSingle(int sequence, Path file) throws Exception {
Retryer<Boolean> retryer = RetryerBuilder.<Boolean>newBuilder().retryIfExceptionOfType(RuntimeException.class).retryIfRuntimeException().withWaitStrategy(WaitStrategies.fixedWait(configuration.getRetryWaitMs(), TimeUnit.MILLISECONDS)).withStopStrategy(StopStrategies.stopAfterAttempt(configuration.getRetryCount())).build();
retryer.call(() -> {
final long start = System.currentTimeMillis();
final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname);
long fileSizeBytes = Files.size(file);
LOG.info("{} Uploading {} to {}/{} (size {})", logIdentifier, file, bucketName, key, fileSizeBytes);
BlobInfo.Builder blobInfoBuilder = BlobInfo.newBuilder(bucketName, key);
UploaderFileAttributes fileAttributes = getFileAttributes(file);
Map<String, String> metadata = new HashMap<>();
if (fileAttributes.getStartTime().isPresent()) {
metadata.put(SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get().toString());
LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get());
}
if (fileAttributes.getEndTime().isPresent()) {
metadata.put(SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getEndTime().get().toString());
LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get());
}
blobInfoBuilder.setMetadata(metadata);
for (SingularityS3UploaderContentHeaders contentHeaders : configuration.getS3ContentHeaders()) {
if (file.toString().endsWith(contentHeaders.getFilenameEndsWith())) {
LOG.debug("{} Using content headers {} for file {}", logIdentifier, contentHeaders, file);
if (contentHeaders.getContentType().isPresent()) {
blobInfoBuilder.setContentType(contentHeaders.getContentType().get());
}
if (contentHeaders.getContentEncoding().isPresent()) {
blobInfoBuilder.setContentEncoding(contentHeaders.getContentEncoding().get());
}
break;
}
}
if (shouldApplyStorageClass(fileSizeBytes, uploadMetadata.getGcsStorageClass())) {
LOG.debug("{} adding storage class {} to {}", logIdentifier, uploadMetadata.getGcsStorageClass().get(), file);
blobInfoBuilder.setStorageClass(StorageClass.valueOf(uploadMetadata.getGcsStorageClass().get()));
}
try (FileInputStream fileInputStream = new FileInputStream(file.toFile())) {
if (uploadMetadata.getEncryptionKey().isPresent()) {
storage.create(blobInfoBuilder.build(), fileInputStream, BlobWriteOption.encryptionKey(uploadMetadata.getEncryptionKey().get()));
} else {
storage.create(blobInfoBuilder.build(), fileInputStream);
}
LOG.info("{} Uploaded {} in {}", logIdentifier, key, JavaUtils.duration(start));
return true;
} catch (StorageException se) {
LOG.warn("{} Couldn't upload {} due to {}", logIdentifier, file, se.getMessage(), se);
throw se;
} catch (Exception e) {
LOG.warn("Exception uploading {}", file, e);
throw e;
}
});
}
use of com.hubspot.singularity.s3uploader.config.SingularityS3UploaderContentHeaders in project Singularity by HubSpot.
the class SingularityS3Uploader method uploadSingle.
protected void uploadSingle(int sequence, Path file) throws Exception {
Retryer<Boolean> retryer = RetryerBuilder.<Boolean>newBuilder().retryIfExceptionOfType(AmazonS3Exception.class).retryIfRuntimeException().withWaitStrategy(WaitStrategies.fixedWait(configuration.getRetryWaitMs(), TimeUnit.MILLISECONDS)).withStopStrategy(StopStrategies.stopAfterAttempt(configuration.getRetryCount())).build();
retryer.call(() -> {
final long start = System.currentTimeMillis();
final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname);
long fileSizeBytes = Files.size(file);
LOG.info("{} Uploading {} to {}/{} (size {})", logIdentifier, file, bucketName, key, fileSizeBytes);
try {
ObjectMetadata objectMetadata = new ObjectMetadata();
UploaderFileAttributes fileAttributes = getFileAttributes(file);
if (fileAttributes.getStartTime().isPresent()) {
objectMetadata.addUserMetadata(SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get().toString());
LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get());
}
if (fileAttributes.getEndTime().isPresent()) {
objectMetadata.addUserMetadata(SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get().toString());
LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get());
}
for (SingularityS3UploaderContentHeaders contentHeaders : configuration.getS3ContentHeaders()) {
if (file.toString().endsWith(contentHeaders.getFilenameEndsWith())) {
LOG.debug("{} Using content headers {} for file {}", logIdentifier, contentHeaders, file);
if (contentHeaders.getContentType().isPresent()) {
objectMetadata.setContentType(contentHeaders.getContentType().get());
}
if (contentHeaders.getContentEncoding().isPresent()) {
objectMetadata.setContentEncoding(contentHeaders.getContentEncoding().get());
}
break;
}
}
Optional<StorageClass> maybeStorageClass = Optional.absent();
if (shouldApplyStorageClass(fileSizeBytes, uploadMetadata.getS3StorageClass())) {
LOG.debug("{} adding storage class {} to {}", logIdentifier, uploadMetadata.getS3StorageClass().get(), file);
maybeStorageClass = Optional.of(StorageClass.fromValue(uploadMetadata.getS3StorageClass().get()));
}
LOG.debug("Uploading object with metadata {}", objectMetadata);
if (fileSizeBytes > configuration.getMaxSingleUploadSizeBytes()) {
multipartUpload(key, file.toFile(), objectMetadata, maybeStorageClass);
} else {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, file.toFile()).withMetadata(objectMetadata);
if (maybeStorageClass.isPresent()) {
putObjectRequest.setStorageClass(maybeStorageClass.get());
}
if (uploadMetadata.getEncryptionKey().isPresent()) {
putObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(uploadMetadata.getEncryptionKey().get()));
}
s3Client.putObject(putObjectRequest);
}
} catch (AmazonS3Exception se) {
LOG.warn("{} Couldn't upload {} due to {} - {}", logIdentifier, file, se.getErrorCode(), se.getErrorMessage(), se);
throw se;
} catch (Exception e) {
LOG.warn("Exception uploading {}", file, e);
throw e;
}
LOG.info("{} Uploaded {} in {}", logIdentifier, key, JavaUtils.duration(start));
return true;
});
}
Aggregations