Examples with SingularityS3UploaderContentHeaders com.hubspot.singularity.s3uploader.config.SingularityS3UploaderContentHeaders used on opensource projects

Example 1 with SingularityS3UploaderContentHeaders

use of com.hubspot.singularity.s3uploader.config.SingularityS3UploaderContentHeaders in project Singularity by HubSpot.

the class SingularityGCSUploader method uploadSingle.

@Override
protected void uploadSingle(int sequence, Path file) throws Exception {
    Retryer<Boolean> retryer = RetryerBuilder.<Boolean>newBuilder().retryIfExceptionOfType(RuntimeException.class).retryIfRuntimeException().withWaitStrategy(WaitStrategies.fixedWait(configuration.getRetryWaitMs(), TimeUnit.MILLISECONDS)).withStopStrategy(StopStrategies.stopAfterAttempt(configuration.getRetryCount())).build();
    retryer.call(() -> {
        final long start = System.currentTimeMillis();
        final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname);
        long fileSizeBytes = Files.size(file);
        LOG.info("{} Uploading {} to {}/{} (size {})", logIdentifier, file, bucketName, key, fileSizeBytes);
        BlobInfo.Builder blobInfoBuilder = BlobInfo.newBuilder(bucketName, key);
        UploaderFileAttributes fileAttributes = getFileAttributes(file);
        Map<String, String> metadata = new HashMap<>();
        if (fileAttributes.getStartTime().isPresent()) {
            metadata.put(SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get().toString());
            LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get());
        }
        if (fileAttributes.getEndTime().isPresent()) {
            metadata.put(SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getEndTime().get().toString());
            LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get());
        }
        blobInfoBuilder.setMetadata(metadata);
        for (SingularityS3UploaderContentHeaders contentHeaders : configuration.getS3ContentHeaders()) {
            if (file.toString().endsWith(contentHeaders.getFilenameEndsWith())) {
                LOG.debug("{} Using content headers {} for file {}", logIdentifier, contentHeaders, file);
                if (contentHeaders.getContentType().isPresent()) {
                    blobInfoBuilder.setContentType(contentHeaders.getContentType().get());
                }
                if (contentHeaders.getContentEncoding().isPresent()) {
                    blobInfoBuilder.setContentEncoding(contentHeaders.getContentEncoding().get());
                }
                break;
            }
        }
        if (shouldApplyStorageClass(fileSizeBytes, uploadMetadata.getGcsStorageClass())) {
            LOG.debug("{} adding storage class {} to {}", logIdentifier, uploadMetadata.getGcsStorageClass().get(), file);
            blobInfoBuilder.setStorageClass(StorageClass.valueOf(uploadMetadata.getGcsStorageClass().get()));
        }
        try (FileInputStream fileInputStream = new FileInputStream(file.toFile())) {
            if (uploadMetadata.getEncryptionKey().isPresent()) {
                storage.create(blobInfoBuilder.build(), fileInputStream, BlobWriteOption.encryptionKey(uploadMetadata.getEncryptionKey().get()));
            } else {
                storage.create(blobInfoBuilder.build(), fileInputStream);
            }
            LOG.info("{} Uploaded {} in {}", logIdentifier, key, JavaUtils.duration(start));
            return true;
        } catch (StorageException se) {
            LOG.warn("{} Couldn't upload {} due to  {}", logIdentifier, file, se.getMessage(), se);
            throw se;
        } catch (Exception e) {
            LOG.warn("Exception uploading {}", file, e);
            throw e;
        }
    });
}

Example 2 with SingularityS3UploaderContentHeaders

use of com.hubspot.singularity.s3uploader.config.SingularityS3UploaderContentHeaders in project Singularity by HubSpot.

the class SingularityS3Uploader method uploadSingle.

protected void uploadSingle(int sequence, Path file) throws Exception {
    Retryer<Boolean> retryer = RetryerBuilder.<Boolean>newBuilder().retryIfExceptionOfType(AmazonS3Exception.class).retryIfRuntimeException().withWaitStrategy(WaitStrategies.fixedWait(configuration.getRetryWaitMs(), TimeUnit.MILLISECONDS)).withStopStrategy(StopStrategies.stopAfterAttempt(configuration.getRetryCount())).build();
    retryer.call(() -> {
        final long start = System.currentTimeMillis();
        final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname);
        long fileSizeBytes = Files.size(file);
        LOG.info("{} Uploading {} to {}/{} (size {})", logIdentifier, file, bucketName, key, fileSizeBytes);
        try {
            ObjectMetadata objectMetadata = new ObjectMetadata();
            UploaderFileAttributes fileAttributes = getFileAttributes(file);
            if (fileAttributes.getStartTime().isPresent()) {
                objectMetadata.addUserMetadata(SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get().toString());
                LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_START_S3_ATTR, fileAttributes.getStartTime().get());
            }
            if (fileAttributes.getEndTime().isPresent()) {
                objectMetadata.addUserMetadata(SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get().toString());
                LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_END_S3_ATTR, fileAttributes.getEndTime().get());
            }
            for (SingularityS3UploaderContentHeaders contentHeaders : configuration.getS3ContentHeaders()) {
                if (file.toString().endsWith(contentHeaders.getFilenameEndsWith())) {
                    LOG.debug("{} Using content headers {} for file {}", logIdentifier, contentHeaders, file);
                    if (contentHeaders.getContentType().isPresent()) {
                        objectMetadata.setContentType(contentHeaders.getContentType().get());
                    }
                    if (contentHeaders.getContentEncoding().isPresent()) {
                        objectMetadata.setContentEncoding(contentHeaders.getContentEncoding().get());
                    }
                    break;
                }
            }
            Optional<StorageClass> maybeStorageClass = Optional.absent();
            if (shouldApplyStorageClass(fileSizeBytes, uploadMetadata.getS3StorageClass())) {
                LOG.debug("{} adding storage class {} to {}", logIdentifier, uploadMetadata.getS3StorageClass().get(), file);
                maybeStorageClass = Optional.of(StorageClass.fromValue(uploadMetadata.getS3StorageClass().get()));
            }
            LOG.debug("Uploading object with metadata {}", objectMetadata);
            if (fileSizeBytes > configuration.getMaxSingleUploadSizeBytes()) {
                multipartUpload(key, file.toFile(), objectMetadata, maybeStorageClass);
            } else {
                PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, file.toFile()).withMetadata(objectMetadata);
                if (maybeStorageClass.isPresent()) {
                    putObjectRequest.setStorageClass(maybeStorageClass.get());
                }
                if (uploadMetadata.getEncryptionKey().isPresent()) {
                    putObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(uploadMetadata.getEncryptionKey().get()));
                }
                s3Client.putObject(putObjectRequest);
            }
        } catch (AmazonS3Exception se) {
            LOG.warn("{} Couldn't upload {} due to {} - {}", logIdentifier, file, se.getErrorCode(), se.getErrorMessage(), se);
            throw se;
        } catch (Exception e) {
            LOG.warn("Exception uploading {}", file, e);
            throw e;
        }
        LOG.info("{} Uploaded {} in {}", logIdentifier, key, JavaUtils.duration(start));
        return true;
    });
}