Examples with TransferManager com.amazonaws.services.s3.transfer.TransferManager used on opensource projects

Example 21 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project hadoop by apache.

the class S3AFileSystem method initTransferManager.

private void initTransferManager() {
    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
    transferConfiguration.setMinimumUploadPartSize(partSize);
    transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
    transferConfiguration.setMultipartCopyPartSize(partSize);
    transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
    transfers = new TransferManager(s3, unboundedThreadPool);
    transfers.setConfiguration(transferConfiguration);
}

Example 22 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project sandbox by irof.

the class S3MultipartUploadTest method 高レベルAPIでマルチパートアップロードする.

@Test
public void 高レベルAPIでマルチパートアップロードする() throws Exception {
    TransferManager transferManager = new TransferManager();
    TransferManagerConfiguration configuration = new TransferManagerConfiguration();
    // パートのサイズ: 5MB-5GB, default: 5MB
    configuration.setMinimumUploadPartSize(5 * Constants.MB);
    // 閾値: デフォルト16MB
    configuration.setMultipartUploadThreshold(5 * Constants.MB);
    transferManager.setConfiguration(configuration);
    // アップロードするデータを雑に用意（閾値より大きいの）
    byte[] bytes = new byte[5 * Constants.MB + 1];
    // 渡すものは通常のputObjectと同じ
    ObjectMetadata meta = new ObjectMetadata();
    meta.setContentLength(bytes.length);
    PutObjectRequest request = new PutObjectRequest("irof-sandbox", "S3MultipartUpload/hi-level-api.dat", new ByteArrayInputStream(bytes), meta);
    // アップロード実行
    Upload upload = transferManager.upload(request);
    // 完了まで待つ
    UploadResult result = upload.waitForUploadResult();
    // ETagは末尾に "-2" がつく
    logger.info(result.getETag());
    assertTrue(result.getETag().endsWith("-2"));
}

Example 23 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project jackrabbit-oak by apache.

the class DataStoreUtils method deleteBucket.

public static void deleteBucket(String bucket, Map<String, ?> map, Date date) throws Exception {
    log.info("cleaning bucket [" + bucket + "]");
    Properties props = new Properties();
    props.putAll(map);
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, date);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        log.info("bucket [ " + bucket + "] cleaned");
    } else {
        log.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}

Example 24 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project herd by FINRAOS.

the class MockS3OperationsImpl method upload.

@Override
public Upload upload(PutObjectRequest putObjectRequest, TransferManager transferManager) {
    LOGGER.debug("upload(): putObjectRequest.getBucketName() = " + putObjectRequest.getBucketName() + ", putObjectRequest.getKey() = " + putObjectRequest.getKey());
    putObject(putObjectRequest, transferManager.getAmazonS3Client());
    long contentLength = putObjectRequest.getFile().length();
    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(contentLength);
    progress.updateProgress(contentLength);
    UploadImpl upload = new UploadImpl(null, progress, null, null);
    upload.setState(TransferState.Completed);
    return upload;
}

Example 25 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project herd by FINRAOS.

the class MockS3OperationsImpl method copyFile.

/**
 * {@inheritDoc} <p/> <p> This implementation simulates a copyFile operation. </p> <p> This method copies files in-memory. </p> <p> The result {@link Copy}
 * has the following properties: <dl> <p/> <dt>description</dt> <dd>"MockTransfer"</dd> <p/> <dt>state</dt> <dd>{@link TransferState#Completed}</dd> <p/>
 * <dt>transferProgress.totalBytesToTransfer</dt> <dd>1024</dd> <p/> <dt>transferProgress.updateProgress</dt> <dd>1024</dd> <p/> </dl> <p/> All other
 * properties are set as default. </p> <p> This operation takes the following hints when suffixed in copyObjectRequest.sourceKey: <dl> <p/>
 * <dt>MOCK_S3_FILE_NAME_SERVICE_EXCEPTION</dt> <dd>Throws a AmazonServiceException</dd> <p/> </dl> </p>
 */
@Override
public Copy copyFile(final CopyObjectRequest copyObjectRequest, TransferManager transferManager) {
    LOGGER.debug("copyFile(): copyObjectRequest.getSourceBucketName() = " + copyObjectRequest.getSourceBucketName() + ", copyObjectRequest.getSourceKey() = " + copyObjectRequest.getSourceKey() + ", copyObjectRequest.getDestinationBucketName() = " + copyObjectRequest.getDestinationBucketName() + ", copyObjectRequest.getDestinationKey() = " + copyObjectRequest.getDestinationKey());
    if (copyObjectRequest.getSourceKey().endsWith(MOCK_S3_FILE_NAME_SERVICE_EXCEPTION)) {
        throw new AmazonServiceException(null);
    }
    String sourceBucketName = copyObjectRequest.getSourceBucketName();
    String sourceKey = copyObjectRequest.getSourceKey();
    MockS3Bucket mockSourceS3Bucket = getOrCreateBucket(sourceBucketName);
    MockS3Object mockSourceS3Object = mockSourceS3Bucket.getObjects().get(sourceKey);
    if (mockSourceS3Object == null) {
        AmazonServiceException amazonServiceException = new AmazonServiceException(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        amazonServiceException.setErrorCode(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        throw amazonServiceException;
    }
    // Set the result CopyImpl and TransferProgress.
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(mockSourceS3Object.getObjectMetadata().getContentLength());
    transferProgress.updateProgress(mockSourceS3Object.getObjectMetadata().getContentLength());
    CopyImpl copy = new CopyImpl(MOCK_TRANSFER_DESCRIPTION, transferProgress, null, null);
    copy.setState(TransferState.Completed);
    // If an invalid KMS Id was passed in, mark the transfer as failed and return an exception via the transfer monitor.
    if (copyObjectRequest.getSSEAwsKeyManagementParams() != null) {
        final String kmsId = copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId();
        if (kmsId.startsWith(MOCK_KMS_ID_FAILED_TRANSFER)) {
            copy.setState(TransferState.Failed);
            copy.setMonitor(new TransferMonitor() {

                @Override
                public Future<?> getFuture() {
                    if (!kmsId.equals(MOCK_KMS_ID_FAILED_TRANSFER_NO_EXCEPTION)) {
                        throw new AmazonServiceException("Key '" + copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId() + "' does not exist (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: 1234567890123456)");
                    }
                    // We don't want an exception to be thrown so return a basic future that won't throw an exception.
                    BasicFuture<?> future = new BasicFuture<Void>(null);
                    future.completed(null);
                    return future;
                }

                @Override
                public boolean isDone() {
                    return true;
                }
            });
        } else if (kmsId.startsWith(MOCK_KMS_ID_CANCELED_TRANSFER)) {
            // If the KMS indicates a cancelled transfer, just update the state to canceled.
            copy.setState(TransferState.Canceled);
        }
    }
    // If copy operation is marked as completed, perform the actual file copy in memory.
    if (copy.getState().equals(TransferState.Completed)) {
        String destinationBucketName = copyObjectRequest.getDestinationBucketName();
        String destinationObjectKey = copyObjectRequest.getDestinationKey();
        String destinationObjectVersion = MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED.equals(destinationBucketName) ? UUID.randomUUID().toString() : null;
        String destinationObjectKeyVersion = destinationObjectKey + (destinationObjectVersion != null ? destinationObjectVersion : "");
        ObjectMetadata objectMetadata = copyObjectRequest.getNewObjectMetadata();
        MockS3Object mockDestinationS3Object = new MockS3Object();
        mockDestinationS3Object.setKey(destinationObjectKey);
        mockDestinationS3Object.setVersion(destinationObjectVersion);
        mockDestinationS3Object.setData(Arrays.copyOf(mockSourceS3Object.getData(), mockSourceS3Object.getData().length));
        mockDestinationS3Object.setObjectMetadata(objectMetadata);
        MockS3Bucket mockDestinationS3Bucket = getOrCreateBucket(destinationBucketName);
        mockDestinationS3Bucket.getObjects().put(destinationObjectKey, mockDestinationS3Object);
        mockDestinationS3Bucket.getVersions().put(destinationObjectKeyVersion, mockDestinationS3Object);
    }
    return copy;
}