use of com.amazonaws.services.s3.transfer.TransferManager in project jackrabbit-oak by apache.
the class DataStoreUtils method deleteBucket.
public static void deleteBucket(String bucket, Map<String, ?> map, Date date) throws Exception {
log.info("cleaning bucket [" + bucket + "]");
Properties props = new Properties();
props.putAll(map);
AmazonS3Client s3service = Utils.openService(props);
TransferManager tmx = new TransferManager(s3service);
if (s3service.doesBucketExist(bucket)) {
for (int i = 0; i < 4; i++) {
tmx.abortMultipartUploads(bucket, date);
ObjectListing prevObjectListing = s3service.listObjects(bucket);
while (prevObjectListing != null) {
List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
}
if (deleteList.size() > 0) {
DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
delObjsReq.setKeys(deleteList);
s3service.deleteObjects(delObjsReq);
}
if (!prevObjectListing.isTruncated())
break;
prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
}
}
s3service.deleteBucket(bucket);
log.info("bucket [ " + bucket + "] cleaned");
} else {
log.info("bucket [" + bucket + "] doesn't exists");
}
tmx.shutdownNow();
s3service.shutdown();
}
use of com.amazonaws.services.s3.transfer.TransferManager in project hadoop by apache.
the class S3AFileSystem method initTransferManager.
private void initTransferManager() {
TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
transferConfiguration.setMinimumUploadPartSize(partSize);
transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
transferConfiguration.setMultipartCopyPartSize(partSize);
transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
transfers = new TransferManager(s3, unboundedThreadPool);
transfers.setConfiguration(transferConfiguration);
}
use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.
the class XferMgrProgress method uploadDirWithSubprogress.
public static void uploadDirWithSubprogress(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) {
System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : ""));
TransferManager xfer_mgr = new TransferManager();
try {
MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive);
// loop with Transfer.isDone()
XferMgrProgress.showMultiUploadProgress(multi_upload);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(multi_upload);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
}
use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.
the class XferMgrUpload method uploadFile.
public static void uploadFile(String file_path, String bucket_name, String key_prefix, boolean pause) {
System.out.println("file: " + file_path + (pause ? " (pause)" : ""));
String key_name = null;
if (key_prefix != null) {
key_name = key_prefix + '/' + file_path;
} else {
key_name = file_path;
}
File f = new File(file_path);
TransferManager xfer_mgr = new TransferManager();
try {
Upload xfer = xfer_mgr.upload(bucket_name, key_name, f);
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
}
use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.
the class XferMgrUpload method uploadDir.
public static void uploadDir(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) {
System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : ""));
TransferManager xfer_mgr = new TransferManager();
try {
MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive);
// loop with Transfer.isDone()
XferMgrProgress.showTransferProgress(xfer);
// or block with Transfer.waitForCompletion()
XferMgrProgress.waitForCompletion(xfer);
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
}
xfer_mgr.shutdownNow();
}
Aggregations