Examples with TransferManager com.amazonaws.services.s3.transfer.TransferManager used on opensource projects

Example 11 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project jackrabbit-oak by apache.

the class DataStoreUtils method deleteBucket.

public static void deleteBucket(String bucket, Map<String, ?> map, Date date) throws Exception {
    log.info("cleaning bucket [" + bucket + "]");
    Properties props = new Properties();
    props.putAll(map);
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, date);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        log.info("bucket [ " + bucket + "] cleaned");
    } else {
        log.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}

Example 12 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project hadoop by apache.

the class S3AFileSystem method initTransferManager.

private void initTransferManager() {
    TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
    transferConfiguration.setMinimumUploadPartSize(partSize);
    transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
    transferConfiguration.setMultipartCopyPartSize(partSize);
    transferConfiguration.setMultipartCopyThreshold(multiPartThreshold);
    transfers = new TransferManager(s3, unboundedThreadPool);
    transfers.setConfiguration(transferConfiguration);
}

Example 13 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.

the class XferMgrProgress method uploadDirWithSubprogress.

public static void uploadDirWithSubprogress(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) {
    System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : ""));
    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive);
        // loop with Transfer.isDone()
        XferMgrProgress.showMultiUploadProgress(multi_upload);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(multi_upload);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}

Example 14 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.

the class XferMgrUpload method uploadFile.

public static void uploadFile(String file_path, String bucket_name, String key_prefix, boolean pause) {
    System.out.println("file: " + file_path + (pause ? " (pause)" : ""));
    String key_name = null;
    if (key_prefix != null) {
        key_name = key_prefix + '/' + file_path;
    } else {
        key_name = file_path;
    }
    File f = new File(file_path);
    TransferManager xfer_mgr = new TransferManager();
    try {
        Upload xfer = xfer_mgr.upload(bucket_name, key_name, f);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        //  or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}

Example 15 with TransferManager

use of com.amazonaws.services.s3.transfer.TransferManager in project aws-doc-sdk-examples by awsdocs.

the class XferMgrUpload method uploadDir.

public static void uploadDir(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) {
    System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : ""));
    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}