Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 41 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit by apache.

the class TestS3Ds method deleteBucket.

public void deleteBucket(String bucket) throws Exception {
    LOG.info("deleting bucket [" + bucket + "]");
    Properties props = Utils.readConfig(config);
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, startTime);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        LOG.info("bucket [ " + bucket + "] deleted");
    } else {
        LOG.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}

Example 42 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit by apache.

the class Utils method deleteBucket.

/**
 * Delete S3 bucket. This method first deletes all objects from bucket and
 * then delete empty bucket.
 *
 * @param bucketName the bucket name.
 */
public static void deleteBucket(final String bucketName) throws IOException {
    Properties prop = readConfig(DEFAULT_CONFIG_FILE);
    AmazonS3 s3service = openService(prop);
    ObjectListing prevObjectListing = s3service.listObjects(bucketName);
    while (true) {
        for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            s3service.deleteObject(bucketName, s3ObjSumm.getKey());
        }
        if (!prevObjectListing.isTruncated()) {
            break;
        }
        prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
    }
    s3service.deleteBucket(bucketName);
}

Example 43 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit by apache.

the class S3Backend method renameKeys.

/**
 * This method rename object keys in S3 concurrently. The number of
 * concurrent threads is defined by 'maxConnections' property in
 * aws.properties. As S3 doesn't have "move" command, this method simulate
 * move as copy object object to new key and then delete older key.
 */
private void renameKeys() throws DataStoreException {
    long startTime = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    long count = 0;
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ObjectListing prevObjectListing = s3service.listObjects(bucket);
        List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
        int nThreads = Integer.parseInt(properties.getProperty("maxConnections"));
        ExecutorService executor = Executors.newFixedThreadPool(nThreads, new NamedThreadFactory("s3-object-rename-worker"));
        boolean taskAdded = false;
        while (true) {
            for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                executor.execute(new KeyRenameThread(s3ObjSumm.getKey()));
                taskAdded = true;
                count++;
                // delete the object if it follows old key name format
                if (s3ObjSumm.getKey().startsWith(KEY_PREFIX)) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
            }
            if (!prevObjectListing.isTruncated())
                break;
            prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
        }
        // This will make the executor accept no new threads
        // and finish all existing threads in the queue
        executor.shutdown();
        try {
            // Wait until all threads are finish
            while (taskAdded && !executor.awaitTermination(10, TimeUnit.SECONDS)) {
                LOG.info("Rename S3 keys tasks timedout. Waiting again");
            }
        } catch (InterruptedException ie) {
        }
        LOG.info("Renamed [{}] keys, time taken [{}]sec", count, ((System.currentTimeMillis() - startTime) / 1000));
        // Delete older keys.
        if (deleteList.size() > 0) {
            DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
            int batchSize = 500, startIndex = 0, size = deleteList.size();
            int endIndex = batchSize < size ? batchSize : size;
            while (endIndex <= size) {
                delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList(startIndex, endIndex)));
                DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq);
                LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", new Object[] { dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1) });
                if (endIndex == size) {
                    break;
                } else {
                    startIndex = endIndex;
                    endIndex = (startIndex + batchSize) < size ? (startIndex + batchSize) : size;
                }
            }
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}

Example 44 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project aws-doc-sdk-examples by awsdocs.

the class DeleteBucket2 method main.

public static void main(String[] args) {
    Regions clientRegion = Regions.DEFAULT_REGION;
    String bucketName = "*** Bucket name ***";
    try {
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withCredentials(new ProfileCredentialsProvider()).withRegion(clientRegion).build();
        // Delete all objects from the bucket. This is sufficient
        // for unversioned buckets. For versioned buckets, when you attempt to delete objects, Amazon S3 inserts
        // delete markers for all objects, but doesn't delete the object versions.
        // To delete objects from versioned buckets, delete all of the object versions before deleting
        // the bucket (see below for an example).
        ObjectListing objectListing = s3Client.listObjects(bucketName);
        while (true) {
            Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
            while (objIter.hasNext()) {
                s3Client.deleteObject(bucketName, objIter.next().getKey());
            }
            // and delete them.
            if (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
        // Delete all object versions (required for versioned buckets).
        VersionListing versionList = s3Client.listVersions(new ListVersionsRequest().withBucketName(bucketName));
        while (true) {
            Iterator<S3VersionSummary> versionIter = versionList.getVersionSummaries().iterator();
            while (versionIter.hasNext()) {
                S3VersionSummary vs = versionIter.next();
                s3Client.deleteVersion(bucketName, vs.getKey(), vs.getVersionId());
            }
            if (versionList.isTruncated()) {
                versionList = s3Client.listNextBatchOfVersions(versionList);
            } else {
                break;
            }
        }
        // After all objects and object versions are deleted, delete the bucket.
        s3Client.deleteBucket(bucketName);
    } catch (AmazonServiceException e) {
        // The call was transmitted successfully, but Amazon S3 couldn't process
        // it, so it returned an error response.
        e.printStackTrace();
    } catch (SdkClientException e) {
        // Amazon S3 couldn't be contacted for a response, or the client couldn't
        // parse the response from Amazon S3.
        e.printStackTrace();
    }
}

Example 45 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project aws-doc-sdk-examples by awsdocs.

the class DeleteBucket method main.

public static void main(String[] args) {
    final String USAGE = "\n" + "To run this example, supply the name of an S3 bucket\n" + "\n" + "Ex: DeleteBucket <bucketname>\n";
    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }
    String bucket_name = args[0];
    System.out.println("Deleting S3 bucket: " + bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    try {
        System.out.println(" - removing objects from bucket");
        ObjectListing object_listing = s3.listObjects(bucket_name);
        while (true) {
            for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext(); ) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                s3.deleteObject(bucket_name, summary.getKey());
            }
            // more object_listing to retrieve?
            if (object_listing.isTruncated()) {
                object_listing = s3.listNextBatchOfObjects(object_listing);
            } else {
                break;
            }
        }
        System.out.println(" - removing versions from bucket");
        VersionListing version_listing = s3.listVersions(new ListVersionsRequest().withBucketName(bucket_name));
        while (true) {
            for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext(); ) {
                S3VersionSummary vs = (S3VersionSummary) iterator.next();
                s3.deleteVersion(bucket_name, vs.getKey(), vs.getVersionId());
            }
            if (version_listing.isTruncated()) {
                version_listing = s3.listNextBatchOfVersions(version_listing);
            } else {
                break;
            }
        }
        System.out.println(" OK, bucket ready to delete!");
        s3.deleteBucket(bucket_name);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}