Examples with ObjectListing com.amazonaws.services.s3.model.ObjectListing used on opensource projects

Example 91 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class S3Backend method getAllMetadataRecords.

@Override
public List<DataRecord> getAllMetadataRecords(String prefix) {
    List<DataRecord> metadataList = new ArrayList<DataRecord>();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
        ObjectListing prevObjectListing = s3service.listObjects(listObjectsRequest);
        for (final S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
            metadataList.add(new S3DataRecord(this, s3service, bucket, new DataIdentifier(stripMetaKeyPrefix(s3ObjSumm.getKey())), s3ObjSumm.getLastModified().getTime(), s3ObjSumm.getSize(), true));
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    return metadataList;
}

Example 92 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project jackrabbit-oak by apache.

the class DataStoreUtils method deleteBucket.

public static void deleteBucket(String bucket, Map<String, ?> map, Date date) throws Exception {
    log.info("cleaning bucket [" + bucket + "]");
    Properties props = new Properties();
    props.putAll(map);
    AmazonS3Client s3service = Utils.openService(props);
    TransferManager tmx = new TransferManager(s3service);
    if (s3service.doesBucketExist(bucket)) {
        for (int i = 0; i < 4; i++) {
            tmx.abortMultipartUploads(bucket, date);
            ObjectListing prevObjectListing = s3service.listObjects(bucket);
            while (prevObjectListing != null) {
                List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) {
                    deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
                }
                if (deleteList.size() > 0) {
                    DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
                    delObjsReq.setKeys(deleteList);
                    s3service.deleteObjects(delObjsReq);
                }
                if (!prevObjectListing.isTruncated())
                    break;
                prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
            }
        }
        s3service.deleteBucket(bucket);
        log.info("bucket [ " + bucket + "] cleaned");
    } else {
        log.info("bucket [" + bucket + "] doesn't exists");
    }
    tmx.shutdownNow();
    s3service.shutdown();
}

Example 93 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project kafka-connect-storage-cloud by confluentinc.

the class TestWithMockedS3 method listObjects.

public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
    List<S3ObjectSummary> objects = new ArrayList<>();
    ObjectListing listing;
    try {
        if (prefix == null) {
            listing = s3.listObjects(bucket);
        } else {
            listing = s3.listObjects(bucket, prefix);
        }
        objects.addAll(listing.getObjectSummaries());
        while (listing.isTruncated()) {
            listing = s3.listNextBatchOfObjects(listing);
            objects.addAll(listing.getObjectSummaries());
        }
    } catch (AmazonS3Exception e) {
        log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
    }
    return objects;
}

Example 94 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project herd by FINRAOS.

the class MockS3OperationsImpl method listObjects.

/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client) {
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());
    String bucketName = listObjectsRequest.getBucketName();
    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName)) {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }
    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);
    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null) {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values()) {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix())) {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);
                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }
    return objectListing;
}

Example 95 with ObjectListing

use of com.amazonaws.services.s3.model.ObjectListing in project herd by FINRAOS.

the class S3DaoImpl method listDirectory.

@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) {
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");
    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;
        do {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L)) {
                    s3ObjectSummaries.add(objectSummary);
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonS3Exception amazonS3Exception) {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode())) {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    } catch (AmazonClientException e) {
        throw new IllegalStateException(String.format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(), e.getMessage()), e);
    } finally {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }
    return s3ObjectSummaries;
}