Examples with AbstractSequentialIterator com.google.common.collect.AbstractSequentialIterator used on opensource projects

Example 1 with AbstractSequentialIterator

use of com.google.common.collect.AbstractSequentialIterator in project presto by prestodb.

the class PrestoS3FileSystem method listPrefix.

private Iterator<LocatedFileStatus> listPrefix(Path path) {
    String key = keyFromPath(path);
    if (!key.isEmpty()) {
        key += PATH_SEPARATOR;
    }
    ListObjectsRequest request = new ListObjectsRequest().withBucketName(uri.getHost()).withPrefix(key).withDelimiter(PATH_SEPARATOR);
    STATS.newListObjectsCall();
    Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) {

        @Override
        protected ObjectListing computeNext(ObjectListing previous) {
            if (!previous.isTruncated()) {
                return null;
            }
            return s3.listNextBatchOfObjects(previous);
        }
    };
    return Iterators.concat(Iterators.transform(listings, this::statusFromListing));
}

Example 2 with AbstractSequentialIterator

use of com.google.common.collect.AbstractSequentialIterator in project presto by prestodb.

the class PrestoS3FileSystem method listPrefix.

private Iterator<LocatedFileStatus> listPrefix(Path path, OptionalInt initialMaxKeys, ListingMode mode) {
    String key = keyFromPath(path);
    if (!key.isEmpty()) {
        key += PATH_SEPARATOR;
    }
    ListObjectsRequest request = new ListObjectsRequest().withBucketName(getBucketName(uri)).withPrefix(key).withDelimiter(mode == ListingMode.RECURSIVE_FILES_ONLY ? null : PATH_SEPARATOR).withMaxKeys(initialMaxKeys.isPresent() ? initialMaxKeys.getAsInt() : null);
    STATS.newListObjectsCall();
    Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) {

        @Override
        protected ObjectListing computeNext(ObjectListing previous) {
            if (!previous.isTruncated()) {
                return null;
            }
            // Clear any max keys set for the initial request before submitting subsequent requests. Values < 0
            // are not sent in the request and the default limit is used
            previous.setMaxKeys(-1);
            return s3.listNextBatchOfObjects(previous);
        }
    };
    Iterator<LocatedFileStatus> result = Iterators.concat(Iterators.transform(listings, this::statusFromListing));
    if (mode.isFilesOnly()) {
        // Even recursive listing can still contain empty "directory" objects, must filter them out
        result = Iterators.filter(result, LocatedFileStatus::isFile);
    }
    return result;
}