Examples with StorageObjectsChunk org.jets3t.service.StorageObjectsChunk used on opensource projects

Example 1 with StorageObjectsChunk

use of org.jets3t.service.StorageObjectsChunk in project druid by druid-io.

the class S3Utils method storageObjectsIterator.

public static Iterator<StorageObject> storageObjectsIterator(final RestS3Service s3Client, final String bucket, final String prefix, final long maxListingLength) {
    return new Iterator<StorageObject>() {

        private StorageObjectsChunk objectsChunk;

        private int objectsChunkOffset;

        @Override
        public boolean hasNext() {
            if (objectsChunk == null) {
                objectsChunk = listObjectsChunkedAfter("");
                objectsChunkOffset = 0;
            }
            if (objectsChunk.getObjects().length <= objectsChunkOffset) {
                if (objectsChunk.isListingComplete()) {
                    return false;
                } else {
                    objectsChunk = listObjectsChunkedAfter(objectsChunk.getPriorLastKey());
                    objectsChunkOffset = 0;
                }
            }
            return true;
        }

        private StorageObjectsChunk listObjectsChunkedAfter(final String priorLastKey) {
            try {
                return retryS3Operation(new Callable<StorageObjectsChunk>() {

                    @Override
                    public StorageObjectsChunk call() throws Exception {
                        return s3Client.listObjectsChunked(bucket, prefix, null, maxListingLength, priorLastKey);
                    }
                });
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }

        @Override
        public StorageObject next() {
            if (!hasNext()) {
                throw new IllegalStateException();
            }
            StorageObject storageObject = objectsChunk.getObjects()[objectsChunkOffset];
            objectsChunkOffset++;
            return storageObject;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}

Example 2 with StorageObjectsChunk

use of org.jets3t.service.StorageObjectsChunk in project alluxio by Alluxio.

the class S3UnderFileSystem method getObjectListingChunk.

@Override
protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException {
    key = PathUtils.normalizePath(key, PATH_SEPARATOR);
    // In case key is root (empty string) do not normalize prefix
    key = key.equals(PATH_SEPARATOR) ? "" : key;
    String delimiter = recursive ? "" : PATH_SEPARATOR;
    StorageObjectsChunk chunk = getObjectListingChunk(key, delimiter, null);
    if (chunk != null) {
        return new S3NObjectListingChunk(chunk);
    }
    return null;
}

Example 3 with StorageObjectsChunk

use of org.jets3t.service.StorageObjectsChunk in project alluxio by Alluxio.

the class GCSUnderFileSystem method getObjectListingChunk.

@Override
protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException {
    key = PathUtils.normalizePath(key, PATH_SEPARATOR);
    // In case key is root (empty string) do not normalize prefix
    key = key.equals(PATH_SEPARATOR) ? "" : key;
    String delimiter = recursive ? "" : PATH_SEPARATOR;
    StorageObjectsChunk chunk = getObjectListingChunk(key, delimiter, null);
    if (chunk != null) {
        return new GCSObjectListingChunk(chunk);
    }
    return null;
}

Example 4 with StorageObjectsChunk

use of org.jets3t.service.StorageObjectsChunk in project hadoop by apache.

the class Jets3tNativeFileSystemStore method list.

/**
   * list objects
   * @param prefix prefix
   * @param delimiter delimiter
   * @param maxListingLength max no. of entries
   * @param priorLastKey last key in any previous search
   * @return a list of matches
   * @throws IOException on any reported failure
   */
private PartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey) throws IOException {
    try {
        if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
            prefix += PATH_DELIMITER;
        }
        StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), prefix, delimiter, maxListingLength, priorLastKey);
        FileMetadata[] fileMetadata = new FileMetadata[chunk.getObjects().length];
        for (int i = 0; i < fileMetadata.length; i++) {
            StorageObject object = chunk.getObjects()[i];
            fileMetadata[i] = new FileMetadata(object.getKey(), object.getContentLength(), object.getLastModifiedDate().getTime());
        }
        return new PartialListing(chunk.getPriorLastKey(), fileMetadata, chunk.getCommonPrefixes());
    } catch (ServiceException e) {
        handleException(e, prefix);
        // never returned - keep compiler happy
        return null;
    }
}