Search in sources :

Example 1 with CloudFilesByteSource

use of io.druid.storage.cloudfiles.CloudFilesByteSource in project druid by druid-io.

the class StaticCloudFilesFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException, ParseException {
    Preconditions.checkNotNull(cloudFilesApi, "null cloudFilesApi");
    final LinkedList<CloudFilesBlob> objectQueue = Lists.newLinkedList(blobs);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final CloudFilesBlob nextURI = objectQueue.poll();
            final String region = nextURI.getRegion();
            final String container = nextURI.getContainer();
            final String path = nextURI.getPath();
            log.info("Retrieving file from region[%s], container[%s] and path [%s]", region, container, path);
            CloudFilesObjectApiProxy objectApi = new CloudFilesObjectApiProxy(cloudFilesApi, region, container);
            final CloudFilesByteSource byteSource = new CloudFilesByteSource(objectApi, path);
            try {
                final InputStream innerInputStream = byteSource.openStream();
                final InputStream outerInputStream = path.endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (IOException e) {
                log.error(e, "Exception opening container[%s] blob[%s] from region[%s]", container, path, region);
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, stringInputRowParser);
}
Also used : CloudFilesByteSource(io.druid.storage.cloudfiles.CloudFilesByteSource) InputStreamReader(java.io.InputStreamReader) InputStream(java.io.InputStream) IOException(java.io.IOException) LineIterator(org.apache.commons.io.LineIterator) CloudFilesObjectApiProxy(io.druid.storage.cloudfiles.CloudFilesObjectApiProxy) BufferedReader(java.io.BufferedReader) FileIteratingFirehose(io.druid.data.input.impl.FileIteratingFirehose)

Aggregations

FileIteratingFirehose (io.druid.data.input.impl.FileIteratingFirehose)1 CloudFilesByteSource (io.druid.storage.cloudfiles.CloudFilesByteSource)1 CloudFilesObjectApiProxy (io.druid.storage.cloudfiles.CloudFilesObjectApiProxy)1 BufferedReader (java.io.BufferedReader)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 InputStreamReader (java.io.InputStreamReader)1 LineIterator (org.apache.commons.io.LineIterator)1