Examples with GoogleByteSource io.druid.storage.google.GoogleByteSource used on opensource projects

Example 1 with GoogleByteSource

use of io.druid.storage.google.GoogleByteSource in project druid by druid-io.

the class StaticGoogleBlobStoreFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException {
    Preconditions.checkNotNull(storage, "null storage");
    final LinkedList<GoogleBlob> objectQueue = Lists.newLinkedList(blobs);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final GoogleBlob nextURI = objectQueue.poll();
            final String bucket = nextURI.getBucket();
            final String path = nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath();
            try {
                final InputStream innerInputStream = new GoogleByteSource(storage, bucket, path).openStream();
                final InputStream outerInputStream = path.endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                LOG.error(e, "Exception opening bucket[%s] blob[%s]", bucket, path);
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, stringInputRowParser);
}