Examples with FileIteratingFirehose io.druid.data.input.impl.FileIteratingFirehose used on opensource projects

Example 1 with FileIteratingFirehose

use of io.druid.data.input.impl.FileIteratingFirehose in project druid by druid-io.

the class StaticS3FirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    Preconditions.checkNotNull(s3Client, "null s3Client");
    final LinkedList<URI> objectQueue = Lists.newLinkedList(uris);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final URI nextURI = objectQueue.poll();
            final String s3Bucket = nextURI.getAuthority();
            final S3Object s3Object = new S3Object(nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath());
            log.info("Reading from bucket[%s] object[%s] (%s)", s3Bucket, s3Object.getKey(), nextURI);
            try {
                final InputStream innerInputStream = s3Client.getObject(new S3Bucket(s3Bucket), s3Object.getKey()).getDataInputStream();
                final InputStream outerInputStream = s3Object.getKey().endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "Exception reading from bucket[%s] object[%s]", s3Bucket, s3Object.getKey());
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

Example 2 with FileIteratingFirehose

use of io.druid.data.input.impl.FileIteratingFirehose in project druid by druid-io.

the class StaticGoogleBlobStoreFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException {
    Preconditions.checkNotNull(storage, "null storage");
    final LinkedList<GoogleBlob> objectQueue = Lists.newLinkedList(blobs);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final GoogleBlob nextURI = objectQueue.poll();
            final String bucket = nextURI.getBucket();
            final String path = nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath();
            try {
                final InputStream innerInputStream = new GoogleByteSource(storage, bucket, path).openStream();
                final InputStream outerInputStream = path.endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                LOG.error(e, "Exception opening bucket[%s] blob[%s]", bucket, path);
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, stringInputRowParser);
}

Example 3 with FileIteratingFirehose

use of io.druid.data.input.impl.FileIteratingFirehose in project druid by druid-io.

the class StaticCloudFilesFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException, ParseException {
    Preconditions.checkNotNull(cloudFilesApi, "null cloudFilesApi");
    final LinkedList<CloudFilesBlob> objectQueue = Lists.newLinkedList(blobs);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final CloudFilesBlob nextURI = objectQueue.poll();
            final String region = nextURI.getRegion();
            final String container = nextURI.getContainer();
            final String path = nextURI.getPath();
            log.info("Retrieving file from region[%s], container[%s] and path [%s]", region, container, path);
            CloudFilesObjectApiProxy objectApi = new CloudFilesObjectApiProxy(cloudFilesApi, region, container);
            final CloudFilesByteSource byteSource = new CloudFilesByteSource(objectApi, path);
            try {
                final InputStream innerInputStream = byteSource.openStream();
                final InputStream outerInputStream = path.endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (IOException e) {
                log.error(e, "Exception opening container[%s] blob[%s] from region[%s]", container, path, region);
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, stringInputRowParser);
}

Example 4 with FileIteratingFirehose

use of io.druid.data.input.impl.FileIteratingFirehose in project druid by druid-io.

the class StaticAzureBlobStoreFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException {
    Preconditions.checkNotNull(azureStorage, "null azureStorage");
    final LinkedList<AzureBlob> objectQueue = Lists.newLinkedList(blobs);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final AzureBlob nextURI = objectQueue.poll();
            final String container = nextURI.getContainer();
            final String path = nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath();
            try {
                final InputStream innerInputStream = new AzureByteSource(azureStorage, container, path).openStream();
                final InputStream outerInputStream = path.endsWith(".gz") ? CompressionUtils.gzipInputStream(innerInputStream) : innerInputStream;
                return IOUtils.lineIterator(new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "Exception opening container[%s] blob[%s]", container, path);
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, stringInputRowParser);
}

Example 5 with FileIteratingFirehose

use of io.druid.data.input.impl.FileIteratingFirehose in project druid by druid-io.

the class LocalFirehoseFactory method connect.

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    if (baseDir == null) {
        throw new IAE("baseDir is null");
    }
    log.info("Searching for all [%s] in and beneath [%s]", filter, baseDir.getAbsoluteFile());
    Collection<File> foundFiles = FileUtils.listFiles(baseDir.getAbsoluteFile(), new WildcardFileFilter(filter), TrueFileFilter.INSTANCE);
    if (foundFiles == null || foundFiles.isEmpty()) {
        throw new ISE("Found no files to ingest! Check your schema.");
    }
    log.info("Found files: " + foundFiles);
    final LinkedList<File> files = Lists.newLinkedList(foundFiles);
    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override
        public boolean hasNext() {
            return !files.isEmpty();
        }

        @Override
        public LineIterator next() {
            try {
                return FileUtils.lineIterator(files.poll());
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}