Examples with Closer org.apache.druid.java.util.common.io.Closer used on opensource projects

Example 26 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class OssDataSegmentPuller method buildFileObject.

private FileObject buildFileObject(final URI uri) throws OSSException {
    final CloudObjectLocation coords = new CloudObjectLocation(OssUtils.checkURI(uri));
    final OSSObjectSummary objectSummary = OssUtils.getSingleObjectSummary(client, coords.getBucket(), coords.getPath());
    final String path = uri.getPath();
    return new FileObject() {

        OSSObject ossObject = null;

        @Override
        public URI toUri() {
            return uri;
        }

        @Override
        public String getName() {
            final String ext = Files.getFileExtension(path);
            return Files.getNameWithoutExtension(path) + (Strings.isNullOrEmpty(ext) ? "" : ("." + ext));
        }

        /**
         * Returns an input stream for an OSS object. The returned input stream is not thread-safe.
         */
        @Override
        public InputStream openInputStream() throws IOException {
            try {
                if (ossObject == null) {
                    // lazily promote to full GET
                    ossObject = client.getObject(objectSummary.getBucketName(), objectSummary.getKey());
                }
                final InputStream in = ossObject.getObjectContent();
                final Closer closer = Closer.create();
                closer.register(in);
                closer.register(ossObject);
                return new FilterInputStream(in) {

                    @Override
                    public void close() throws IOException {
                        closer.close();
                    }
                };
            } catch (OSSException e) {
                throw new IOE(e, "Could not load OSS URI [%s]", uri);
            }
        }

        @Override
        public OutputStream openOutputStream() {
            throw new UOE("Cannot stream OSS output");
        }

        @Override
        public Reader openReader(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open reader");
        }

        @Override
        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open character sequence");
        }

        @Override
        public Writer openWriter() {
            throw new UOE("Cannot open writer");
        }

        @Override
        public long getLastModified() {
            return objectSummary.getLastModified().getTime();
        }

        @Override
        public boolean delete() {
            throw new UOE("Cannot delete OSS items anonymously. jetS3t doesn't support authenticated deletes easily.");
        }
    };
}

Example 27 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class AvroOCFReader method intermediateRowIterator.

@Override
protected CloseableIterator<GenericRecord> intermediateRowIterator() throws IOException {
    final Closer closer = Closer.create();
    final byte[] buffer = new byte[InputEntity.DEFAULT_FETCH_BUFFER_SIZE];
    try {
        final InputEntity.CleanableFile file = closer.register(source.fetch(temporaryDirectory, buffer));
        final GenericDatumReader<GenericRecord> datumReader = new GenericDatumReader<>();
        final DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file.file(), datumReader);
        final Schema writerSchema = dataFileReader.getSchema();
        if (readerSchema == null) {
            readerSchema = writerSchema;
        }
        datumReader.setSchema(writerSchema);
        datumReader.setExpected(readerSchema);
        closer.register(dataFileReader);
        return new CloseableIterator<GenericRecord>() {

            @Override
            public boolean hasNext() {
                return dataFileReader.hasNext();
            }

            @Override
            public GenericRecord next() {
                return dataFileReader.next();
            }

            @Override
            public void close() throws IOException {
                closer.close();
            }
        };
    } catch (Exception e) {
        closer.close();
        throw new RuntimeException(e);
    }
}

Example 28 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class FixedBucketsHistogramGroupByQueryTest method constructorFeeder.

@Parameterized.Parameters(name = "{0}")
public static Iterable<Object[]> constructorFeeder() {
    final GroupByQueryConfig v1Config = new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return GroupByStrategySelector.STRATEGY_V1;
        }

        @Override
        public String toString() {
            return "v1";
        }
    };
    final GroupByQueryConfig v1SingleThreadedConfig = new GroupByQueryConfig() {

        @Override
        public boolean isSingleThreaded() {
            return true;
        }

        @Override
        public String getDefaultStrategy() {
            return GroupByStrategySelector.STRATEGY_V1;
        }

        @Override
        public String toString() {
            return "v1SingleThreaded";
        }
    };
    final GroupByQueryConfig v2Config = new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return GroupByStrategySelector.STRATEGY_V2;
        }

        @Override
        public String toString() {
            return "v2";
        }
    };
    v1Config.setMaxIntermediateRows(10000);
    v1SingleThreadedConfig.setMaxIntermediateRows(10000);
    final List<Object[]> constructors = new ArrayList<>();
    final List<GroupByQueryConfig> configs = ImmutableList.of(v1Config, v1SingleThreadedConfig, v2Config);
    for (GroupByQueryConfig config : configs) {
        final Pair<GroupByQueryRunnerFactory, Closer> factoryAndCloser = GroupByQueryRunnerTest.makeQueryRunnerFactory(config);
        final GroupByQueryRunnerFactory factory = factoryAndCloser.lhs;
        RESOURCE_CLOSER.register(factoryAndCloser.rhs);
        for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(factory)) {
            final String testName = StringUtils.format("config=%s, runner=%s", config.toString(), runner.toString());
            constructors.add(new Object[] { testName, factory, runner });
        }
    }
    return constructors;
}

Example 29 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class TimedShutoffInputSourceReader method decorateShutdownTimeout.

private <T> CloseableIterator<T> decorateShutdownTimeout(ScheduledExecutorService exec, CloseableIterator<T> delegateIterator) {
    final Closer closer = Closer.create();
    closer.register(delegateIterator);
    closer.register(exec::shutdownNow);
    final CloseableIterator<T> wrappingIterator = new CloseableIterator<T>() {

        /**
         * Indicates this iterator has been closed or not.
         * Volatile since there is a happens-before relationship between {@link #hasNext()} and {@link #close()}.
         */
        volatile boolean closed;

        /**
         * Caching the next item. The item returned from the underling iterator is either a non-null {@link InputRow}
         * or {@link InputRowListPlusRawValues}.
         * Not volatile since {@link #hasNext()} and {@link #next()} are supposed to be called by the same thread.
         */
        T next = null;

        @Override
        public boolean hasNext() {
            if (next != null) {
                return true;
            }
            if (!closed && delegateIterator.hasNext()) {
                next = delegateIterator.next();
                return true;
            } else {
                return false;
            }
        }

        @Override
        public T next() {
            if (next != null) {
                final T returnValue = next;
                next = null;
                return returnValue;
            } else {
                throw new NoSuchElementException();
            }
        }

        @Override
        public void close() throws IOException {
            closed = true;
            closer.close();
        }
    };
    exec.schedule(() -> {
        LOG.info("Closing delegate inputSource.");
        try {
            wrappingIterator.close();
        } catch (IOException e) {
            LOG.warn(e, "Failed to close delegate inputSource, ignoring.");
        }
    }, shutoffTime.getMillis() - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
    return wrappingIterator;
}

Example 30 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class CloseableIterators method concat.

public static <T> CloseableIterator<T> concat(List<? extends CloseableIterator<? extends T>> iterators) {
    final Closer closer = Closer.create();
    iterators.forEach(closer::register);
    final Iterator<T> innerIterator = Iterators.concat(iterators.iterator());
    return wrap(innerIterator, closer);
}