Examples with Closer org.apache.druid.java.util.common.io.Closer used on opensource projects

Example 36 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class DistinctCountGroupByQueryTest method setup.

@Before
public void setup() {
    final GroupByQueryConfig config = new GroupByQueryConfig();
    config.setMaxIntermediateRows(10000);
    final Pair<GroupByQueryRunnerFactory, Closer> factoryCloserPair = GroupByQueryRunnerTest.makeQueryRunnerFactory(config);
    factory = factoryCloserPair.lhs;
    resourceCloser = factoryCloserPair.rhs;
}

Example 37 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class S3DataSegmentPuller method buildFileObject.

public FileObject buildFileObject(final URI uri) throws AmazonServiceException {
    final CloudObjectLocation coords = new CloudObjectLocation(S3Utils.checkURI(uri));
    final String path = uri.getPath();
    return new FileObject() {

        S3Object s3Object = null;

        S3ObjectSummary objectSummary = null;

        @Override
        public URI toUri() {
            return uri;
        }

        @Override
        public String getName() {
            final String ext = Files.getFileExtension(path);
            return Files.getNameWithoutExtension(path) + (Strings.isNullOrEmpty(ext) ? "" : ("." + ext));
        }

        /**
         * Returns an input stream for a s3 object. The returned input stream is not thread-safe.
         */
        @Override
        public InputStream openInputStream() throws IOException {
            try {
                if (s3Object == null) {
                    // lazily promote to full GET
                    s3Object = s3Client.getObject(coords.getBucket(), coords.getPath());
                }
                final InputStream in = s3Object.getObjectContent();
                final Closer closer = Closer.create();
                closer.register(in);
                closer.register(s3Object);
                return new FilterInputStream(in) {

                    @Override
                    public void close() throws IOException {
                        closer.close();
                    }
                };
            } catch (AmazonServiceException e) {
                throw new IOE(e, "Could not load S3 URI [%s]", uri);
            }
        }

        @Override
        public OutputStream openOutputStream() {
            throw new UOE("Cannot stream S3 output");
        }

        @Override
        public Reader openReader(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open reader");
        }

        @Override
        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
            throw new UOE("Cannot open character sequence");
        }

        @Override
        public Writer openWriter() {
            throw new UOE("Cannot open writer");
        }

        @Override
        public long getLastModified() {
            if (s3Object != null) {
                return s3Object.getObjectMetadata().getLastModified().getTime();
            }
            if (objectSummary == null) {
                objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.getBucket(), coords.getPath());
            }
            return objectSummary.getLastModified().getTime();
        }

        @Override
        public boolean delete() {
            throw new UOE("Cannot delete S3 items anonymously. jetS3t doesn't support authenticated deletes easily.");
        }
    };
}

Example 38 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class GroupByQueryRunnerTest method makeQueryRunnerFactory.

public static Pair<GroupByQueryRunnerFactory, Closer> makeQueryRunnerFactory(final ObjectMapper mapper, final GroupByQueryConfig config, final DruidProcessingConfig processingConfig) {
    final Supplier<GroupByQueryConfig> configSupplier = Suppliers.ofInstance(config);
    final CloseableStupidPool<ByteBuffer> bufferPool = new CloseableStupidPool<>("GroupByQueryEngine-bufferPool", new Supplier<ByteBuffer>() {

        @Override
        public ByteBuffer get() {
            return ByteBuffer.allocateDirect(processingConfig.intermediateComputeSizeBytes());
        }
    });
    final CloseableDefaultBlockingPool<ByteBuffer> mergeBufferPool = new CloseableDefaultBlockingPool<>(new Supplier<ByteBuffer>() {

        @Override
        public ByteBuffer get() {
            return ByteBuffer.allocateDirect(processingConfig.intermediateComputeSizeBytes());
        }
    }, processingConfig.getNumMergeBuffers());
    final GroupByStrategySelector strategySelector = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), QueryRunnerTestHelper.NOOP_QUERYWATCHER), new GroupByStrategyV2(processingConfig, configSupplier, bufferPool, mergeBufferPool, mapper, QueryRunnerTestHelper.NOOP_QUERYWATCHER));
    final GroupByQueryQueryToolChest toolChest = new GroupByQueryQueryToolChest(strategySelector);
    final Closer closer = Closer.create();
    closer.register(() -> {
        // Verify that all objects have been returned to the pool.
        Assert.assertEquals(bufferPool.poolSize(), bufferPool.objectsCreatedCount());
        bufferPool.close();
    });
    closer.register(mergeBufferPool);
    return Pair.of(new GroupByQueryRunnerFactory(strategySelector, toolChest), closer);
}

Example 39 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class BroadcastSegmentIndexedTableTest method checkNonIndexedReader.

private void checkNonIndexedReader(String columnName) {
    checkColumnSelectorFactory(columnName);
    try (final Closer closer = Closer.create()) {
        final int columnIndex = columnNames.indexOf(columnName);
        final int numRows = backingSegment.asStorageAdapter().getNumRows();
        final IndexedTable.Reader reader = broadcastTable.columnReader(columnIndex);
        closer.register(reader);
        final SimpleAscendingOffset offset = new SimpleAscendingOffset(numRows);
        final BaseColumn theColumn = backingSegment.asQueryableIndex().getColumnHolder(columnName).getColumn();
        closer.register(theColumn);
        final BaseObjectColumnValueSelector<?> selector = theColumn.makeColumnValueSelector(offset);
        // compare with selector make sure reader can read correct values
        for (int row = 0; row < numRows; row++) {
            offset.setCurrentOffset(row);
            Assert.assertEquals(selector.getObject(), reader.read(row));
        }
        // make sure it doesn't have an index since it isn't a key column
        try {
            Assert.assertEquals(null, broadcastTable.columnIndex(columnIndex));
        } catch (IAE iae) {
            Assert.assertEquals(StringUtils.format("Column[%d] is not a key column", columnIndex), iae.getMessage());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 40 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class SqlReader method intermediateRowIterator.

@Override
protected CloseableIterator<Map<String, Object>> intermediateRowIterator() throws IOException {
    final Closer closer = Closer.create();
    // The results are fetched into local storage as this avoids having to keep a persistent database connection for a long time
    final InputEntity.CleanableFile resultFile = closer.register(source.fetch(temporaryDirectory, null));
    FileInputStream inputStream = new FileInputStream(resultFile.file());
    JsonIterator<Map<String, Object>> jsonIterator = new JsonIterator<>(new TypeReference<Map<String, Object>>() {
    }, inputStream, closer, objectMapper);
    return jsonIterator;
}