Search in sources :

Example 6 with CloseableStupidPool

use of org.apache.druid.collections.CloseableStupidPool in project druid by druid-io.

the class IncrementalIndexStorageAdapterTest method testSanity.

@Test
public void testSanity() throws Exception {
    IncrementalIndex index = indexCreator.createIndex();
    index.add(new MapBasedInputRow(System.currentTimeMillis() - 1, Collections.singletonList("billy"), ImmutableMap.of("billy", "hi")));
    index.add(new MapBasedInputRow(System.currentTimeMillis() - 1, Collections.singletonList("sally"), ImmutableMap.of("sally", "bo")));
    try (CloseableStupidPool<ByteBuffer> pool = new CloseableStupidPool<>("GroupByQueryEngine-bufferPool", () -> ByteBuffer.allocate(50000))) {
        final GroupByQueryEngine engine = new GroupByQueryEngine(Suppliers.ofInstance(new GroupByQueryConfig() {

            @Override
            public int getMaxIntermediateRows() {
                return 5;
            }
        }), pool);
        final Sequence<Row> rows = engine.process(GroupByQuery.builder().setDataSource("test").setGranularity(Granularities.ALL).setInterval(new Interval(DateTimes.EPOCH, DateTimes.nowUtc())).addDimension("billy").addDimension("sally").addAggregator(new LongSumAggregatorFactory("cnt", "cnt")).build(), new IncrementalIndexStorageAdapter(index));
        final List<Row> results = rows.toList();
        Assert.assertEquals(2, results.size());
        MapBasedRow row = (MapBasedRow) results.get(0);
        Assert.assertEquals(ImmutableMap.of("sally", "bo", "cnt", 1L), row.getEvent());
        row = (MapBasedRow) results.get(1);
        Assert.assertEquals(ImmutableMap.of("billy", "hi", "cnt", 1L), row.getEvent());
    }
}
Also used : GroupByQueryConfig(org.apache.druid.query.groupby.GroupByQueryConfig) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) CloseableStupidPool(org.apache.druid.collections.CloseableStupidPool) ByteBuffer(java.nio.ByteBuffer) MapBasedRow(org.apache.druid.data.input.MapBasedRow) MapBasedInputRow(org.apache.druid.data.input.MapBasedInputRow) MapBasedRow(org.apache.druid.data.input.MapBasedRow) Row(org.apache.druid.data.input.Row) MapBasedInputRow(org.apache.druid.data.input.MapBasedInputRow) GroupByQueryEngine(org.apache.druid.query.groupby.GroupByQueryEngine) Interval(org.joda.time.Interval) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 7 with CloseableStupidPool

use of org.apache.druid.collections.CloseableStupidPool in project druid by druid-io.

the class QueryStackTests method createQueryRunnerFactoryConglomerate.

public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate(final Closer closer, final DruidProcessingConfig processingConfig, final Supplier<Integer> minTopNThresholdSupplier) {
    final CloseableStupidPool<ByteBuffer> stupidPool = new CloseableStupidPool<>("TopNQueryRunnerFactory-bufferPool", () -> ByteBuffer.allocate(COMPUTE_BUFFER_SIZE));
    closer.register(stupidPool);
    final Pair<GroupByQueryRunnerFactory, Closer> factoryCloserPair = GroupByQueryRunnerTest.makeQueryRunnerFactory(GroupByQueryRunnerTest.DEFAULT_MAPPER, new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return GroupByStrategySelector.STRATEGY_V2;
        }
    }, processingConfig);
    final GroupByQueryRunnerFactory groupByQueryRunnerFactory = factoryCloserPair.lhs;
    closer.register(factoryCloserPair.rhs);
    final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(ImmutableMap.<Class<? extends Query>, QueryRunnerFactory>builder().put(SegmentMetadataQuery.class, new SegmentMetadataQueryRunnerFactory(new SegmentMetadataQueryQueryToolChest(new SegmentMetadataQueryConfig("P1W")), QueryRunnerTestHelper.NOOP_QUERYWATCHER)).put(ScanQuery.class, new ScanQueryRunnerFactory(new ScanQueryQueryToolChest(new ScanQueryConfig(), new DefaultGenericQueryMetricsFactory()), new ScanQueryEngine(), new ScanQueryConfig())).put(TimeseriesQuery.class, new TimeseriesQueryRunnerFactory(new TimeseriesQueryQueryToolChest(), new TimeseriesQueryEngine(), QueryRunnerTestHelper.NOOP_QUERYWATCHER)).put(TopNQuery.class, new TopNQueryRunnerFactory(stupidPool, new TopNQueryQueryToolChest(new TopNQueryConfig() {

        @Override
        public int getMinTopNThreshold() {
            return minTopNThresholdSupplier.get();
        }
    }), QueryRunnerTestHelper.NOOP_QUERYWATCHER)).put(GroupByQuery.class, groupByQueryRunnerFactory).build());
    return conglomerate;
}
Also used : ScanQueryConfig(org.apache.druid.query.scan.ScanQueryConfig) ScanQuery(org.apache.druid.query.scan.ScanQuery) TimeseriesQueryQueryToolChest(org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest) QueryRunnerFactoryConglomerate(org.apache.druid.query.QueryRunnerFactoryConglomerate) DefaultQueryRunnerFactoryConglomerate(org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate) TimeseriesQueryEngine(org.apache.druid.query.timeseries.TimeseriesQueryEngine) SegmentMetadataQueryRunnerFactory(org.apache.druid.query.metadata.SegmentMetadataQueryRunnerFactory) TopNQuery(org.apache.druid.query.topn.TopNQuery) TopNQueryRunnerFactory(org.apache.druid.query.topn.TopNQueryRunnerFactory) TopNQueryQueryToolChest(org.apache.druid.query.topn.TopNQueryQueryToolChest) DefaultGenericQueryMetricsFactory(org.apache.druid.query.DefaultGenericQueryMetricsFactory) SegmentMetadataQueryConfig(org.apache.druid.query.metadata.SegmentMetadataQueryConfig) ScanQueryRunnerFactory(org.apache.druid.query.scan.ScanQueryRunnerFactory) Closer(org.apache.druid.java.util.common.io.Closer) GroupByQueryRunnerFactory(org.apache.druid.query.groupby.GroupByQueryRunnerFactory) GroupByQueryConfig(org.apache.druid.query.groupby.GroupByQueryConfig) DefaultQueryRunnerFactoryConglomerate(org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate) ScanQueryEngine(org.apache.druid.query.scan.ScanQueryEngine) CloseableStupidPool(org.apache.druid.collections.CloseableStupidPool) ByteBuffer(java.nio.ByteBuffer) TimeseriesQueryRunnerFactory(org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory) ScanQueryQueryToolChest(org.apache.druid.query.scan.ScanQueryQueryToolChest) TopNQueryConfig(org.apache.druid.query.topn.TopNQueryConfig) SegmentMetadataQueryQueryToolChest(org.apache.druid.query.metadata.SegmentMetadataQueryQueryToolChest)

Example 8 with CloseableStupidPool

use of org.apache.druid.collections.CloseableStupidPool in project druid by druid-io.

the class GroupByQueryRunnerTest method makeQueryRunnerFactory.

public static Pair<GroupByQueryRunnerFactory, Closer> makeQueryRunnerFactory(final ObjectMapper mapper, final GroupByQueryConfig config, final DruidProcessingConfig processingConfig) {
    final Supplier<GroupByQueryConfig> configSupplier = Suppliers.ofInstance(config);
    final CloseableStupidPool<ByteBuffer> bufferPool = new CloseableStupidPool<>("GroupByQueryEngine-bufferPool", new Supplier<ByteBuffer>() {

        @Override
        public ByteBuffer get() {
            return ByteBuffer.allocateDirect(processingConfig.intermediateComputeSizeBytes());
        }
    });
    final CloseableDefaultBlockingPool<ByteBuffer> mergeBufferPool = new CloseableDefaultBlockingPool<>(new Supplier<ByteBuffer>() {

        @Override
        public ByteBuffer get() {
            return ByteBuffer.allocateDirect(processingConfig.intermediateComputeSizeBytes());
        }
    }, processingConfig.getNumMergeBuffers());
    final GroupByStrategySelector strategySelector = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), QueryRunnerTestHelper.NOOP_QUERYWATCHER), new GroupByStrategyV2(processingConfig, configSupplier, bufferPool, mergeBufferPool, mapper, QueryRunnerTestHelper.NOOP_QUERYWATCHER));
    final GroupByQueryQueryToolChest toolChest = new GroupByQueryQueryToolChest(strategySelector);
    final Closer closer = Closer.create();
    closer.register(() -> {
        // Verify that all objects have been returned to the pool.
        Assert.assertEquals(bufferPool.poolSize(), bufferPool.objectsCreatedCount());
        bufferPool.close();
    });
    closer.register(mergeBufferPool);
    return Pair.of(new GroupByQueryRunnerFactory(strategySelector, toolChest), closer);
}
Also used : GroupByStrategySelector(org.apache.druid.query.groupby.strategy.GroupByStrategySelector) Closer(org.apache.druid.java.util.common.io.Closer) CloseableStupidPool(org.apache.druid.collections.CloseableStupidPool) ByteBuffer(java.nio.ByteBuffer) GroupByStrategyV1(org.apache.druid.query.groupby.strategy.GroupByStrategyV1) CloseableDefaultBlockingPool(org.apache.druid.collections.CloseableDefaultBlockingPool) GroupByStrategyV2(org.apache.druid.query.groupby.strategy.GroupByStrategyV2)

Example 9 with CloseableStupidPool

use of org.apache.druid.collections.CloseableStupidPool in project druid by druid-io.

the class GroupByLimitPushDownMultiNodeMergeTest method setupGroupByFactory.

private void setupGroupByFactory() {
    executorService = Execs.multiThreaded(3, "GroupByThreadPool[%d]");
    final CloseableStupidPool<ByteBuffer> bufferPool = new CloseableStupidPool<>("GroupByBenchmark-computeBufferPool", new OffheapBufferGenerator("compute", 10_000_000), 0, Integer.MAX_VALUE);
    // limit of 2 is required since we simulate both historical merge and broker merge in the same process
    final CloseableDefaultBlockingPool<ByteBuffer> mergePool = new CloseableDefaultBlockingPool<>(new OffheapBufferGenerator("merge", 10_000_000), 2);
    // limit of 2 is required since we simulate both historical merge and broker merge in the same process
    final CloseableDefaultBlockingPool<ByteBuffer> mergePool2 = new CloseableDefaultBlockingPool<>(new OffheapBufferGenerator("merge", 10_000_000), 2);
    resourceCloser.register(bufferPool);
    resourceCloser.register(mergePool);
    resourceCloser.register(mergePool2);
    final GroupByQueryConfig config = new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return "v2";
        }

        @Override
        public int getBufferGrouperInitialBuckets() {
            return -1;
        }

        @Override
        public long getMaxOnDiskStorage() {
            return 1_000_000_000L;
        }
    };
    config.setSingleThreaded(false);
    config.setMaxIntermediateRows(Integer.MAX_VALUE);
    config.setMaxResults(Integer.MAX_VALUE);
    DruidProcessingConfig druidProcessingConfig = new DruidProcessingConfig() {

        @Override
        public int getNumThreads() {
            // Used by "v2" strategy for concurrencyHint
            return 2;
        }

        @Override
        public String getFormatString() {
            return null;
        }
    };
    final Supplier<GroupByQueryConfig> configSupplier = Suppliers.ofInstance(config);
    final GroupByStrategySelector strategySelector = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), NOOP_QUERYWATCHER), new GroupByStrategyV2(druidProcessingConfig, configSupplier, bufferPool, mergePool, new ObjectMapper(new SmileFactory()), NOOP_QUERYWATCHER));
    final GroupByStrategySelector strategySelector2 = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), NOOP_QUERYWATCHER), new GroupByStrategyV2(druidProcessingConfig, configSupplier, bufferPool, mergePool2, new ObjectMapper(new SmileFactory()), NOOP_QUERYWATCHER));
    groupByFactory = new GroupByQueryRunnerFactory(strategySelector, new GroupByQueryQueryToolChest(strategySelector));
    groupByFactory2 = new GroupByQueryRunnerFactory(strategySelector2, new GroupByQueryQueryToolChest(strategySelector2));
}
Also used : GroupByStrategySelector(org.apache.druid.query.groupby.strategy.GroupByStrategySelector) CloseableStupidPool(org.apache.druid.collections.CloseableStupidPool) ByteBuffer(java.nio.ByteBuffer) SmileFactory(com.fasterxml.jackson.dataformat.smile.SmileFactory) GroupByStrategyV1(org.apache.druid.query.groupby.strategy.GroupByStrategyV1) CloseableDefaultBlockingPool(org.apache.druid.collections.CloseableDefaultBlockingPool) GroupByStrategyV2(org.apache.druid.query.groupby.strategy.GroupByStrategyV2) DruidProcessingConfig(org.apache.druid.query.DruidProcessingConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper)

Example 10 with CloseableStupidPool

use of org.apache.druid.collections.CloseableStupidPool in project druid by druid-io.

the class ApproximateHistogramTopNQueryTest method constructorFeeder.

@Parameterized.Parameters(name = "{0}")
public static Iterable<Object[]> constructorFeeder() {
    final CloseableStupidPool<ByteBuffer> defaultPool = TestQueryRunners.createDefaultNonBlockingPool();
    final CloseableStupidPool<ByteBuffer> customPool = new CloseableStupidPool<>("TopNQueryRunnerFactory-bufferPool", () -> ByteBuffer.allocate(2000));
    RESOURCE_CLOSER.register(defaultPool);
    RESOURCE_CLOSER.register(customPool);
    return QueryRunnerTestHelper.transformToConstructionFeeder(Iterables.concat(QueryRunnerTestHelper.makeQueryRunners(new TopNQueryRunnerFactory(defaultPool, new TopNQueryQueryToolChest(new TopNQueryConfig()), QueryRunnerTestHelper.NOOP_QUERYWATCHER)), QueryRunnerTestHelper.makeQueryRunners(new TopNQueryRunnerFactory(customPool, new TopNQueryQueryToolChest(new TopNQueryConfig()), QueryRunnerTestHelper.NOOP_QUERYWATCHER))));
}
Also used : TopNQueryConfig(org.apache.druid.query.topn.TopNQueryConfig) TopNQueryRunnerFactory(org.apache.druid.query.topn.TopNQueryRunnerFactory) TopNQueryQueryToolChest(org.apache.druid.query.topn.TopNQueryQueryToolChest) CloseableStupidPool(org.apache.druid.collections.CloseableStupidPool) ByteBuffer(java.nio.ByteBuffer)

Aggregations

ByteBuffer (java.nio.ByteBuffer)12 CloseableStupidPool (org.apache.druid.collections.CloseableStupidPool)12 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)4 CloseableDefaultBlockingPool (org.apache.druid.collections.CloseableDefaultBlockingPool)4 MapBasedInputRow (org.apache.druid.data.input.MapBasedInputRow)4 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)4 GroupByQueryConfig (org.apache.druid.query.groupby.GroupByQueryConfig)4 GroupByStrategySelector (org.apache.druid.query.groupby.strategy.GroupByStrategySelector)4 GroupByStrategyV1 (org.apache.druid.query.groupby.strategy.GroupByStrategyV1)4 GroupByStrategyV2 (org.apache.druid.query.groupby.strategy.GroupByStrategyV2)4 TopNQueryConfig (org.apache.druid.query.topn.TopNQueryConfig)4 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)4 Interval (org.joda.time.Interval)4 Test (org.junit.Test)4 SmileFactory (com.fasterxml.jackson.dataformat.smile.SmileFactory)3 MapBasedRow (org.apache.druid.data.input.MapBasedRow)3 Row (org.apache.druid.data.input.Row)3 DefaultObjectMapper (org.apache.druid.jackson.DefaultObjectMapper)3 Closer (org.apache.druid.java.util.common.io.Closer)3 DruidProcessingConfig (org.apache.druid.query.DruidProcessingConfig)3