Search in sources :

Example 6 with StupidPool

use of org.apache.druid.collections.StupidPool in project druid by druid-io.

the class CachingClusteredClientBenchmark method makeGroupByQueryRunnerFactory.

private static GroupByQueryRunnerFactory makeGroupByQueryRunnerFactory(final ObjectMapper mapper, final GroupByQueryConfig config, final DruidProcessingConfig processingConfig) {
    final Supplier<GroupByQueryConfig> configSupplier = Suppliers.ofInstance(config);
    final Supplier<ByteBuffer> bufferSupplier = () -> ByteBuffer.allocateDirect(processingConfig.intermediateComputeSizeBytes());
    final NonBlockingPool<ByteBuffer> bufferPool = new StupidPool<>("GroupByQueryEngine-bufferPool", bufferSupplier);
    final BlockingPool<ByteBuffer> mergeBufferPool = new DefaultBlockingPool<>(bufferSupplier, processingConfig.getNumMergeBuffers());
    final GroupByStrategySelector strategySelector = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), QueryRunnerTestHelper.NOOP_QUERYWATCHER), new GroupByStrategyV2(processingConfig, configSupplier, bufferPool, mergeBufferPool, mapper, QueryRunnerTestHelper.NOOP_QUERYWATCHER));
    final GroupByQueryQueryToolChest toolChest = new GroupByQueryQueryToolChest(strategySelector);
    return new GroupByQueryRunnerFactory(strategySelector, toolChest);
}
Also used : GroupByStrategySelector(org.apache.druid.query.groupby.strategy.GroupByStrategySelector) GroupByQueryRunnerFactory(org.apache.druid.query.groupby.GroupByQueryRunnerFactory) GroupByQueryConfig(org.apache.druid.query.groupby.GroupByQueryConfig) ByteBuffer(java.nio.ByteBuffer) GroupByQueryQueryToolChest(org.apache.druid.query.groupby.GroupByQueryQueryToolChest) GroupByStrategyV1(org.apache.druid.query.groupby.strategy.GroupByStrategyV1) GroupByStrategyV2(org.apache.druid.query.groupby.strategy.GroupByStrategyV2) DefaultBlockingPool(org.apache.druid.collections.DefaultBlockingPool) StupidPool(org.apache.druid.collections.StupidPool) GroupByQueryEngine(org.apache.druid.query.groupby.GroupByQueryEngine)

Example 7 with StupidPool

use of org.apache.druid.collections.StupidPool in project druid by druid-io.

the class GroupByBenchmark method setup.

/**
 * Setup everything common for benchmarking both the incremental-index and the queriable-index.
 */
@Setup(Level.Trial)
public void setup() {
    log.info("SETUP CALLED AT " + +System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    setupQueries();
    String[] schemaQuery = schemaAndQuery.split("\\.");
    String schemaName = schemaQuery[0];
    String queryName = schemaQuery[1];
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schemaName);
    query = SCHEMA_QUERY_MAP.get(schemaName).get(queryName);
    generator = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval(), rowsPerSegment);
    NonBlockingPool<ByteBuffer> bufferPool = new StupidPool<>("GroupByBenchmark-computeBufferPool", new OffheapBufferGenerator("compute", 250_000_000), 0, Integer.MAX_VALUE);
    // limit of 2 is required since we simulate both historical merge and broker merge in the same process
    BlockingPool<ByteBuffer> mergePool = new DefaultBlockingPool<>(new OffheapBufferGenerator("merge", 250_000_000), 2);
    final GroupByQueryConfig config = new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return defaultStrategy;
        }

        @Override
        public int getBufferGrouperInitialBuckets() {
            return initialBuckets;
        }

        @Override
        public long getMaxOnDiskStorage() {
            return 1_000_000_000L;
        }
    };
    config.setSingleThreaded(false);
    config.setMaxIntermediateRows(Integer.MAX_VALUE);
    config.setMaxResults(Integer.MAX_VALUE);
    DruidProcessingConfig druidProcessingConfig = new DruidProcessingConfig() {

        @Override
        public int getNumThreads() {
            // Used by "v2" strategy for concurrencyHint
            return numProcessingThreads;
        }

        @Override
        public String getFormatString() {
            return null;
        }
    };
    final Supplier<GroupByQueryConfig> configSupplier = Suppliers.ofInstance(config);
    final GroupByStrategySelector strategySelector = new GroupByStrategySelector(configSupplier, new GroupByStrategyV1(configSupplier, new GroupByQueryEngine(configSupplier, bufferPool), QueryBenchmarkUtil.NOOP_QUERYWATCHER), new GroupByStrategyV2(druidProcessingConfig, configSupplier, bufferPool, mergePool, new ObjectMapper(new SmileFactory()), QueryBenchmarkUtil.NOOP_QUERYWATCHER));
    factory = new GroupByQueryRunnerFactory(strategySelector, new GroupByQueryQueryToolChest(strategySelector));
}
Also used : GroupByStrategySelector(org.apache.druid.query.groupby.strategy.GroupByStrategySelector) GroupByQueryRunnerFactory(org.apache.druid.query.groupby.GroupByQueryRunnerFactory) GroupByQueryConfig(org.apache.druid.query.groupby.GroupByQueryConfig) HyperUniquesSerde(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde) ByteBuffer(java.nio.ByteBuffer) GroupByQueryQueryToolChest(org.apache.druid.query.groupby.GroupByQueryQueryToolChest) SmileFactory(com.fasterxml.jackson.dataformat.smile.SmileFactory) OffheapBufferGenerator(org.apache.druid.offheap.OffheapBufferGenerator) GroupByStrategyV1(org.apache.druid.query.groupby.strategy.GroupByStrategyV1) GroupByStrategyV2(org.apache.druid.query.groupby.strategy.GroupByStrategyV2) DefaultBlockingPool(org.apache.druid.collections.DefaultBlockingPool) DataGenerator(org.apache.druid.segment.generator.DataGenerator) StupidPool(org.apache.druid.collections.StupidPool) DruidProcessingConfig(org.apache.druid.query.DruidProcessingConfig) GroupByQueryEngine(org.apache.druid.query.groupby.GroupByQueryEngine) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Setup(org.openjdk.jmh.annotations.Setup)

Example 8 with StupidPool

use of org.apache.druid.collections.StupidPool in project druid by druid-io.

the class TopNTypeInterfaceBenchmark method setup.

@Setup
public void setup() throws IOException {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    setupQueries();
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("basic");
    queryBuilder = SCHEMA_QUERY_MAP.get("basic").get("string");
    queryBuilder.threshold(threshold);
    stringQuery = queryBuilder.build();
    TopNQueryBuilder longBuilder = SCHEMA_QUERY_MAP.get("basic").get("long");
    longBuilder.threshold(threshold);
    longQuery = longBuilder.build();
    TopNQueryBuilder floatBuilder = SCHEMA_QUERY_MAP.get("basic").get("float");
    floatBuilder.threshold(threshold);
    floatQuery = floatBuilder.build();
    incIndexes = new ArrayList<>();
    for (int i = 0; i < numSegments; i++) {
        log.info("Generating rows for segment " + i);
        DataGenerator gen = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED + i, schemaInfo.getDataInterval(), rowsPerSegment);
        IncrementalIndex incIndex = makeIncIndex();
        for (int j = 0; j < rowsPerSegment; j++) {
            InputRow row = gen.nextRow();
            if (j % 10000 == 0) {
                log.info(j + " rows generated.");
            }
            incIndex.add(row);
        }
        incIndexes.add(incIndex);
    }
    File tmpFile = FileUtils.createTempDir();
    log.info("Using temp dir: " + tmpFile.getAbsolutePath());
    tmpFile.deleteOnExit();
    qIndexes = new ArrayList<>();
    for (int i = 0; i < numSegments; i++) {
        File indexFile = INDEX_MERGER_V9.persist(incIndexes.get(i), tmpFile, new IndexSpec(), null);
        QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile);
        qIndexes.add(qIndex);
    }
    factory = new TopNQueryRunnerFactory(new StupidPool<>("TopNBenchmark-compute-bufferPool", new OffheapBufferGenerator("compute", 250000000), 0, Integer.MAX_VALUE), new TopNQueryQueryToolChest(new TopNQueryConfig()), QueryBenchmarkUtil.NOOP_QUERYWATCHER);
}
Also used : TopNQueryBuilder(org.apache.druid.query.topn.TopNQueryBuilder) IndexSpec(org.apache.druid.segment.IndexSpec) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) HyperUniquesSerde(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde) OffheapBufferGenerator(org.apache.druid.offheap.OffheapBufferGenerator) TopNQueryConfig(org.apache.druid.query.topn.TopNQueryConfig) QueryableIndex(org.apache.druid.segment.QueryableIndex) DataGenerator(org.apache.druid.segment.generator.DataGenerator) InputRow(org.apache.druid.data.input.InputRow) TopNQueryRunnerFactory(org.apache.druid.query.topn.TopNQueryRunnerFactory) StupidPool(org.apache.druid.collections.StupidPool) TopNQueryQueryToolChest(org.apache.druid.query.topn.TopNQueryQueryToolChest) File(java.io.File) Setup(org.openjdk.jmh.annotations.Setup)

Aggregations

StupidPool (org.apache.druid.collections.StupidPool)8 DefaultBlockingPool (org.apache.druid.collections.DefaultBlockingPool)5 DruidProcessingConfig (org.apache.druid.query.DruidProcessingConfig)5 GroupByQueryConfig (org.apache.druid.query.groupby.GroupByQueryConfig)5 GroupByQueryQueryToolChest (org.apache.druid.query.groupby.GroupByQueryQueryToolChest)5 GroupByQueryRunnerFactory (org.apache.druid.query.groupby.GroupByQueryRunnerFactory)5 GroupByStrategySelector (org.apache.druid.query.groupby.strategy.GroupByStrategySelector)5 GroupByStrategyV2 (org.apache.druid.query.groupby.strategy.GroupByStrategyV2)5 Setup (org.openjdk.jmh.annotations.Setup)5 ByteBuffer (java.nio.ByteBuffer)4 DefaultObjectMapper (org.apache.druid.jackson.DefaultObjectMapper)4 OffheapBufferGenerator (org.apache.druid.offheap.OffheapBufferGenerator)4 HyperUniquesSerde (org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde)4 GroupByStrategyV1 (org.apache.druid.query.groupby.strategy.GroupByStrategyV1)4 DataGenerator (org.apache.druid.segment.generator.DataGenerator)4 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)3 SmileFactory (com.fasterxml.jackson.dataformat.smile.SmileFactory)3 GroupByQueryEngine (org.apache.druid.query.groupby.GroupByQueryEngine)3 TopNQueryConfig (org.apache.druid.query.topn.TopNQueryConfig)3 TopNQueryQueryToolChest (org.apache.druid.query.topn.TopNQueryQueryToolChest)3