Search in sources :

Example 1 with DataGenerator

use of org.apache.druid.segment.generator.DataGenerator in project druid by druid-io.

the class ScanBenchmark method setup.

/**
 * Setup everything common for benchmarking both the incremental-index and the queriable-index.
 */
@Setup
public void setup() {
    log.info("SETUP CALLED AT " + +System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    setupQueries();
    String[] schemaQuery = schemaAndQuery.split("\\.");
    String schemaName = schemaQuery[0];
    String queryName = schemaQuery[1];
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schemaName);
    queryBuilder = SCHEMA_QUERY_MAP.get(schemaName).get(queryName);
    queryBuilder.limit(limit);
    query = queryBuilder.build();
    generator = new DataGenerator(schemaInfo.getColumnSchemas(), System.currentTimeMillis(), schemaInfo.getDataInterval(), rowsPerSegment);
    final ScanQueryConfig config = new ScanQueryConfig().setLegacy(false);
    factory = new ScanQueryRunnerFactory(new ScanQueryQueryToolChest(config, DefaultGenericQueryMetricsFactory.instance()), new ScanQueryEngine(), new ScanQueryConfig());
}
Also used : ScanQueryConfig(org.apache.druid.query.scan.ScanQueryConfig) ScanQueryQueryToolChest(org.apache.druid.query.scan.ScanQueryQueryToolChest) DataGenerator(org.apache.druid.segment.generator.DataGenerator) HyperUniquesSerde(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde) ScanQueryEngine(org.apache.druid.query.scan.ScanQueryEngine) ScanQueryRunnerFactory(org.apache.druid.query.scan.ScanQueryRunnerFactory) Setup(org.openjdk.jmh.annotations.Setup)

Example 2 with DataGenerator

use of org.apache.druid.segment.generator.DataGenerator in project druid by druid-io.

the class IndexMergeBenchmark method setup.

@Setup
public void setup() throws IOException {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    indexMergerV9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, getSegmentWriteOutMediumFactory(factoryType));
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    indexesToMerge = new ArrayList<>();
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schema);
    for (int i = 0; i < numSegments; i++) {
        DataGenerator gen = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED + i, schemaInfo.getDataInterval(), rowsPerSegment);
        IncrementalIndex incIndex = makeIncIndex();
        gen.addToIndex(incIndex, rowsPerSegment);
        tmpDir = FileUtils.createTempDir();
        log.info("Using temp dir: " + tmpDir.getAbsolutePath());
        File indexFile = indexMergerV9.persist(incIndex, tmpDir, new IndexSpec(), null);
        QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile);
        indexesToMerge.add(qIndex);
    }
}
Also used : IndexSpec(org.apache.druid.segment.IndexSpec) IncrementalIndex(org.apache.druid.segment.incremental.IncrementalIndex) OnheapIncrementalIndex(org.apache.druid.segment.incremental.OnheapIncrementalIndex) QueryableIndex(org.apache.druid.segment.QueryableIndex) IndexMergerV9(org.apache.druid.segment.IndexMergerV9) DataGenerator(org.apache.druid.segment.generator.DataGenerator) HyperUniquesSerde(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde) File(java.io.File) Setup(org.openjdk.jmh.annotations.Setup)

Example 3 with DataGenerator

use of org.apache.druid.segment.generator.DataGenerator in project druid by druid-io.

the class TimeseriesBenchmark method setup.

/**
 * Setup everything common for benchmarking both the incremental-index and the queriable-index.
 */
@Setup
public void setup() {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    setupQueries();
    String[] schemaQuery = schemaAndQuery.split("\\.");
    String schemaName = schemaQuery[0];
    String queryName = schemaQuery[1];
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schemaName);
    query = SCHEMA_QUERY_MAP.get(schemaName).get(queryName);
    generator = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval(), rowsPerSegment);
    factory = new TimeseriesQueryRunnerFactory(new TimeseriesQueryQueryToolChest(), new TimeseriesQueryEngine(), QueryBenchmarkUtil.NOOP_QUERYWATCHER);
}
Also used : TimeseriesQueryEngine(org.apache.druid.query.timeseries.TimeseriesQueryEngine) TimeseriesQueryRunnerFactory(org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory) DataGenerator(org.apache.druid.segment.generator.DataGenerator) HyperUniquesSerde(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde) TimeseriesQueryQueryToolChest(org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest) Setup(org.openjdk.jmh.annotations.Setup)

Example 4 with DataGenerator

use of org.apache.druid.segment.generator.DataGenerator in project druid by druid-io.

the class TopNBenchmark method setup.

/**
 * Setup everything common for benchmarking both the incremental-index and the queriable-index.
 */
@Setup
public void setup() {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    setupQueries();
    String[] schemaQuery = schemaAndQuery.split("\\.");
    String schemaName = schemaQuery[0];
    String queryName = schemaQuery[1];
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schemaName);
    queryBuilder = SCHEMA_QUERY_MAP.get(schemaName).get(queryName);
    queryBuilder.threshold(threshold);
    query = queryBuilder.build();
    generator = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval(), rowsPerSegment);
    factory = new TopNQueryRunnerFactory(new StupidPool<>("TopNBenchmark-compute-bufferPool", new OffheapBufferGenerator("compute", 250000000), 0, Integer.MAX_VALUE), new TopNQueryQueryToolChest(new TopNQueryConfig()), QueryBenchmarkUtil.NOOP_QUERYWATCHER);
}
Also used : OffheapBufferGenerator(org.apache.druid.offheap.OffheapBufferGenerator) TopNQueryConfig(org.apache.druid.query.topn.TopNQueryConfig) DataGenerator(org.apache.druid.segment.generator.DataGenerator) HyperUniquesSerde(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde) TopNQueryRunnerFactory(org.apache.druid.query.topn.TopNQueryRunnerFactory) StupidPool(org.apache.druid.collections.StupidPool) TopNQueryQueryToolChest(org.apache.druid.query.topn.TopNQueryQueryToolChest) Setup(org.openjdk.jmh.annotations.Setup)

Example 5 with DataGenerator

use of org.apache.druid.segment.generator.DataGenerator in project druid by druid-io.

the class IndexIngestionBenchmark method setup.

@Setup
public void setup() throws JsonProcessingException {
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schema);
    // Creates an AppendableIndexSpec that corresponds to the indexType parametrization.
    // It is used in {@code makeIncIndex()} to instanciate an incremental-index of the specified type.
    appendableIndexSpec = IncrementalIndexCreator.parseIndexType(indexType);
    DataGenerator gen = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval().getStartMillis(), IndexPersistBenchmark.getValuesPerTimestamp(rollupOpportunity), 1000.0);
    rows = gen.toList(rowsPerSegment);
}
Also used : DataGenerator(org.apache.druid.segment.generator.DataGenerator) HyperUniquesSerde(org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde) Setup(org.openjdk.jmh.annotations.Setup)

Aggregations

DataGenerator (org.apache.druid.segment.generator.DataGenerator)15 HyperUniquesSerde (org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde)14 Setup (org.openjdk.jmh.annotations.Setup)14 InputRow (org.apache.druid.data.input.InputRow)5 IndexSpec (org.apache.druid.segment.IndexSpec)5 File (java.io.File)4 StupidPool (org.apache.druid.collections.StupidPool)4 OffheapBufferGenerator (org.apache.druid.offheap.OffheapBufferGenerator)4 IncrementalIndex (org.apache.druid.segment.incremental.IncrementalIndex)4 OnheapIncrementalIndex (org.apache.druid.segment.incremental.OnheapIncrementalIndex)4 TimeseriesQueryQueryToolChest (org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest)3 TopNQueryQueryToolChest (org.apache.druid.query.topn.TopNQueryQueryToolChest)3 QueryableIndex (org.apache.druid.segment.QueryableIndex)3 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 SmileFactory (com.fasterxml.jackson.dataformat.smile.SmileFactory)2 ByteBuffer (java.nio.ByteBuffer)2 DefaultBlockingPool (org.apache.druid.collections.DefaultBlockingPool)2 DefaultObjectMapper (org.apache.druid.jackson.DefaultObjectMapper)2 DruidProcessingConfig (org.apache.druid.query.DruidProcessingConfig)2 BoundDimFilter (org.apache.druid.query.filter.BoundDimFilter)2