Search in sources :

Example 1 with SegmentGenerator

use of org.apache.druid.segment.generator.SegmentGenerator in project druid by druid-io.

the class SqlVectorizedExpressionSanityTest method setupClass.

@BeforeClass
public static void setupClass() {
    Calcites.setSystemProperties();
    ExpressionProcessing.initializeForStrictBooleansTests(true);
    CLOSER = Closer.create();
    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
    final DataSegment dataSegment = DataSegment.builder().dataSource("foo").interval(schemaInfo.getDataInterval()).version("1").shardSpec(new LinearShardSpec(0)).size(0).build();
    final SegmentGenerator segmentGenerator = CLOSER.register(new SegmentGenerator());
    INDEX = CLOSER.register(segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, ROWS_PER_SEGMENT));
    CONGLOMERATE = QueryStackTests.createQueryRunnerFactoryConglomerate(CLOSER);
    WALKER = new SpecificSegmentsQuerySegmentWalker(CONGLOMERATE).add(dataSegment, INDEX);
    CLOSER.register(WALKER);
    final PlannerConfig plannerConfig = new PlannerConfig();
    final DruidSchemaCatalog rootSchema = CalciteTests.createMockRootSchema(CONGLOMERATE, WALKER, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
    PLANNER_FACTORY = new PlannerFactory(rootSchema, CalciteTests.createMockQueryMakerFactory(WALKER, CONGLOMERATE), CalciteTests.createOperatorTable(), CalciteTests.createExprMacroTable(), plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER, CalciteTests.getJsonMapper(), CalciteTests.DRUID_SCHEMA_NAME);
}
Also used : SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) SpecificSegmentsQuerySegmentWalker(org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker) LinearShardSpec(org.apache.druid.timeline.partition.LinearShardSpec) GeneratorSchemaInfo(org.apache.druid.segment.generator.GeneratorSchemaInfo) PlannerConfig(org.apache.druid.sql.calcite.planner.PlannerConfig) DruidSchemaCatalog(org.apache.druid.sql.calcite.schema.DruidSchemaCatalog) PlannerFactory(org.apache.druid.sql.calcite.planner.PlannerFactory) DataSegment(org.apache.druid.timeline.DataSegment) BeforeClass(org.junit.BeforeClass)

Example 2 with SegmentGenerator

use of org.apache.druid.segment.generator.SegmentGenerator in project druid by druid-io.

the class QueryRunnerBasedOnClusteredClientTestBase method setupTestBase.

@Before
public void setupTestBase() {
    segmentGenerator = new SegmentGenerator();
    httpClient = new TestHttpClient(objectMapper);
    simpleServerView = new SimpleServerView(toolChestWarehouse, objectMapper, httpClient);
    cachingClusteredClient = new CachingClusteredClient(toolChestWarehouse, simpleServerView, MapCache.create(0), objectMapper, new ForegroundCachePopulator(objectMapper, new CachePopulatorStats(), 0), new CacheConfig(), new DruidHttpClientConfig(), QueryStackTests.getProcessingConfig(USE_PARALLEL_MERGE_POOL_CONFIGURED, DruidProcessingConfig.DEFAULT_NUM_MERGE_BUFFERS), ForkJoinPool.commonPool(), QueryStackTests.DEFAULT_NOOP_SCHEDULER, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new NoopServiceEmitter());
    servers = new ArrayList<>();
}
Also used : SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) CachingClusteredClient(org.apache.druid.client.CachingClusteredClient) CachePopulatorStats(org.apache.druid.client.cache.CachePopulatorStats) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) ForegroundCachePopulator(org.apache.druid.client.cache.ForegroundCachePopulator) CacheConfig(org.apache.druid.client.cache.CacheConfig) MapJoinableFactory(org.apache.druid.segment.join.MapJoinableFactory) SimpleServerView(org.apache.druid.client.SimpleServerView) TestHttpClient(org.apache.druid.client.TestHttpClient) DruidHttpClientConfig(org.apache.druid.guice.http.DruidHttpClientConfig) Before(org.junit.Before)

Example 3 with SegmentGenerator

use of org.apache.druid.segment.generator.SegmentGenerator in project druid by druid-io.

the class ExpressionVectorSelectorsTest method setupClass.

@BeforeClass
public static void setupClass() {
    CLOSER = Closer.create();
    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
    final DataSegment dataSegment = DataSegment.builder().dataSource("foo").interval(schemaInfo.getDataInterval()).version("1").shardSpec(new LinearShardSpec(0)).size(0).build();
    final SegmentGenerator segmentGenerator = CLOSER.register(new SegmentGenerator());
    INDEX = CLOSER.register(segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, ROWS_PER_SEGMENT));
}
Also used : SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) LinearShardSpec(org.apache.druid.timeline.partition.LinearShardSpec) GeneratorSchemaInfo(org.apache.druid.segment.generator.GeneratorSchemaInfo) DataSegment(org.apache.druid.timeline.DataSegment) BeforeClass(org.junit.BeforeClass)

Example 4 with SegmentGenerator

use of org.apache.druid.segment.generator.SegmentGenerator in project druid by druid-io.

the class CachingClusteredClientBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    final String schemaName = "basic";
    parallelCombine = parallelism > 0;
    GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schemaName);
    Map<DataSegment, QueryableIndex> queryableIndexes = Maps.newHashMapWithExpectedSize(numServers);
    for (int i = 0; i < numServers; i++) {
        final DataSegment dataSegment = DataSegment.builder().dataSource(DATA_SOURCE).interval(schemaInfo.getDataInterval()).version("1").shardSpec(new LinearShardSpec(i)).size(0).build();
        final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
        LOG.info("Starting benchmark setup using cacheDir[%s], rows[%,d].", segmentGenerator.getCacheDir(), rowsPerSegment);
        final QueryableIndex index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment);
        queryableIndexes.put(dataSegment, index);
    }
    final DruidProcessingConfig processingConfig = new DruidProcessingConfig() {

        @Override
        public String getFormatString() {
            return null;
        }

        @Override
        public int intermediateComputeSizeBytes() {
            return PROCESSING_BUFFER_SIZE;
        }

        @Override
        public int getNumMergeBuffers() {
            return 1;
        }

        @Override
        public int getNumThreads() {
            return numProcessingThreads;
        }

        @Override
        public boolean useParallelMergePool() {
            return true;
        }
    };
    conglomerate = new DefaultQueryRunnerFactoryConglomerate(ImmutableMap.<Class<? extends Query>, QueryRunnerFactory>builder().put(TimeseriesQuery.class, new TimeseriesQueryRunnerFactory(new TimeseriesQueryQueryToolChest(), new TimeseriesQueryEngine(), QueryRunnerTestHelper.NOOP_QUERYWATCHER)).put(TopNQuery.class, new TopNQueryRunnerFactory(new StupidPool<>("TopNQueryRunnerFactory-bufferPool", () -> ByteBuffer.allocate(PROCESSING_BUFFER_SIZE)), new TopNQueryQueryToolChest(new TopNQueryConfig()), QueryRunnerTestHelper.NOOP_QUERYWATCHER)).put(GroupByQuery.class, makeGroupByQueryRunnerFactory(GroupByQueryRunnerTest.DEFAULT_MAPPER, new GroupByQueryConfig() {

        @Override
        public String getDefaultStrategy() {
            return GroupByStrategySelector.STRATEGY_V2;
        }
    }, processingConfig)).build());
    toolChestWarehouse = new QueryToolChestWarehouse() {

        @Override
        public <T, QueryType extends Query<T>> QueryToolChest<T, QueryType> getToolChest(final QueryType query) {
            return conglomerate.findFactory(query).getToolchest();
        }
    };
    SimpleServerView serverView = new SimpleServerView();
    int serverSuffx = 1;
    for (Entry<DataSegment, QueryableIndex> entry : queryableIndexes.entrySet()) {
        serverView.addServer(createServer(serverSuffx++), entry.getKey(), entry.getValue());
    }
    processingPool = Execs.multiThreaded(processingConfig.getNumThreads(), "caching-clustered-client-benchmark");
    forkJoinPool = new ForkJoinPool((int) Math.ceil(Runtime.getRuntime().availableProcessors() * 0.75), ForkJoinPool.defaultForkJoinWorkerThreadFactory, null, true);
    cachingClusteredClient = new CachingClusteredClient(toolChestWarehouse, serverView, MapCache.create(0), JSON_MAPPER, new ForegroundCachePopulator(JSON_MAPPER, new CachePopulatorStats(), 0), new CacheConfig(), new DruidHttpClientConfig(), processingConfig, forkJoinPool, QueryStackTests.DEFAULT_NOOP_SCHEDULER, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new NoopServiceEmitter());
}
Also used : TimeseriesQuery(org.apache.druid.query.timeseries.TimeseriesQuery) TopNQuery(org.apache.druid.query.topn.TopNQuery) Query(org.apache.druid.query.Query) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) LinearShardSpec(org.apache.druid.timeline.partition.LinearShardSpec) TimeseriesQueryQueryToolChest(org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest) TopNQueryQueryToolChest(org.apache.druid.query.topn.TopNQueryQueryToolChest) GroupByQueryQueryToolChest(org.apache.druid.query.groupby.GroupByQueryQueryToolChest) TimeseriesQueryQueryToolChest(org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest) QueryToolChest(org.apache.druid.query.QueryToolChest) DataSegment(org.apache.druid.timeline.DataSegment) DruidHttpClientConfig(org.apache.druid.guice.http.DruidHttpClientConfig) SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) TimeseriesQueryEngine(org.apache.druid.query.timeseries.TimeseriesQueryEngine) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) CachePopulatorStats(org.apache.druid.client.cache.CachePopulatorStats) TopNQueryRunnerFactory(org.apache.druid.query.topn.TopNQueryRunnerFactory) TopNQueryQueryToolChest(org.apache.druid.query.topn.TopNQueryQueryToolChest) QueryToolChestWarehouse(org.apache.druid.query.QueryToolChestWarehouse) CacheConfig(org.apache.druid.client.cache.CacheConfig) MapJoinableFactory(org.apache.druid.segment.join.MapJoinableFactory) CachingClusteredClient(org.apache.druid.client.CachingClusteredClient) TimeseriesQuery(org.apache.druid.query.timeseries.TimeseriesQuery) GroupByQueryConfig(org.apache.druid.query.groupby.GroupByQueryConfig) GeneratorSchemaInfo(org.apache.druid.segment.generator.GeneratorSchemaInfo) DefaultQueryRunnerFactoryConglomerate(org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) TimeseriesQueryRunnerFactory(org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory) TopNQueryRunnerFactory(org.apache.druid.query.topn.TopNQueryRunnerFactory) QueryRunnerFactory(org.apache.druid.query.QueryRunnerFactory) GroupByQueryRunnerFactory(org.apache.druid.query.groupby.GroupByQueryRunnerFactory) TimeseriesQueryRunnerFactory(org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory) TopNQueryConfig(org.apache.druid.query.topn.TopNQueryConfig) QueryableIndex(org.apache.druid.segment.QueryableIndex) StupidPool(org.apache.druid.collections.StupidPool) DruidProcessingConfig(org.apache.druid.query.DruidProcessingConfig) ForegroundCachePopulator(org.apache.druid.client.cache.ForegroundCachePopulator) ForkJoinPool(java.util.concurrent.ForkJoinPool) Setup(org.openjdk.jmh.annotations.Setup)

Example 5 with SegmentGenerator

use of org.apache.druid.segment.generator.SegmentGenerator in project druid by druid-io.

the class SqlBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("basic");
    final DataSegment dataSegment = DataSegment.builder().dataSource("foo").interval(schemaInfo.getDataInterval()).version("1").shardSpec(new LinearShardSpec(0)).size(0).build();
    final PlannerConfig plannerConfig = new PlannerConfig();
    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
    log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].", segmentGenerator.getCacheDir(), rowsPerSegment);
    final QueryableIndex index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment);
    final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(closer);
    final SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add(dataSegment, index);
    closer.register(walker);
    final DruidSchemaCatalog rootSchema = CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
    plannerFactory = new PlannerFactory(rootSchema, CalciteTests.createMockQueryMakerFactory(walker, conglomerate), createOperatorTable(), CalciteTests.createExprMacroTable(), plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER, CalciteTests.getJsonMapper(), CalciteTests.DRUID_SCHEMA_NAME);
}
Also used : SegmentGenerator(org.apache.druid.segment.generator.SegmentGenerator) QueryRunnerFactoryConglomerate(org.apache.druid.query.QueryRunnerFactoryConglomerate) SpecificSegmentsQuerySegmentWalker(org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker) LinearShardSpec(org.apache.druid.timeline.partition.LinearShardSpec) QueryableIndex(org.apache.druid.segment.QueryableIndex) GeneratorSchemaInfo(org.apache.druid.segment.generator.GeneratorSchemaInfo) PlannerConfig(org.apache.druid.sql.calcite.planner.PlannerConfig) DruidSchemaCatalog(org.apache.druid.sql.calcite.schema.DruidSchemaCatalog) PlannerFactory(org.apache.druid.sql.calcite.planner.PlannerFactory) DataSegment(org.apache.druid.timeline.DataSegment) Setup(org.openjdk.jmh.annotations.Setup)

Aggregations

SegmentGenerator (org.apache.druid.segment.generator.SegmentGenerator)13 GeneratorSchemaInfo (org.apache.druid.segment.generator.GeneratorSchemaInfo)12 DataSegment (org.apache.druid.timeline.DataSegment)12 LinearShardSpec (org.apache.druid.timeline.partition.LinearShardSpec)12 Setup (org.openjdk.jmh.annotations.Setup)8 QueryableIndex (org.apache.druid.segment.QueryableIndex)5 PlannerConfig (org.apache.druid.sql.calcite.planner.PlannerConfig)4 PlannerFactory (org.apache.druid.sql.calcite.planner.PlannerFactory)4 DruidSchemaCatalog (org.apache.druid.sql.calcite.schema.DruidSchemaCatalog)4 SpecificSegmentsQuerySegmentWalker (org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker)4 QueryRunnerFactoryConglomerate (org.apache.druid.query.QueryRunnerFactoryConglomerate)3 BeforeClass (org.junit.BeforeClass)3 CachingClusteredClient (org.apache.druid.client.CachingClusteredClient)2 CacheConfig (org.apache.druid.client.cache.CacheConfig)2 CachePopulatorStats (org.apache.druid.client.cache.CachePopulatorStats)2 ForegroundCachePopulator (org.apache.druid.client.cache.ForegroundCachePopulator)2 DruidHttpClientConfig (org.apache.druid.guice.http.DruidHttpClientConfig)2 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)2 MapJoinableFactory (org.apache.druid.segment.join.MapJoinableFactory)2 NoopServiceEmitter (org.apache.druid.server.metrics.NoopServiceEmitter)2