Search in sources :

Example 21 with OrDimFilter

use of io.druid.query.filter.OrDimFilter in project druid by druid-io.

the class FilteredAggregatorBenchmark method setup.

@Setup
public void setup() throws IOException {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    if (ComplexMetrics.getSerdeForType("hyperUnique") == null) {
        ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde(HyperLogLogHash.getDefault()));
    }
    schemaInfo = BenchmarkSchemas.SCHEMA_MAP.get(schema);
    BenchmarkDataGenerator gen = new BenchmarkDataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval(), rowsPerSegment);
    incIndex = makeIncIndex(schemaInfo.getAggsArray());
    filter = new OrDimFilter(Arrays.asList(new BoundDimFilter("dimSequential", "-1", "-1", true, true, null, null, StringComparators.ALPHANUMERIC), new JavaScriptDimFilter("dimSequential", "function(x) { return false }", null, JavaScriptConfig.getEnabledInstance()), new RegexDimFilter("dimSequential", "X", null), new SearchQueryDimFilter("dimSequential", new ContainsSearchQuerySpec("X", false), null), new InDimFilter("dimSequential", Arrays.asList("X"), null)));
    filteredMetrics = new AggregatorFactory[1];
    filteredMetrics[0] = new FilteredAggregatorFactory(new CountAggregatorFactory("rows"), filter);
    incIndexFilteredAgg = makeIncIndex(filteredMetrics);
    inputRows = new ArrayList<>();
    for (int j = 0; j < rowsPerSegment; j++) {
        InputRow row = gen.nextRow();
        if (j % 10000 == 0) {
            log.info(j + " rows generated.");
        }
        incIndex.add(row);
        inputRows.add(row);
    }
    tmpDir = Files.createTempDir();
    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
    indexFile = INDEX_MERGER_V9.persist(incIndex, tmpDir, new IndexSpec());
    qIndex = INDEX_IO.loadIndex(indexFile);
    factory = new TimeseriesQueryRunnerFactory(new TimeseriesQueryQueryToolChest(QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator()), new TimeseriesQueryEngine(), QueryBenchmarkUtil.NOOP_QUERYWATCHER);
    BenchmarkSchemaInfo basicSchema = BenchmarkSchemas.SCHEMA_MAP.get("basic");
    QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Arrays.asList(basicSchema.getDataInterval()));
    List<AggregatorFactory> queryAggs = new ArrayList<>();
    queryAggs.add(filteredMetrics[0]);
    query = Druids.newTimeseriesQueryBuilder().dataSource("blah").granularity(Granularities.ALL).intervals(intervalSpec).aggregators(queryAggs).descending(false).build();
}
Also used : FilteredAggregatorFactory(io.druid.query.aggregation.FilteredAggregatorFactory) RegexDimFilter(io.druid.query.filter.RegexDimFilter) IndexSpec(io.druid.segment.IndexSpec) BoundDimFilter(io.druid.query.filter.BoundDimFilter) ContainsSearchQuerySpec(io.druid.query.search.search.ContainsSearchQuerySpec) BenchmarkDataGenerator(io.druid.benchmark.datagen.BenchmarkDataGenerator) ArrayList(java.util.ArrayList) HyperUniquesSerde(io.druid.query.aggregation.hyperloglog.HyperUniquesSerde) MultipleIntervalSegmentSpec(io.druid.query.spec.MultipleIntervalSegmentSpec) TimeseriesQueryQueryToolChest(io.druid.query.timeseries.TimeseriesQueryQueryToolChest) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) AggregatorFactory(io.druid.query.aggregation.AggregatorFactory) FilteredAggregatorFactory(io.druid.query.aggregation.FilteredAggregatorFactory) TimeseriesQueryEngine(io.druid.query.timeseries.TimeseriesQueryEngine) TimeseriesQueryRunnerFactory(io.druid.query.timeseries.TimeseriesQueryRunnerFactory) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) BenchmarkSchemaInfo(io.druid.benchmark.datagen.BenchmarkSchemaInfo) OrDimFilter(io.druid.query.filter.OrDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) InputRow(io.druid.data.input.InputRow) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) QuerySegmentSpec(io.druid.query.spec.QuerySegmentSpec) Setup(org.openjdk.jmh.annotations.Setup)

Example 22 with OrDimFilter

use of io.druid.query.filter.OrDimFilter in project druid by druid-io.

the class FilterPartitionBenchmark method readComplexOrFilterCNF.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readComplexOrFilterCNF(Blackhole blackhole) throws Exception {
    DimFilter dimFilter1 = new OrDimFilter(Arrays.<DimFilter>asList(new SelectorDimFilter("dimSequential", "199", null), new AndDimFilter(Arrays.<DimFilter>asList(new NoBitmapSelectorDimFilter("dimMultivalEnumerated2", "Corundum", null), new SelectorDimFilter("dimMultivalEnumerated", "Bar", null)))));
    DimFilter dimFilter2 = new OrDimFilter(Arrays.<DimFilter>asList(new SelectorDimFilter("dimSequential", "299", null), new SelectorDimFilter("dimSequential", "399", null), new AndDimFilter(Arrays.<DimFilter>asList(new NoBitmapSelectorDimFilter("dimMultivalEnumerated2", "Xylophone", null), new SelectorDimFilter("dimMultivalEnumerated", "Foo", null)))));
    DimFilter dimFilter3 = new OrDimFilter(Arrays.<DimFilter>asList(dimFilter1, dimFilter2, new AndDimFilter(Arrays.<DimFilter>asList(new NoBitmapSelectorDimFilter("dimMultivalEnumerated2", "Orange", null), new SelectorDimFilter("dimMultivalEnumerated", "World", null)))));
    StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
    Sequence<Cursor> cursors = makeCursors(sa, Filters.convertToCNF(dimFilter3.toFilter()));
    Sequence<List<String>> stringListSeq = readCursors(cursors, blackhole);
    List<String> strings = Sequences.toList(Sequences.limit(stringListSeq, 1), Lists.<List<String>>newArrayList()).get(0);
    for (String st : strings) {
        blackhole.consume(st);
    }
}
Also used : AndDimFilter(io.druid.query.filter.AndDimFilter) QueryableIndexStorageAdapter(io.druid.segment.QueryableIndexStorageAdapter) StorageAdapter(io.druid.segment.StorageAdapter) QueryableIndexStorageAdapter(io.druid.segment.QueryableIndexStorageAdapter) Cursor(io.druid.segment.Cursor) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) OrDimFilter(io.druid.query.filter.OrDimFilter) List(java.util.List) ArrayList(java.util.ArrayList) BoundDimFilter(io.druid.query.filter.BoundDimFilter) OrDimFilter(io.druid.query.filter.OrDimFilter) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) DimFilter(io.druid.query.filter.DimFilter) AndDimFilter(io.druid.query.filter.AndDimFilter) BenchmarkMode(org.openjdk.jmh.annotations.BenchmarkMode) Benchmark(org.openjdk.jmh.annotations.Benchmark) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit)

Example 23 with OrDimFilter

use of io.druid.query.filter.OrDimFilter in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByTimeExtractionNamedUnderUnderTime.

@Test
public void testGroupByTimeExtractionNamedUnderUnderTime() {
    expectedException.expect(IAE.class);
    expectedException.expectMessage("'__time' cannot be used as an output name for dimensions, aggregators, or post-aggregators.");
    GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource).setQuerySegmentSpec(QueryRunnerTestHelper.fullOnInterval).setDimensions(Lists.newArrayList(new DefaultDimensionSpec("market", "market"), new ExtractionDimensionSpec(Column.TIME_COLUMN_NAME, Column.TIME_COLUMN_NAME, new TimeFormatExtractionFn("EEEE", null, null, null, false)))).setAggregatorSpecs(Arrays.asList(QueryRunnerTestHelper.rowsCount, QueryRunnerTestHelper.indexDoubleSum)).setPostAggregatorSpecs(Arrays.<PostAggregator>asList(QueryRunnerTestHelper.addRowsIndexConstant)).setGranularity(QueryRunnerTestHelper.allGran).setDimFilter(new OrDimFilter(Arrays.<DimFilter>asList(new SelectorDimFilter("market", "spot", null), new SelectorDimFilter("market", "upfront", null)))).setLimitSpec(new DefaultLimitSpec(ImmutableList.<OrderByColumnSpec>of(), 1)).build();
}
Also used : TimeFormatExtractionFn(io.druid.query.extraction.TimeFormatExtractionFn) DefaultLimitSpec(io.druid.query.groupby.orderby.DefaultLimitSpec) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) OrDimFilter(io.druid.query.filter.OrDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) ExtractionDimFilter(io.druid.query.filter.ExtractionDimFilter) RegexDimFilter(io.druid.query.filter.RegexDimFilter) BoundDimFilter(io.druid.query.filter.BoundDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) OrDimFilter(io.druid.query.filter.OrDimFilter) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) DimFilter(io.druid.query.filter.DimFilter) AndDimFilter(io.druid.query.filter.AndDimFilter) DefaultDimensionSpec(io.druid.query.dimension.DefaultDimensionSpec) ExtractionDimensionSpec(io.druid.query.dimension.ExtractionDimensionSpec) Test(org.junit.Test)

Aggregations

OrDimFilter (io.druid.query.filter.OrDimFilter)23 DimFilter (io.druid.query.filter.DimFilter)18 AndDimFilter (io.druid.query.filter.AndDimFilter)17 SelectorDimFilter (io.druid.query.filter.SelectorDimFilter)15 BoundDimFilter (io.druid.query.filter.BoundDimFilter)13 Test (org.junit.Test)12 List (java.util.List)8 DefaultDimensionSpec (io.druid.query.dimension.DefaultDimensionSpec)7 ExtractionDimensionSpec (io.druid.query.dimension.ExtractionDimensionSpec)6 InDimFilter (io.druid.query.filter.InDimFilter)6 NotDimFilter (io.druid.query.filter.NotDimFilter)6 ArrayList (java.util.ArrayList)6 Row (io.druid.data.input.Row)5 JavaScriptDimFilter (io.druid.query.filter.JavaScriptDimFilter)5 RegexDimFilter (io.druid.query.filter.RegexDimFilter)5 SearchQueryDimFilter (io.druid.query.filter.SearchQueryDimFilter)5 TimeFormatExtractionFn (io.druid.query.extraction.TimeFormatExtractionFn)4 LongSumAggregatorFactory (io.druid.query.aggregation.LongSumAggregatorFactory)3 DimensionSpec (io.druid.query.dimension.DimensionSpec)3 ListFilteredDimensionSpec (io.druid.query.dimension.ListFilteredDimensionSpec)3