Search in sources :

Example 11 with RegexDimFilter

use of io.druid.query.filter.RegexDimFilter in project druid by druid-io.

the class FilteredAggregatorTest method testAggregateWithPredicateFilters.

@Test
public void testAggregateWithPredicateFilters() {
    final float[] values = { 0.15f, 0.27f };
    TestFloatColumnSelector selector;
    FilteredAggregatorFactory factory;
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new BoundDimFilter("dim", "a", "a", false, false, true, null, StringComparators.ALPHANUMERIC));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new RegexDimFilter("dim", "a", null));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("a", true), null));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    String jsFn = "function(x) { return(x === 'a') }";
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new JavaScriptDimFilter("dim", jsFn, null, JavaScriptConfig.getEnabledInstance()));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
}
Also used : RegexDimFilter(io.druid.query.filter.RegexDimFilter) BoundDimFilter(io.druid.query.filter.BoundDimFilter) ContainsSearchQuerySpec(io.druid.query.search.search.ContainsSearchQuerySpec) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) Test(org.junit.Test)

Example 12 with RegexDimFilter

use of io.druid.query.filter.RegexDimFilter in project druid by druid-io.

the class FilteredAggregatorBenchmark method setup.

@Setup
public void setup() throws IOException {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    if (ComplexMetrics.getSerdeForType("hyperUnique") == null) {
        ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde(HyperLogLogHash.getDefault()));
    }
    schemaInfo = BenchmarkSchemas.SCHEMA_MAP.get(schema);
    BenchmarkDataGenerator gen = new BenchmarkDataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval(), rowsPerSegment);
    incIndex = makeIncIndex(schemaInfo.getAggsArray());
    filter = new OrDimFilter(Arrays.asList(new BoundDimFilter("dimSequential", "-1", "-1", true, true, null, null, StringComparators.ALPHANUMERIC), new JavaScriptDimFilter("dimSequential", "function(x) { return false }", null, JavaScriptConfig.getEnabledInstance()), new RegexDimFilter("dimSequential", "X", null), new SearchQueryDimFilter("dimSequential", new ContainsSearchQuerySpec("X", false), null), new InDimFilter("dimSequential", Arrays.asList("X"), null)));
    filteredMetrics = new AggregatorFactory[1];
    filteredMetrics[0] = new FilteredAggregatorFactory(new CountAggregatorFactory("rows"), filter);
    incIndexFilteredAgg = makeIncIndex(filteredMetrics);
    inputRows = new ArrayList<>();
    for (int j = 0; j < rowsPerSegment; j++) {
        InputRow row = gen.nextRow();
        if (j % 10000 == 0) {
            log.info(j + " rows generated.");
        }
        incIndex.add(row);
        inputRows.add(row);
    }
    tmpDir = Files.createTempDir();
    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
    indexFile = INDEX_MERGER_V9.persist(incIndex, tmpDir, new IndexSpec());
    qIndex = INDEX_IO.loadIndex(indexFile);
    factory = new TimeseriesQueryRunnerFactory(new TimeseriesQueryQueryToolChest(QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator()), new TimeseriesQueryEngine(), QueryBenchmarkUtil.NOOP_QUERYWATCHER);
    BenchmarkSchemaInfo basicSchema = BenchmarkSchemas.SCHEMA_MAP.get("basic");
    QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Arrays.asList(basicSchema.getDataInterval()));
    List<AggregatorFactory> queryAggs = new ArrayList<>();
    queryAggs.add(filteredMetrics[0]);
    query = Druids.newTimeseriesQueryBuilder().dataSource("blah").granularity(Granularities.ALL).intervals(intervalSpec).aggregators(queryAggs).descending(false).build();
}
Also used : FilteredAggregatorFactory(io.druid.query.aggregation.FilteredAggregatorFactory) RegexDimFilter(io.druid.query.filter.RegexDimFilter) IndexSpec(io.druid.segment.IndexSpec) BoundDimFilter(io.druid.query.filter.BoundDimFilter) ContainsSearchQuerySpec(io.druid.query.search.search.ContainsSearchQuerySpec) BenchmarkDataGenerator(io.druid.benchmark.datagen.BenchmarkDataGenerator) ArrayList(java.util.ArrayList) HyperUniquesSerde(io.druid.query.aggregation.hyperloglog.HyperUniquesSerde) MultipleIntervalSegmentSpec(io.druid.query.spec.MultipleIntervalSegmentSpec) TimeseriesQueryQueryToolChest(io.druid.query.timeseries.TimeseriesQueryQueryToolChest) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) AggregatorFactory(io.druid.query.aggregation.AggregatorFactory) FilteredAggregatorFactory(io.druid.query.aggregation.FilteredAggregatorFactory) TimeseriesQueryEngine(io.druid.query.timeseries.TimeseriesQueryEngine) TimeseriesQueryRunnerFactory(io.druid.query.timeseries.TimeseriesQueryRunnerFactory) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) BenchmarkSchemaInfo(io.druid.benchmark.datagen.BenchmarkSchemaInfo) OrDimFilter(io.druid.query.filter.OrDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) InputRow(io.druid.data.input.InputRow) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) QuerySegmentSpec(io.druid.query.spec.QuerySegmentSpec) Setup(org.openjdk.jmh.annotations.Setup)

Example 13 with RegexDimFilter

use of io.druid.query.filter.RegexDimFilter in project druid by druid-io.

the class GroupByQueryRunnerTest method testBySegmentResultsWithAllFiltersWithExtractionFns.

@Test
public void testBySegmentResultsWithAllFiltersWithExtractionFns() {
    int segmentCount = 32;
    Result<BySegmentResultValue> singleSegmentResult = new Result<BySegmentResultValue>(new DateTime("2011-01-12T00:00:00.000Z"), new BySegmentResultValueClass(Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L)), "testSegment", new Interval("2011-04-02T00:00:00.000Z/2011-04-04T00:00:00.000Z")));
    List<Result> bySegmentResults = Lists.newArrayList();
    for (int i = 0; i < segmentCount; i++) {
        bySegmentResults.add(singleSegmentResult);
    }
    String extractionJsFn = "function(str) { return 'super-' + str; }";
    String jsFn = "function(x) { return(x === 'super-mezzanine') }";
    ExtractionFn extractionFn = new JavaScriptExtractionFn(extractionJsFn, false, JavaScriptConfig.getEnabledInstance());
    List<DimFilter> superFilterList = new ArrayList<>();
    superFilterList.add(new SelectorDimFilter("quality", "super-mezzanine", extractionFn));
    superFilterList.add(new InDimFilter("quality", Arrays.asList("not-super-mezzanine", "FOOBAR", "super-mezzanine"), extractionFn));
    superFilterList.add(new BoundDimFilter("quality", "super-mezzanine", "super-mezzanine", false, false, true, extractionFn, StringComparators.ALPHANUMERIC));
    superFilterList.add(new RegexDimFilter("quality", "super-mezzanine", extractionFn));
    superFilterList.add(new SearchQueryDimFilter("quality", new ContainsSearchQuerySpec("super-mezzanine", true), extractionFn));
    superFilterList.add(new JavaScriptDimFilter("quality", jsFn, extractionFn, JavaScriptConfig.getEnabledInstance()));
    DimFilter superFilter = new AndDimFilter(superFilterList);
    GroupByQuery.Builder builder = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource).setInterval("2011-04-02/2011-04-04").setDimensions(Lists.<DimensionSpec>newArrayList(new DefaultDimensionSpec("quality", "alias"))).setAggregatorSpecs(Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))).setGranularity(new PeriodGranularity(new Period("P1M"), null, null)).setDimFilter(superFilter).setContext(ImmutableMap.<String, Object>of("bySegment", true));
    final GroupByQuery fullQuery = builder.build();
    QueryToolChest toolChest = factory.getToolchest();
    List<QueryRunner<Row>> singleSegmentRunners = Lists.newArrayList();
    for (int i = 0; i < segmentCount; i++) {
        singleSegmentRunners.add(toolChest.preMergeQueryDecoration(runner));
    }
    ExecutorService exec = Executors.newCachedThreadPool();
    QueryRunner theRunner = toolChest.postMergeQueryDecoration(new FinalizeResultsQueryRunner<>(toolChest.mergeResults(factory.mergeRunners(Executors.newCachedThreadPool(), singleSegmentRunners)), toolChest));
    TestHelper.assertExpectedObjects(bySegmentResults, theRunner.run(fullQuery, Maps.newHashMap()), "");
    exec.shutdownNow();
}
Also used : BoundDimFilter(io.druid.query.filter.BoundDimFilter) ContainsSearchQuerySpec(io.druid.query.search.search.ContainsSearchQuerySpec) ArrayList(java.util.ArrayList) LongSumAggregatorFactory(io.druid.query.aggregation.LongSumAggregatorFactory) PeriodGranularity(io.druid.java.util.common.granularity.PeriodGranularity) QueryToolChest(io.druid.query.QueryToolChest) DateTime(org.joda.time.DateTime) Result(io.druid.query.Result) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) BySegmentResultValue(io.druid.query.BySegmentResultValue) RegexDimFilter(io.druid.query.filter.RegexDimFilter) AndDimFilter(io.druid.query.filter.AndDimFilter) JavaScriptExtractionFn(io.druid.query.extraction.JavaScriptExtractionFn) BySegmentResultValueClass(io.druid.query.BySegmentResultValueClass) Period(org.joda.time.Period) DefaultDimensionSpec(io.druid.query.dimension.DefaultDimensionSpec) FinalizeResultsQueryRunner(io.druid.query.FinalizeResultsQueryRunner) QueryRunner(io.druid.query.QueryRunner) LookupExtractionFn(io.druid.query.lookup.LookupExtractionFn) StrlenExtractionFn(io.druid.query.extraction.StrlenExtractionFn) JavaScriptExtractionFn(io.druid.query.extraction.JavaScriptExtractionFn) CascadeExtractionFn(io.druid.query.extraction.CascadeExtractionFn) TimeFormatExtractionFn(io.druid.query.extraction.TimeFormatExtractionFn) ExtractionFn(io.druid.query.extraction.ExtractionFn) RegexDimExtractionFn(io.druid.query.extraction.RegexDimExtractionFn) DimExtractionFn(io.druid.query.extraction.DimExtractionFn) ExecutorService(java.util.concurrent.ExecutorService) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) ExtractionDimFilter(io.druid.query.filter.ExtractionDimFilter) RegexDimFilter(io.druid.query.filter.RegexDimFilter) BoundDimFilter(io.druid.query.filter.BoundDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) OrDimFilter(io.druid.query.filter.OrDimFilter) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) DimFilter(io.druid.query.filter.DimFilter) AndDimFilter(io.druid.query.filter.AndDimFilter) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 14 with RegexDimFilter

use of io.druid.query.filter.RegexDimFilter in project druid by druid-io.

the class IncrementalIndexReadBenchmark method readWithFilters.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readWithFilters(Blackhole blackhole) throws Exception {
    DimFilter filter = new OrDimFilter(Arrays.asList(new BoundDimFilter("dimSequential", "-1", "-1", true, true, null, null, StringComparators.ALPHANUMERIC), new JavaScriptDimFilter("dimSequential", "function(x) { return false }", null, JavaScriptConfig.getEnabledInstance()), new RegexDimFilter("dimSequential", "X", null), new SearchQueryDimFilter("dimSequential", new ContainsSearchQuerySpec("X", false), null), new InDimFilter("dimSequential", Arrays.asList("X"), null)));
    IncrementalIndexStorageAdapter sa = new IncrementalIndexStorageAdapter(incIndex);
    Sequence<Cursor> cursors = makeCursors(sa, filter);
    Cursor cursor = Sequences.toList(Sequences.limit(cursors, 1), Lists.<Cursor>newArrayList()).get(0);
    List<DimensionSelector> selectors = new ArrayList<>();
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimSequential", null)));
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimZipf", null)));
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimUniform", null)));
    selectors.add(cursor.makeDimensionSelector(new DefaultDimensionSpec("dimSequentialHalfNull", null)));
    cursor.reset();
    while (!cursor.isDone()) {
        for (DimensionSelector selector : selectors) {
            IndexedInts row = selector.getRow();
            blackhole.consume(selector.lookupName(row.get(0)));
        }
        cursor.advance();
    }
}
Also used : RegexDimFilter(io.druid.query.filter.RegexDimFilter) DimensionSelector(io.druid.segment.DimensionSelector) BoundDimFilter(io.druid.query.filter.BoundDimFilter) ContainsSearchQuerySpec(io.druid.query.search.search.ContainsSearchQuerySpec) ArrayList(java.util.ArrayList) Cursor(io.druid.segment.Cursor) DefaultDimensionSpec(io.druid.query.dimension.DefaultDimensionSpec) IndexedInts(io.druid.segment.data.IndexedInts) OrDimFilter(io.druid.query.filter.OrDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) IncrementalIndexStorageAdapter(io.druid.segment.incremental.IncrementalIndexStorageAdapter) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) RegexDimFilter(io.druid.query.filter.RegexDimFilter) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) BoundDimFilter(io.druid.query.filter.BoundDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) OrDimFilter(io.druid.query.filter.OrDimFilter) DimFilter(io.druid.query.filter.DimFilter) BenchmarkMode(org.openjdk.jmh.annotations.BenchmarkMode) Benchmark(org.openjdk.jmh.annotations.Benchmark) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit)

Example 15 with RegexDimFilter

use of io.druid.query.filter.RegexDimFilter in project druid by druid-io.

the class TimeFilteringTest method testTimeFilterWithExtractionFn.

@Test
public void testTimeFilterWithExtractionFn() {
    final Map<String, String> stringMap = new HashMap<>();
    stringMap.put("0", "Monday");
    stringMap.put("1", "Tuesday");
    stringMap.put("2", "Wednesday");
    stringMap.put("3", "Thursday");
    stringMap.put("4", "Friday");
    stringMap.put("5", "Saturday");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn exfn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Monday", exfn), ImmutableList.<String>of("0"));
    assertFilterMatches(new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Notaday", exfn), ImmutableList.<String>of());
    assertFilterMatches(new BoundDimFilter(Column.TIME_COLUMN_NAME, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.<String>of("4"));
    assertFilterMatches(new BoundDimFilter(Column.TIME_COLUMN_NAME, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.<String>of());
    assertFilterMatches(new InDimFilter(Column.TIME_COLUMN_NAME, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), ImmutableList.<String>of("1", "5"));
    // test InFilter HashSet implementation
    List<String> bigList = Arrays.asList("Saturday", "Tuesday", "Caturday", "Xanaday", "Vojuday", "Gribaday", "Kipoday", "Dheferday", "Fakeday", "Qeearaday", "Hello", "World", "1", "2", "3", "4", "5", "6", "7");
    assertFilterMatches(new InDimFilter(Column.TIME_COLUMN_NAME, bigList, exfn), ImmutableList.<String>of("1", "5"));
    String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday') }";
    assertFilterMatches(new JavaScriptDimFilter(Column.TIME_COLUMN_NAME, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), ImmutableList.<String>of("2", "3"));
    assertFilterMatches(new RegexDimFilter(Column.TIME_COLUMN_NAME, ".*day", exfn), ImmutableList.<String>of("0", "1", "2", "3", "4", "5"));
    assertFilterMatches(new SearchQueryDimFilter(Column.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("s", true), exfn), ImmutableList.<String>of("1", "2", "3"));
}
Also used : RegexDimFilter(io.druid.query.filter.RegexDimFilter) BoundDimFilter(io.druid.query.filter.BoundDimFilter) HashMap(java.util.HashMap) ContainsSearchQuerySpec(io.druid.query.search.search.ContainsSearchQuerySpec) MapLookupExtractor(io.druid.query.extraction.MapLookupExtractor) LookupExtractor(io.druid.query.lookup.LookupExtractor) LookupExtractionFn(io.druid.query.lookup.LookupExtractionFn) SelectorDimFilter(io.druid.query.filter.SelectorDimFilter) InDimFilter(io.druid.query.filter.InDimFilter) SearchQueryDimFilter(io.druid.query.filter.SearchQueryDimFilter) MapLookupExtractor(io.druid.query.extraction.MapLookupExtractor) JavaScriptDimFilter(io.druid.query.filter.JavaScriptDimFilter) Test(org.junit.Test)

Aggregations

RegexDimFilter (io.druid.query.filter.RegexDimFilter)19 Test (org.junit.Test)17 BoundDimFilter (io.druid.query.filter.BoundDimFilter)12 JavaScriptDimFilter (io.druid.query.filter.JavaScriptDimFilter)12 SearchQueryDimFilter (io.druid.query.filter.SearchQueryDimFilter)12 ContainsSearchQuerySpec (io.druid.query.search.search.ContainsSearchQuerySpec)12 InDimFilter (io.druid.query.filter.InDimFilter)11 SelectorDimFilter (io.druid.query.filter.SelectorDimFilter)9 ArrayList (java.util.ArrayList)7 LookupExtractionFn (io.druid.query.lookup.LookupExtractionFn)5 DefaultDimensionSpec (io.druid.query.dimension.DefaultDimensionSpec)4 OrDimFilter (io.druid.query.filter.OrDimFilter)4 LongSumAggregatorFactory (io.druid.query.aggregation.LongSumAggregatorFactory)3 MapLookupExtractor (io.druid.query.extraction.MapLookupExtractor)3 DimFilter (io.druid.query.filter.DimFilter)3 HashMap (java.util.HashMap)3 Row (io.druid.data.input.Row)2 PeriodGranularity (io.druid.java.util.common.granularity.PeriodGranularity)2 AggregatorFactory (io.druid.query.aggregation.AggregatorFactory)2 CountAggregatorFactory (io.druid.query.aggregation.CountAggregatorFactory)2