Examples with BoundDimFilter io.druid.query.filter.BoundDimFilter used on opensource projects

Example 11 with BoundDimFilter

use of io.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class CachingClusteredClientTest method testTimeSeriesWithFilter.

@Test
public void testTimeSeriesWithFilter() throws Exception {
    DimFilter filter = Druids.newAndDimFilterBuilder().fields(Arrays.asList(Druids.newOrDimFilterBuilder().fields(Arrays.asList(new SelectorDimFilter("dim0", "1", null), new BoundDimFilter("dim0", "222", "333", false, false, false, null, StringComparators.LEXICOGRAPHIC))).build(), Druids.newAndDimFilterBuilder().fields(Arrays.asList(new InDimFilter("dim1", Arrays.asList("0", "1", "2", "3", "4"), null), new BoundDimFilter("dim1", "0", "3", false, true, false, null, StringComparators.LEXICOGRAPHIC), new BoundDimFilter("dim1", "1", "9999", true, false, false, null, StringComparators.LEXICOGRAPHIC))).build())).build();
    final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder().dataSource(DATA_SOURCE).intervals(SEG_SPEC).filters(filter).granularity(GRANULARITY).aggregators(AGGS).postAggregators(POST_AGGS).context(CONTEXT);
    QueryRunner runner = new FinalizeResultsQueryRunner(client, new TimeseriesQueryQueryToolChest(QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()));
    /*
    For dim0 (2011-01-01/2011-01-05), the combined range is {[1,1], [222,333]}, so segments [-inf,1], [1,2], [2,3], and
    [3,4] is needed
    For dim1 (2011-01-06/2011-01-10), the combined range for the bound filters is {(1,3)}, combined this with the in
    filter result in {[2,2]}, so segments [1,2] and [2,3] is needed
    */
    List<Iterable<Result<TimeseriesResultValue>>> expectedResult = Arrays.asList(makeTimeResults(new DateTime("2011-01-01"), 50, 5000, new DateTime("2011-01-02"), 10, 1252, new DateTime("2011-01-03"), 20, 6213, new DateTime("2011-01-04"), 30, 743), makeTimeResults(new DateTime("2011-01-07"), 60, 6020, new DateTime("2011-01-08"), 70, 250));
    testQueryCachingWithFilter(runner, 3, builder.build(), expectedResult, new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000), new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-02"), 10, 1252), new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-03"), 20, 6213), new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-04"), 30, 743), new Interval("2011-01-01/2011-01-05"), makeTimeResults(new DateTime("2011-01-05"), 40, 6000), new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-06"), 50, 425), new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-07"), 60, 6020), new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-08"), 70, 250), new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-09"), 23, 85312), new Interval("2011-01-06/2011-01-10"), makeTimeResults(new DateTime("2011-01-10"), 100, 512));
}

Example 12 with BoundDimFilter

use of io.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class CombineAndSimplifyBounds method doSimplify.

/**
   * Simplify BoundDimFilters that are children of an OR or an AND.
   *
   * @param children    the filters
   * @param disjunction true for disjunction, false for conjunction
   *
   * @return simplified filters
   */
private static DimFilter doSimplify(final List<DimFilter> children, boolean disjunction) {
    // Copy children list
    final List<DimFilter> newChildren = Lists.newArrayList(children);
    // Group Bound filters by dimension, extractionFn, and comparator and compute a RangeSet for each one.
    final Map<BoundRefKey, List<BoundDimFilter>> bounds = Maps.newHashMap();
    final Iterator<DimFilter> iterator = newChildren.iterator();
    while (iterator.hasNext()) {
        final DimFilter child = iterator.next();
        if (child.equals(Filtration.matchNothing())) {
            // AND with FALSE => always false, short circuit
            if (disjunction) {
                iterator.remove();
            } else {
                return Filtration.matchNothing();
            }
        } else if (child.equals(Filtration.matchEverything())) {
            // AND with TRUE => ignore
            if (disjunction) {
                return Filtration.matchEverything();
            } else {
                iterator.remove();
            }
        } else if (child instanceof BoundDimFilter) {
            final BoundDimFilter bound = (BoundDimFilter) child;
            final BoundRefKey boundRefKey = BoundRefKey.from(bound);
            List<BoundDimFilter> filterList = bounds.get(boundRefKey);
            if (filterList == null) {
                filterList = Lists.newArrayList();
                bounds.put(boundRefKey, filterList);
            }
            filterList.add(bound);
        }
    }
    // Try to simplify filters within each group.
    for (Map.Entry<BoundRefKey, List<BoundDimFilter>> entry : bounds.entrySet()) {
        final BoundRefKey boundRefKey = entry.getKey();
        final List<BoundDimFilter> filterList = entry.getValue();
        // Create a RangeSet for this group.
        final RangeSet<BoundValue> rangeSet = disjunction ? RangeSets.unionRanges(Bounds.toRanges(filterList)) : RangeSets.intersectRanges(Bounds.toRanges(filterList));
        if (rangeSet.asRanges().size() < filterList.size()) {
            // We found a simplification. Remove the old filters and add new ones.
            for (final BoundDimFilter bound : filterList) {
                if (!newChildren.remove(bound)) {
                    throw new ISE("WTF?! Tried to remove bound but couldn't?");
                }
            }
            if (rangeSet.asRanges().isEmpty()) {
                // AND with FALSE => always false, short circuit
                if (disjunction) {
                    newChildren.add(Filtration.matchNothing());
                } else {
                    return Filtration.matchNothing();
                }
            }
            for (final Range<BoundValue> range : rangeSet.asRanges()) {
                if (!range.hasLowerBound() && !range.hasUpperBound()) {
                    // OR with TRUE => always true; short circuit
                    if (disjunction) {
                        return Filtration.matchEverything();
                    } else {
                        newChildren.add(Filtration.matchEverything());
                    }
                } else {
                    newChildren.add(Bounds.toFilter(boundRefKey, range));
                }
            }
        }
    }
    Preconditions.checkState(newChildren.size() > 0, "newChildren.size > 0");
    if (newChildren.size() == 1) {
        return newChildren.get(0);
    } else {
        return disjunction ? new OrDimFilter(newChildren) : new AndDimFilter(newChildren);
    }
}

Example 13 with BoundDimFilter

use of io.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class FilteredAggregatorTest method testAggregateWithExtractionFns.

@Test
public void testAggregateWithExtractionFns() {
    final float[] values = { 0.15f, 0.27f };
    TestFloatColumnSelector selector;
    FilteredAggregatorFactory factory;
    String extractionJsFn = "function(str) { return str + 'AARDVARK'; }";
    ExtractionFn extractionFn = new JavaScriptExtractionFn(extractionJsFn, false, JavaScriptConfig.getEnabledInstance());
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SelectorDimFilter("dim", "aAARDVARK", extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new InDimFilter("dim", Arrays.asList("NOT-aAARDVARK", "FOOBAR", "aAARDVARK"), extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new BoundDimFilter("dim", "aAARDVARK", "aAARDVARK", false, false, true, extractionFn, StringComparators.ALPHANUMERIC));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new RegexDimFilter("dim", "aAARDVARK", extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("aAARDVARK", true), extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    String jsFn = "function(x) { return(x === 'aAARDVARK') }";
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new JavaScriptDimFilter("dim", jsFn, extractionFn, JavaScriptConfig.getEnabledInstance()));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
}

Example 14 with BoundDimFilter

use of io.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByWithAllFiltersOnNullDimsWithExtractionFns.

@Test
public void testGroupByWithAllFiltersOnNullDimsWithExtractionFns() {
    Map<String, String> extractionMap = new HashMap<>();
    extractionMap.put("", "EMPTY");
    extractionMap.put(null, "EMPTY");
    MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false);
    LookupExtractionFn extractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true);
    String jsFn = "function(x) { return(x === 'EMPTY') }";
    List<DimFilter> superFilterList = new ArrayList<>();
    superFilterList.add(new SelectorDimFilter("null_column", "EMPTY", extractionFn));
    superFilterList.add(new InDimFilter("null_column", Arrays.asList("NOT-EMPTY", "FOOBAR", "EMPTY"), extractionFn));
    superFilterList.add(new BoundDimFilter("null_column", "EMPTY", "EMPTY", false, false, true, extractionFn, StringComparators.ALPHANUMERIC));
    superFilterList.add(new RegexDimFilter("null_column", "EMPTY", extractionFn));
    superFilterList.add(new SearchQueryDimFilter("null_column", new ContainsSearchQuerySpec("EMPTY", true), extractionFn));
    superFilterList.add(new JavaScriptDimFilter("null_column", jsFn, extractionFn, JavaScriptConfig.getEnabledInstance()));
    DimFilter superFilter = new AndDimFilter(superFilterList);
    GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource).setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird).setDimensions(Lists.<DimensionSpec>newArrayList(new DefaultDimensionSpec("null_column", "alias"))).setAggregatorSpecs(Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))).setGranularity(QueryRunnerTestHelper.dayGran).setDimFilter(superFilter).build();
    List<Row> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 13L, "idx", 6619L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", null, "rows", 13L, "idx", 5827L));
    Iterable<Row> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "");
}

Example 15 with BoundDimFilter

use of io.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class SelectQueryRunnerTest method testFullOnSelectWithFilterOnVirtualColumn.

@Test
public void testFullOnSelectWithFilterOnVirtualColumn() {
    SelectQuery query = newTestQuery().intervals("2011-01-13/2011-01-14").filters(new AndDimFilter(Arrays.asList(new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot", null), new BoundDimFilter("expr", "11.1", null, false, false, null, null, StringComparators.NUMERIC)))).granularity(QueryRunnerTestHelper.allGran).dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.qualityDimension)).metrics(Lists.<String>newArrayList(QueryRunnerTestHelper.indexMetric)).pagingSpec(new PagingSpec(null, 10, true)).virtualColumns(new ExpressionVirtualColumn("expr", "index / 10.0")).build();
    HashMap<String, Object> context = new HashMap<String, Object>();
    Iterable<Result<SelectResultValue>> results = Sequences.toList(runner.run(query, context), Lists.<Result<SelectResultValue>>newArrayList());
    final List<List<Map<String, Object>>> events = toEvents(new String[] { EventHolder.timestampKey + ":TIME", null, QueryRunnerTestHelper.qualityDimension + ":STRING", null, null, QueryRunnerTestHelper.indexMetric + ":FLOAT" }, // filtered values with all granularity
    new String[] { "2011-01-13T00:00:00.000Z\tspot\thealth\tpreferred\th
preferred\t114.947403", "2011-01-13T00:00:00.000Z\tspot\ttechnology\tpreferred\tt
preferred\t111.356672" });
    PagingOffset offset = query.getPagingOffset(QueryRunnerTestHelper.segmentId);
    List<Result<SelectResultValue>> expectedResults = toExpected(events, Lists.newArrayList("quality"), Lists.<String>newArrayList("index"), offset.startOffset(), offset.threshold());
    verify(expectedResults, results);
}