Examples with BoundDimFilter org.apache.druid.query.filter.BoundDimFilter used on opensource projects

Example 61 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class BoundFilterTest method testLexicographicMatchNull.

@Test
public void testLexicographicMatchNull() {
    assertFilterMatches(new BoundDimFilter("dim0", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of());
    assertFilterMatches(new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0"));
    if (NullHandling.replaceWithDefault()) {
        assertFilterMatches(new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("1", "2", "5"));
    } else {
        assertFilterMatches(new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("2"));
    }
}

Example 62 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class CachingClusteredClientTest method testTimeSeriesWithFilter.

@Test
public void testTimeSeriesWithFilter() {
    DimFilter filter = new AndDimFilter(new OrDimFilter(new SelectorDimFilter("dim0", "1", null), new BoundDimFilter("dim0", "222", "333", false, false, false, null, StringComparators.LEXICOGRAPHIC)), new AndDimFilter(new InDimFilter("dim1", Arrays.asList("0", "1", "2", "3", "4"), null), new BoundDimFilter("dim1", "0", "3", false, true, false, null, StringComparators.LEXICOGRAPHIC), new BoundDimFilter("dim1", "1", "9999", true, false, false, null, StringComparators.LEXICOGRAPHIC)));
    final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder().dataSource(DATA_SOURCE).intervals(SEG_SPEC).filters(filter).granularity(GRANULARITY).aggregators(AGGS).postAggregators(POST_AGGS).context(CONTEXT);
    QueryRunner runner = new FinalizeResultsQueryRunner(getDefaultQueryRunner(), new TimeseriesQueryQueryToolChest());
    /*
    For dim0 (2011-01-01/2011-01-05), the combined range is {[1,1], [222,333]}, so segments [-inf,1], [1,2], [2,3], and
    [3,4] is needed
    For dim1 (2011-01-06/2011-01-10), the combined range for the bound filters is {(1,3)}, combined this with the in
    filter result in {[2,2]}, so segments [1,2] and [2,3] is needed
    */
    List<Iterable<Result<TimeseriesResultValue>>> expectedResult = Arrays.asList(makeTimeResults(DateTimes.of("2011-01-01"), 50, 5000, DateTimes.of("2011-01-02"), 10, 1252, DateTimes.of("2011-01-03"), 20, 6213, DateTimes.of("2011-01-04"), 30, 743), makeTimeResults(DateTimes.of("2011-01-07"), 60, 6020, DateTimes.of("2011-01-08"), 70, 250));
    testQueryCachingWithFilter(runner, 3, builder.randomQueryId().build(), expectedResult, Intervals.of("2011-01-01/2011-01-05"), makeTimeResults(DateTimes.of("2011-01-01"), 50, 5000), Intervals.of("2011-01-01/2011-01-05"), makeTimeResults(DateTimes.of("2011-01-02"), 10, 1252), Intervals.of("2011-01-01/2011-01-05"), makeTimeResults(DateTimes.of("2011-01-03"), 20, 6213), Intervals.of("2011-01-01/2011-01-05"), makeTimeResults(DateTimes.of("2011-01-04"), 30, 743), Intervals.of("2011-01-01/2011-01-05"), makeTimeResults(DateTimes.of("2011-01-05"), 40, 6000), Intervals.of("2011-01-06/2011-01-10"), makeTimeResults(DateTimes.of("2011-01-06"), 50, 425), Intervals.of("2011-01-06/2011-01-10"), makeTimeResults(DateTimes.of("2011-01-07"), 60, 6020), Intervals.of("2011-01-06/2011-01-10"), makeTimeResults(DateTimes.of("2011-01-08"), 70, 250), Intervals.of("2011-01-06/2011-01-10"), makeTimeResults(DateTimes.of("2011-01-09"), 23, 85312), Intervals.of("2011-01-06/2011-01-10"), makeTimeResults(DateTimes.of("2011-01-10"), 100, 512));
}

Example 63 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class CachingClusteredClientTest method testHashBasedPruningQueryContextEnabledWithPartitionFunctionAndPartitionDimensionsDoSegmentPruning.

@Test
public void testHashBasedPruningQueryContextEnabledWithPartitionFunctionAndPartitionDimensionsDoSegmentPruning() {
    DimFilter filter = new AndDimFilter(new SelectorDimFilter("dim1", "a", null), new BoundDimFilter("dim2", "e", "zzz", true, true, false, null, StringComparators.LEXICOGRAPHIC), // Equivalent filter of dim3 below is InDimFilter("dim3", Arrays.asList("c"), null)
    new AndDimFilter(new InDimFilter("dim3", Arrays.asList("a", "c", "e", "g"), null), new BoundDimFilter("dim3", "aaa", "ddd", false, false, false, null, StringComparators.LEXICOGRAPHIC)));
    final Druids.TimeseriesQueryBuilder builder = Druids.newTimeseriesQueryBuilder().dataSource(DATA_SOURCE).filters(filter).granularity(GRANULARITY).intervals(SEG_SPEC).context(CONTEXT).intervals("2011-01-05/2011-01-10").aggregators(RENAMED_AGGS).postAggregators(RENAMED_POST_AGGS).randomQueryId();
    TimeseriesQuery query = builder.build();
    QueryRunner runner = new FinalizeResultsQueryRunner(getDefaultQueryRunner(), new TimeseriesQueryQueryToolChest());
    final Interval interval1 = Intervals.of("2011-01-06/2011-01-07");
    final Interval interval2 = Intervals.of("2011-01-07/2011-01-08");
    final Interval interval3 = Intervals.of("2011-01-08/2011-01-09");
    final DruidServer lastServer = servers[random.nextInt(servers.length)];
    List<String> partitionDimensions1 = ImmutableList.of("dim1");
    ServerSelector selector1 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 0, 6);
    ServerSelector selector2 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 1, 6);
    ServerSelector selector3 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 2, 6);
    ServerSelector selector4 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 3, 6);
    ServerSelector selector5 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 4, 6);
    ServerSelector selector6 = makeMockHashBasedSelector(lastServer, partitionDimensions1, HashPartitionFunction.MURMUR3_32_ABS, 5, 6);
    List<String> partitionDimensions2 = ImmutableList.of("dim2");
    ServerSelector selector7 = makeMockHashBasedSelector(lastServer, partitionDimensions2, HashPartitionFunction.MURMUR3_32_ABS, 0, 3);
    ServerSelector selector8 = makeMockHashBasedSelector(lastServer, partitionDimensions2, HashPartitionFunction.MURMUR3_32_ABS, 1, 3);
    ServerSelector selector9 = makeMockHashBasedSelector(lastServer, partitionDimensions2, HashPartitionFunction.MURMUR3_32_ABS, 2, 3);
    List<String> partitionDimensions3 = ImmutableList.of("dim1", "dim3");
    ServerSelector selector10 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 0, 4);
    ServerSelector selector11 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 1, 4);
    ServerSelector selector12 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 2, 4);
    ServerSelector selector13 = makeMockHashBasedSelector(lastServer, partitionDimensions3, HashPartitionFunction.MURMUR3_32_ABS, 3, 4);
    timeline.add(interval1, "v", new NumberedPartitionChunk<>(0, 6, selector1));
    timeline.add(interval1, "v", new NumberedPartitionChunk<>(1, 6, selector2));
    timeline.add(interval1, "v", new NumberedPartitionChunk<>(2, 6, selector3));
    timeline.add(interval1, "v", new NumberedPartitionChunk<>(3, 6, selector4));
    timeline.add(interval1, "v", new NumberedPartitionChunk<>(4, 6, selector5));
    timeline.add(interval1, "v", new NumberedPartitionChunk<>(5, 6, selector6));
    timeline.add(interval2, "v", new NumberedPartitionChunk<>(0, 3, selector7));
    timeline.add(interval2, "v", new NumberedPartitionChunk<>(1, 3, selector8));
    timeline.add(interval2, "v", new NumberedPartitionChunk<>(2, 3, selector9));
    timeline.add(interval3, "v", new NumberedPartitionChunk<>(0, 3, selector10));
    timeline.add(interval3, "v", new NumberedPartitionChunk<>(1, 3, selector11));
    timeline.add(interval3, "v", new NumberedPartitionChunk<>(2, 3, selector12));
    timeline.add(interval3, "v", new NumberedPartitionChunk<>(2, 3, selector13));
    final Capture<QueryPlus> capture = Capture.newInstance();
    final Capture<ResponseContext> contextCap = Capture.newInstance();
    QueryRunner mockRunner = EasyMock.createNiceMock(QueryRunner.class);
    EasyMock.expect(mockRunner.run(EasyMock.capture(capture), EasyMock.capture(contextCap))).andReturn(Sequences.empty()).anyTimes();
    EasyMock.expect(serverView.getQueryRunner(lastServer)).andReturn(mockRunner).anyTimes();
    EasyMock.replay(serverView);
    EasyMock.replay(mockRunner);
    List<SegmentDescriptor> expcetedDescriptors = new ArrayList<>();
    // Narrow down to 1 chunk
    expcetedDescriptors.add(new SegmentDescriptor(interval1, "v", 3));
    // Can't filter out any chunks
    expcetedDescriptors.add(new SegmentDescriptor(interval2, "v", 0));
    expcetedDescriptors.add(new SegmentDescriptor(interval2, "v", 1));
    expcetedDescriptors.add(new SegmentDescriptor(interval2, "v", 2));
    // Narrow down to 1 chunk
    expcetedDescriptors.add(new SegmentDescriptor(interval3, "v", 2));
    MultipleSpecificSegmentSpec expected = new MultipleSpecificSegmentSpec(expcetedDescriptors);
    runner.run(QueryPlus.wrap(query)).toList();
    Assert.assertEquals(expected, ((TimeseriesQuery) capture.getValue().getQuery()).getQuerySegmentSpec());
}

Example 64 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class MoveTimeFiltersToIntervals method extractConvertibleTimeBounds.

/**
 * Extract bound filters on __time that can be converted to query-level "intervals".
 *
 * @return pair of new dimFilter + RangeSet of __time that should be ANDed together. Either can be null but not both.
 */
private static Pair<DimFilter, RangeSet<Long>> extractConvertibleTimeBounds(final DimFilter filter) {
    if (filter instanceof AndDimFilter) {
        final List<DimFilter> children = ((AndDimFilter) filter).getFields();
        final List<DimFilter> newChildren = new ArrayList<>();
        final List<RangeSet<Long>> rangeSets = new ArrayList<>();
        for (DimFilter child : children) {
            final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
            if (pair.lhs != null) {
                newChildren.add(pair.lhs);
            }
            if (pair.rhs != null) {
                rangeSets.add(pair.rhs);
            }
        }
        final DimFilter newFilter;
        if (newChildren.size() == 0) {
            newFilter = null;
        } else if (newChildren.size() == 1) {
            newFilter = newChildren.get(0);
        } else {
            newFilter = new AndDimFilter(newChildren);
        }
        return Pair.of(newFilter, rangeSets.isEmpty() ? null : RangeSets.intersectRangeSets(rangeSets));
    } else if (filter instanceof OrDimFilter) {
        final List<DimFilter> children = ((OrDimFilter) filter).getFields();
        final List<RangeSet<Long>> rangeSets = new ArrayList<>();
        boolean allCompletelyConverted = true;
        boolean allHadIntervals = true;
        for (DimFilter child : children) {
            final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
            if (pair.lhs != null) {
                allCompletelyConverted = false;
            }
            if (pair.rhs != null) {
                rangeSets.add(pair.rhs);
            } else {
                allHadIntervals = false;
            }
        }
        if (allCompletelyConverted) {
            return Pair.of(null, RangeSets.unionRangeSets(rangeSets));
        } else {
            return Pair.of(filter, allHadIntervals ? RangeSets.unionRangeSets(rangeSets) : null);
        }
    } else if (filter instanceof NotDimFilter) {
        final DimFilter child = ((NotDimFilter) filter).getField();
        final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
        if (pair.rhs != null && pair.lhs == null) {
            return Pair.of(null, pair.rhs.complement());
        } else {
            return Pair.of(filter, null);
        }
    } else if (filter instanceof BoundDimFilter) {
        final BoundDimFilter bound = (BoundDimFilter) filter;
        if (BoundRefKey.from(bound).equals(TIME_BOUND_REF_KEY)) {
            return Pair.of(null, RangeSets.of(toLongRange(Bounds.toRange(bound))));
        } else {
            return Pair.of(filter, null);
        }
    } else {
        return Pair.of(filter, null);
    }
}

Example 65 with BoundDimFilter

use of org.apache.druid.query.filter.BoundDimFilter in project druid by druid-io.

the class CalciteJoinQueryTest method testLeftJoinRightTableCanBeEmpty.

@Test
public void testLeftJoinRightTableCanBeEmpty() throws Exception {
    // HashJoinSegmentStorageAdapter is not vectorizable
    cannotVectorize();
    final DataSource rightTable;
    if (useDefault) {
        rightTable = InlineDataSource.fromIterable(ImmutableList.of(), RowSignature.builder().add("dim2", ColumnType.STRING).add("m2", ColumnType.DOUBLE).build());
    } else {
        rightTable = new QueryDataSource(Druids.newScanQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST).filters(new SelectorDimFilter("m2", null, null)).columns("dim2").legacy(false).build());
    }
    testQuery("SELECT v1.dim2, count(1) " + "FROM (SELECT * FROM foo where m1 > 2) v1 " + "LEFT OUTER JOIN (" + "  select dim2 from (select * from foo where m2 is null)" + ") sm ON v1.dim2 = sm.dim2 " + "group by 1", ImmutableList.of(new GroupByQuery.Builder().setDataSource(JoinDataSource.create(new QueryDataSource(Druids.newScanQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST).filters(new BoundDimFilter("m1", "2", null, true, false, null, null, StringComparators.NUMERIC)).columns("dim2").legacy(false).build()), rightTable, "j0.", "(\"dim2\" == \"j0.dim2\")", JoinType.LEFT, null, ExprMacroTable.nil())).setInterval(querySegmentSpec(Filtration.eternity())).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("dim2", "d0", ColumnType.STRING)).setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))).setContext(QUERY_CONTEXT_DEFAULT).build()), useDefault ? ImmutableList.of(new Object[] { "", 2L }, new Object[] { "a", 1L }, new Object[] { "abc", 1L }) : ImmutableList.of(new Object[] { null, 1L }, new Object[] { "", 1L }, new Object[] { "a", 1L }, new Object[] { "abc", 1L }));
}