Examples with InDimFilter org.apache.druid.query.filter.InDimFilter used on opensource projects

Example 51 with InDimFilter

use of org.apache.druid.query.filter.InDimFilter in project druid by druid-io.

the class LongFilteringTest method testLongColumnFiltering.

@Test
public void testLongColumnFiltering() {
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "0", null), ImmutableList.of());
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "3", null), ImmutableList.of("3"));
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "3.0", null), ImmutableList.of("3"));
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "3.00000000000000000000001", null), ImmutableList.of());
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "100000001.0", null), ImmutableList.of("8"));
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "-100000001.0", null), ImmutableList.of("10"));
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "111119223372036854775807.674398674398", null), ImmutableList.of());
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "2", "5", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "1", "4", true, true, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "2.0", "5.0", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "2.0", "5.0", true, true, null, null, StringComparators.NUMERIC), ImmutableList.of("3", "4"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "1.9", "5.9", true, true, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "2.1", "5.9", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("3", "4", "5"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "111119223372036854775807.67", "5.9", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of());
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "-111119223372036854775807.67", "5.9", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("1", "2", "3", "4", "5", "9", "10"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "2.1", "111119223372036854775807.67", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("3", "4", "5", "6", "7", "8"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "2.1", "-111119223372036854775807.67", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of());
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "100000000.0", "100000001.0", true, true, null, null, StringComparators.NUMERIC), ImmutableList.of());
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "100000000.0", "100000001.0", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("7", "8"));
    assertFilterMatches(new InDimFilter(LONG_COLUMN, Arrays.asList("2", "4", "8"), null), ImmutableList.of("2", "4"));
    assertFilterMatches(new InDimFilter(LONG_COLUMN, Arrays.asList("1.999999999999999999", "4.00000000000000000000001"), null), ImmutableList.of());
    assertFilterMatches(new InDimFilter(LONG_COLUMN, Arrays.asList("100000001.0", "99999999.999999999"), null), ImmutableList.of("8"));
    assertFilterMatches(new InDimFilter(LONG_COLUMN, Arrays.asList("-25.0", "-99999999.999999999"), null), ImmutableList.of("9"));
    // cross the hashing threshold to test hashset implementation, filter on even values
    List<String> infilterValues = new ArrayList<>(NUM_FILTER_VALUES);
    for (int i = 0; i < NUM_FILTER_VALUES; i++) {
        infilterValues.add(String.valueOf(i * 2));
    }
    assertFilterMatches(new InDimFilter(LONG_COLUMN, infilterValues, null), ImmutableList.of("2", "4", "6"));
    String jsFn = "function(x) { return(x === 3 || x === 5) }";
    assertFilterMatchesSkipVectorize(new JavaScriptDimFilter(LONG_COLUMN, jsFn, null, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("3", "5"));
    assertFilterMatches(new RegexDimFilter(LONG_COLUMN, "4", null), ImmutableList.of("4"));
    assertFilterMatches(new SearchQueryDimFilter(LONG_COLUMN, new ContainsSearchQuerySpec("2", true), null), ImmutableList.of("2", "9"));
}

Example 52 with InDimFilter

use of org.apache.druid.query.filter.InDimFilter in project druid by druid-io.

the class FloatAndDoubleFilteringTest method doTestMultithreaded.

private void doTestMultithreaded(final String columnName) {
    assertFilterMatchesMultithreaded(new SelectorDimFilter(columnName, "3", null), ImmutableList.of("3"));
    assertFilterMatchesMultithreaded(new SelectorDimFilter(columnName, "3.0", null), ImmutableList.of("3"));
    assertFilterMatchesMultithreaded(new InDimFilter(columnName, Arrays.asList("2", "4", "8"), null), ImmutableList.of("2", "4"));
    assertFilterMatchesMultithreaded(new InDimFilter(columnName, Arrays.asList("2.0", "4.0", "8.0"), null), ImmutableList.of("2", "4"));
    // cross the hashing threshold to test hashset implementation, filter on even values
    List<String> infilterValues = new ArrayList<>(NUM_FILTER_VALUES);
    for (int i = 0; i < NUM_FILTER_VALUES; i++) {
        infilterValues.add(String.valueOf(i * 2));
    }
    assertFilterMatchesMultithreaded(new InDimFilter(columnName, infilterValues, null), ImmutableList.of("2", "4", "6"));
    assertFilterMatches(new BoundDimFilter(columnName, "2", "5", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5"));
    assertFilterMatches(new BoundDimFilter(columnName, "2.0", "5.0", false, false, null, null, StringComparators.NUMERIC), ImmutableList.of("2", "3", "4", "5"));
}

Example 53 with InDimFilter

use of org.apache.druid.query.filter.InDimFilter in project druid by druid-io.

the class InvalidFilteringTest method testFilterTheUnfilterable.

@Test
public void testFilterTheUnfilterable() {
    // single value matching
    assertFilterMatches(new SelectorDimFilter("hyperion", "a string", null), ImmutableList.of());
    assertFilterMatches(new SelectorDimFilter("hyperion", null, null), ImmutableList.of("1", "2", "3", "4", "5", "6"));
    // predicate based matching
    assertFilterMatches(new InDimFilter("hyperion", Arrays.asList("hello", "world"), null), ImmutableList.of());
    assertFilterMatches(new InDimFilter("hyperion", Arrays.asList("hello", "world", null), null), ImmutableList.of("1", "2", "3", "4", "5", "6"));
}

Example 54 with InDimFilter

use of org.apache.druid.query.filter.InDimFilter in project druid by druid-io.

the class JoinFilterAnalyzer method rewriteSelectorFilter.

/**
 * Rewrites a selector filter on a join table into an IN filter on the base table.
 *
 * @param selectorFilter                    SelectorFilter to be rewritten
 * @param joinFilterPreAnalysis             The pre-analysis computed by {@link #computeJoinFilterPreAnalysis)}
 * @param pushDownVirtualColumnsForLhsExprs See comments on {@link #analyzeJoinFilterClause}
 *
 * @return A JoinFilterAnalysis that indicates how to handle the potentially rewritten filter
 */
private static JoinFilterAnalysis rewriteSelectorFilter(SelectorFilter selectorFilter, JoinFilterPreAnalysis joinFilterPreAnalysis, Map<Expr, VirtualColumn> pushDownVirtualColumnsForLhsExprs) {
    List<Filter> newFilters = new ArrayList<>();
    String filteringColumn = selectorFilter.getDimension();
    String filteringValue = selectorFilter.getValue();
    if (areSomeColumnsFromPostJoinVirtualColumns(joinFilterPreAnalysis.getPostJoinVirtualColumns(), selectorFilter.getRequiredColumns())) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    if (!joinFilterPreAnalysis.getJoinableClauses().areSomeColumnsFromJoin(selectorFilter.getRequiredColumns())) {
        return new JoinFilterAnalysis(false, selectorFilter, selectorFilter);
    }
    List<JoinFilterColumnCorrelationAnalysis> correlationAnalyses = joinFilterPreAnalysis.getCorrelationsByFilteringColumn().get(filteringColumn);
    if (correlationAnalyses == null) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    for (JoinFilterColumnCorrelationAnalysis correlationAnalysis : correlationAnalyses) {
        if (correlationAnalysis.supportsPushDown()) {
            Optional<Set<String>> correlatedValues = correlationAnalysis.getCorrelatedValuesMap().get(Pair.of(filteringColumn, filteringValue));
            if (!correlatedValues.isPresent()) {
                return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
            }
            Set<String> newFilterValues = correlatedValues.get();
            // in nothing => match nothing
            if (newFilterValues.isEmpty()) {
                return new JoinFilterAnalysis(true, selectorFilter, FalseFilter.instance());
            }
            for (String correlatedBaseColumn : correlationAnalysis.getBaseColumns()) {
                Filter rewrittenFilter = new InDimFilter(correlatedBaseColumn, newFilterValues).toFilter();
                newFilters.add(rewrittenFilter);
            }
            for (Expr correlatedBaseExpr : correlationAnalysis.getBaseExpressions()) {
                // We need to create a virtual column for the expressions when pushing down
                VirtualColumn pushDownVirtualColumn = pushDownVirtualColumnsForLhsExprs.computeIfAbsent(correlatedBaseExpr, (expr) -> {
                    String vcName = getCorrelatedBaseExprVirtualColumnName(pushDownVirtualColumnsForLhsExprs.size());
                    return new ExpressionVirtualColumn(vcName, correlatedBaseExpr, ColumnType.STRING);
                });
                Filter rewrittenFilter = new InDimFilter(pushDownVirtualColumn.getOutputName(), newFilterValues).toFilter();
                newFilters.add(rewrittenFilter);
            }
        }
    }
    if (newFilters.isEmpty()) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    return new JoinFilterAnalysis(true, selectorFilter, Filters.maybeAnd(newFilters).orElse(null));
}

Example 55 with InDimFilter

use of org.apache.druid.query.filter.InDimFilter in project druid by druid-io.

the class FilteredAggregatorTest method testAggregateWithExtractionFns.

@Test
public void testAggregateWithExtractionFns() {
    final float[] values = { 0.15f, 0.27f };
    TestFloatColumnSelector selector;
    FilteredAggregatorFactory factory;
    String extractionJsFn = "function(str) { return str + 'AARDVARK'; }";
    ExtractionFn extractionFn = new JavaScriptExtractionFn(extractionJsFn, false, JavaScriptConfig.getEnabledInstance());
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SelectorDimFilter("dim", "aAARDVARK", extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new InDimFilter("dim", Arrays.asList("NOT-aAARDVARK", "FOOBAR", "aAARDVARK"), extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new BoundDimFilter("dim", "aAARDVARK", "aAARDVARK", false, false, true, extractionFn, StringComparators.ALPHANUMERIC));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new RegexDimFilter("dim", "aAARDVARK", extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("aAARDVARK", true), extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    String jsFn = "function(x) { return(x === 'aAARDVARK') }";
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new JavaScriptDimFilter("dim", jsFn, extractionFn, JavaScriptConfig.getEnabledInstance()));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
}