Examples with InDimFilter io.druid.query.filter.InDimFilter used on opensource projects

Example 6 with InDimFilter

use of io.druid.query.filter.InDimFilter in project druid by druid-io.

the class FilteredAggregatorTest method testAggregateWithExtractionFns.

@Test
public void testAggregateWithExtractionFns() {
    final float[] values = { 0.15f, 0.27f };
    TestFloatColumnSelector selector;
    FilteredAggregatorFactory factory;
    String extractionJsFn = "function(str) { return str + 'AARDVARK'; }";
    ExtractionFn extractionFn = new JavaScriptExtractionFn(extractionJsFn, false, JavaScriptConfig.getEnabledInstance());
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SelectorDimFilter("dim", "aAARDVARK", extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new InDimFilter("dim", Arrays.asList("NOT-aAARDVARK", "FOOBAR", "aAARDVARK"), extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new BoundDimFilter("dim", "aAARDVARK", "aAARDVARK", false, false, true, extractionFn, StringComparators.ALPHANUMERIC));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new RegexDimFilter("dim", "aAARDVARK", extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("aAARDVARK", true), extractionFn));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
    String jsFn = "function(x) { return(x === 'aAARDVARK') }";
    factory = new FilteredAggregatorFactory(new DoubleSumAggregatorFactory("billy", "value"), new JavaScriptDimFilter("dim", jsFn, extractionFn, JavaScriptConfig.getEnabledInstance()));
    selector = new TestFloatColumnSelector(values);
    validateFilteredAggs(factory, values, selector);
}

Example 7 with InDimFilter

use of io.druid.query.filter.InDimFilter in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByWithAllFiltersOnNullDimsWithExtractionFns.

@Test
public void testGroupByWithAllFiltersOnNullDimsWithExtractionFns() {
    Map<String, String> extractionMap = new HashMap<>();
    extractionMap.put("", "EMPTY");
    extractionMap.put(null, "EMPTY");
    MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false);
    LookupExtractionFn extractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true);
    String jsFn = "function(x) { return(x === 'EMPTY') }";
    List<DimFilter> superFilterList = new ArrayList<>();
    superFilterList.add(new SelectorDimFilter("null_column", "EMPTY", extractionFn));
    superFilterList.add(new InDimFilter("null_column", Arrays.asList("NOT-EMPTY", "FOOBAR", "EMPTY"), extractionFn));
    superFilterList.add(new BoundDimFilter("null_column", "EMPTY", "EMPTY", false, false, true, extractionFn, StringComparators.ALPHANUMERIC));
    superFilterList.add(new RegexDimFilter("null_column", "EMPTY", extractionFn));
    superFilterList.add(new SearchQueryDimFilter("null_column", new ContainsSearchQuerySpec("EMPTY", true), extractionFn));
    superFilterList.add(new JavaScriptDimFilter("null_column", jsFn, extractionFn, JavaScriptConfig.getEnabledInstance()));
    DimFilter superFilter = new AndDimFilter(superFilterList);
    GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource).setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird).setDimensions(Lists.<DimensionSpec>newArrayList(new DefaultDimensionSpec("null_column", "alias"))).setAggregatorSpecs(Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))).setGranularity(QueryRunnerTestHelper.dayGran).setDimFilter(superFilter).build();
    List<Row> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 13L, "idx", 6619L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", null, "rows", 13L, "idx", 5827L));
    Iterable<Row> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "");
}

Example 8 with InDimFilter

use of io.druid.query.filter.InDimFilter in project druid by druid-io.

the class SearchBenchmark method basicB.

private static SearchQueryBuilder basicB(final BenchmarkSchemaInfo basicSchema) {
    final QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Arrays.asList(basicSchema.getDataInterval()));
    final List<String> dimUniformFilterVals = Lists.newArrayList();
    int resultNum = (int) (100000 * 0.1);
    int step = 100000 / resultNum;
    for (int i = 1; i < 100001 && dimUniformFilterVals.size() < resultNum; i += step) {
        dimUniformFilterVals.add(String.valueOf(i));
    }
    List<String> dimHyperUniqueFilterVals = Lists.newArrayList();
    resultNum = (int) (100000 * 0.1);
    step = 100000 / resultNum;
    for (int i = 0; i < 100001 && dimHyperUniqueFilterVals.size() < resultNum; i += step) {
        dimHyperUniqueFilterVals.add(String.valueOf(i));
    }
    final List<DimFilter> dimFilters = Lists.newArrayList();
    dimFilters.add(new InDimFilter("dimUniform", dimUniformFilterVals, null));
    dimFilters.add(new InDimFilter("dimHyperUnique", dimHyperUniqueFilterVals, null));
    return Druids.newSearchQueryBuilder().dataSource("blah").granularity(Granularities.ALL).intervals(intervalSpec).query("").dimensions(Lists.newArrayList("dimUniform", "dimHyperUnique")).filters(new AndDimFilter(dimFilters));
}

Example 9 with InDimFilter

use of io.druid.query.filter.InDimFilter in project druid by druid-io.

the class SearchBenchmark method basicC.

private static SearchQueryBuilder basicC(final BenchmarkSchemaInfo basicSchema) {
    final QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Arrays.asList(basicSchema.getDataInterval()));
    final List<String> dimUniformFilterVals = Lists.newArrayList();
    final int resultNum = (int) (100000 * 0.1);
    final int step = 100000 / resultNum;
    for (int i = 1; i < 100001 && dimUniformFilterVals.size() < resultNum; i += step) {
        dimUniformFilterVals.add(String.valueOf(i));
    }
    final String dimName = "dimUniform";
    final List<DimFilter> dimFilters = Lists.newArrayList();
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, IdentityExtractionFn.getInstance()));
    dimFilters.add(new SelectorDimFilter(dimName, "3", StrlenExtractionFn.instance()));
    dimFilters.add(new BoundDimFilter(dimName, "100", "10000", true, true, true, new DimExtractionFn() {

        @Override
        public byte[] getCacheKey() {
            return new byte[] { 0xF };
        }

        @Override
        public String apply(String value) {
            return String.valueOf(Long.parseLong(value) + 1);
        }

        @Override
        public boolean preservesOrdering() {
            return false;
        }

        @Override
        public ExtractionType getExtractionType() {
            return ExtractionType.ONE_TO_ONE;
        }
    }, null));
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, new LowerExtractionFn(null)));
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, new UpperExtractionFn(null)));
    dimFilters.add(new InDimFilter(dimName, dimUniformFilterVals, new SubstringDimExtractionFn(1, 3)));
    return Druids.newSearchQueryBuilder().dataSource("blah").granularity(Granularities.ALL).intervals(intervalSpec).query("").dimensions(Lists.newArrayList("dimUniform")).filters(new AndDimFilter(dimFilters));
}

Example 10 with InDimFilter

use of io.druid.query.filter.InDimFilter in project druid by druid-io.

the class ConvertSelectorsToIns method process.

@Override
public DimFilter process(DimFilter filter) {
    if (filter instanceof OrDimFilter) {
        // Copy children list
        final List<DimFilter> children = Lists.newArrayList(((OrDimFilter) filter).getFields());
        // Group filters by dimension and extractionFn.
        final Map<BoundRefKey, List<SelectorDimFilter>> selectors = Maps.newHashMap();
        for (DimFilter child : children) {
            if (child instanceof SelectorDimFilter) {
                final SelectorDimFilter selector = (SelectorDimFilter) child;
                final BoundRefKey boundRefKey = BoundRefKey.from(selector, sourceRowSignature.naturalStringComparator(RowExtraction.of(selector.getDimension(), selector.getExtractionFn())));
                List<SelectorDimFilter> filterList = selectors.get(boundRefKey);
                if (filterList == null) {
                    filterList = Lists.newArrayList();
                    selectors.put(boundRefKey, filterList);
                }
                filterList.add(selector);
            }
        }
        // Emit IN filters for each group of size > 1.
        for (Map.Entry<BoundRefKey, List<SelectorDimFilter>> entry : selectors.entrySet()) {
            final List<SelectorDimFilter> filterList = entry.getValue();
            if (filterList.size() > 1) {
                // We found a simplification. Remove the old filters and add new ones.
                final List<String> values = Lists.newArrayList();
                for (final SelectorDimFilter selector : filterList) {
                    values.add(selector.getValue());
                    if (!children.remove(selector)) {
                        throw new ISE("WTF?! Tried to remove selector but couldn't?");
                    }
                }
                children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn()));
            }
        }
        if (!children.equals(((OrDimFilter) filter).getFields())) {
            return children.size() == 1 ? children.get(0) : new OrDimFilter(children);
        } else {
            return filter;
        }
    } else {
        return filter;
    }
}