Examples with AndFilter io.druid.segment.filter.AndFilter used on opensource projects

Example 1 with AndFilter

use of io.druid.segment.filter.AndFilter in project druid by druid-io.

the class QueryableIndexStorageAdapter method makeCursors.

@Override
public Sequence<Cursor> makeCursors(Filter filter, Interval interval, VirtualColumns virtualColumns, Granularity gran, boolean descending) {
    Interval actualInterval = interval;
    long minDataTimestamp = getMinTime().getMillis();
    long maxDataTimestamp = getMaxTime().getMillis();
    final Interval dataInterval = new Interval(minDataTimestamp, gran.bucketEnd(getMaxTime()).getMillis());
    if (!actualInterval.overlaps(dataInterval)) {
        return Sequences.empty();
    }
    if (actualInterval.getStart().isBefore(dataInterval.getStart())) {
        actualInterval = actualInterval.withStart(dataInterval.getStart());
    }
    if (actualInterval.getEnd().isAfter(dataInterval.getEnd())) {
        actualInterval = actualInterval.withEnd(dataInterval.getEnd());
    }
    final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(index.getBitmapFactoryForDimensions(), virtualColumns, index);
    /**
     * Filters can be applied in two stages:
     * pre-filtering: Use bitmap indexes to prune the set of rows to be scanned.
     * post-filtering: Iterate through rows and apply the filter to the row values
     *
     * The pre-filter and post-filter step have an implicit AND relationship. (i.e., final rows are those that
     * were not pruned AND those that matched the filter during row scanning)
     *
     * An AND filter can have its subfilters partitioned across the two steps. The subfilters that can be
     * processed entirely with bitmap indexes (subfilter returns true for supportsBitmapIndex())
     * will be moved to the pre-filtering stage.
     *
     * Any subfilters that cannot be processed entirely with bitmap indexes will be moved to the post-filtering stage.
     */
    final Offset offset;
    final List<Filter> postFilters = new ArrayList<>();
    if (filter == null) {
        offset = new NoFilterOffset(0, index.getNumRows(), descending);
    } else {
        final List<Filter> preFilters = new ArrayList<>();
        if (filter instanceof AndFilter) {
            // If we get an AndFilter, we can split the subfilters across both filtering stages
            for (Filter subfilter : ((AndFilter) filter).getFilters()) {
                if (subfilter.supportsBitmapIndex(selector)) {
                    preFilters.add(subfilter);
                } else {
                    postFilters.add(subfilter);
                }
            }
        } else {
            // If we get an OrFilter or a single filter, handle the filter in one stage
            if (filter.supportsBitmapIndex(selector)) {
                preFilters.add(filter);
            } else {
                postFilters.add(filter);
            }
        }
        if (preFilters.size() == 0) {
            offset = new NoFilterOffset(0, index.getNumRows(), descending);
        } else {
            // Use AndFilter.getBitmapIndex to intersect the preFilters to get its short-circuiting behavior.
            offset = BitmapOffset.of(AndFilter.getBitmapIndex(selector, preFilters), descending, (long) getNumRows());
        }
    }
    final Filter postFilter;
    if (postFilters.size() == 0) {
        postFilter = null;
    } else if (postFilters.size() == 1) {
        postFilter = postFilters.get(0);
    } else {
        postFilter = new AndFilter(postFilters);
    }
    return Sequences.filter(new CursorSequenceBuilder(this, actualInterval, virtualColumns, gran, offset, minDataTimestamp, maxDataTimestamp, descending, postFilter, selector).build(), Predicates.<Cursor>notNull());
}

Example 2 with AndFilter

use of io.druid.segment.filter.AndFilter in project druid by druid-io.

the class FilterPartitionBenchmark method readOrFilter.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readOrFilter(Blackhole blackhole) throws Exception {
    Filter filter = new NoBitmapSelectorFilter("dimSequential", "199");
    Filter filter2 = new AndFilter(Arrays.<Filter>asList(new SelectorFilter("dimMultivalEnumerated2", "Corundum"), new NoBitmapSelectorFilter("dimMultivalEnumerated", "Bar")));
    Filter orFilter = new OrFilter(Arrays.<Filter>asList(filter, filter2));
    StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
    Sequence<Cursor> cursors = makeCursors(sa, orFilter);
    Sequence<List<String>> stringListSeq = readCursors(cursors, blackhole);
    List<String> strings = Sequences.toList(Sequences.limit(stringListSeq, 1), Lists.<List<String>>newArrayList()).get(0);
    for (String st : strings) {
        blackhole.consume(st);
    }
}

Example 3 with AndFilter

use of io.druid.segment.filter.AndFilter in project druid by druid-io.

the class FilterPartitionBenchmark method readOrFilterCNF.

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void readOrFilterCNF(Blackhole blackhole) throws Exception {
    Filter filter = new NoBitmapSelectorFilter("dimSequential", "199");
    Filter filter2 = new AndFilter(Arrays.<Filter>asList(new SelectorFilter("dimMultivalEnumerated2", "Corundum"), new NoBitmapSelectorFilter("dimMultivalEnumerated", "Bar")));
    Filter orFilter = new OrFilter(Arrays.<Filter>asList(filter, filter2));
    StorageAdapter sa = new QueryableIndexStorageAdapter(qIndex);
    Sequence<Cursor> cursors = makeCursors(sa, Filters.convertToCNF(orFilter));
    Sequence<List<String>> stringListSeq = readCursors(cursors, blackhole);
    List<String> strings = Sequences.toList(Sequences.limit(stringListSeq, 1), Lists.<List<String>>newArrayList()).get(0);
    for (String st : strings) {
        blackhole.consume(st);
    }
}