Search in sources :

Example 1 with ScanBasedFilterOperator

use of com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator in project pinot by linkedin.

the class RealtimeSegmentTest method testNoRangeMatchFilteringMetricPredicateWithoutInvIdx.

@Test
public void testNoRangeMatchFilteringMetricPredicateWithoutInvIdx() throws Exception {
    DataSource ds1 = segmentWithoutInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("[0\t\t100)");
    Predicate predicate = new RangePredicate("count", rhs);
    ScanBasedFilterOperator op = new ScanBasedFilterOperator(predicate, ds1, 0, segmentWithoutInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    int counter = 0;
    int docId = iterator.next();
    while (docId != Constants.EOF) {
        // shouldn't reach here.
        Assert.assertTrue(false);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 0);
}
Also used : RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) ArrayList(java.util.ArrayList) Block(com.linkedin.pinot.core.common.Block) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) DataSource(com.linkedin.pinot.core.common.DataSource) RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) NEqPredicate(com.linkedin.pinot.core.common.predicate.NEqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) Predicate(com.linkedin.pinot.core.common.Predicate) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test) RealtimeSegmentImplTest(com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)

Example 2 with ScanBasedFilterOperator

use of com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator in project pinot by linkedin.

the class RealtimeSegmentTest method testRangeMatchFilteringMetricPredicateWithoutInvIdx.

@Test
public void testRangeMatchFilteringMetricPredicateWithoutInvIdx() throws Exception {
    DataSource ds1 = segmentWithoutInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("[0\t\t*)");
    Predicate predicate = new RangePredicate("count", rhs);
    ScanBasedFilterOperator op = new ScanBasedFilterOperator(predicate, ds1, 0, segmentWithoutInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    DataSource ds2 = segmentWithoutInvIdx.getDataSource("count");
    BlockSingleValIterator blockValIterator = (BlockSingleValIterator) ds2.nextBlock().getBlockValueSet().iterator();
    int docId = iterator.next();
    int counter = 0;
    while (docId != Constants.EOF) {
        blockValIterator.skipTo(docId);
        Assert.assertEquals(ds1.getDictionary().get(blockValIterator.nextIntVal()), 890662862);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 100000);
}
Also used : RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) BlockSingleValIterator(com.linkedin.pinot.core.common.BlockSingleValIterator) ArrayList(java.util.ArrayList) Block(com.linkedin.pinot.core.common.Block) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) DataSource(com.linkedin.pinot.core.common.DataSource) RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) NEqPredicate(com.linkedin.pinot.core.common.predicate.NEqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) Predicate(com.linkedin.pinot.core.common.Predicate) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test) RealtimeSegmentImplTest(com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)

Example 3 with ScanBasedFilterOperator

use of com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator in project pinot by linkedin.

the class FilterPlanNode method reorder.

/**
   * Re orders operators, puts Sorted -> Inverted and then Raw scan. TODO: With Inverted, we can
   * further optimize based on cardinality
   * @param operators
   */
private static void reorder(List<BaseFilterOperator> operators) {
    final Map<Operator, Integer> operatorPriorityMap = new HashMap<Operator, Integer>();
    for (Operator operator : operators) {
        Integer priority = Integer.MAX_VALUE;
        if (operator instanceof SortedInvertedIndexBasedFilterOperator) {
            priority = 0;
        } else if (operator instanceof AndOperator) {
            priority = 1;
        } else if (operator instanceof BitmapBasedFilterOperator) {
            priority = 2;
        } else if (operator instanceof ScanBasedFilterOperator) {
            priority = 3;
        } else if (operator instanceof OrOperator) {
            priority = 4;
        }
        operatorPriorityMap.put(operator, priority);
    }
    Comparator<? super Operator> comparator = new Comparator<Operator>() {

        @Override
        public int compare(Operator o1, Operator o2) {
            return Integer.compare(operatorPriorityMap.get(o1), operatorPriorityMap.get(o2));
        }
    };
    Collections.sort(operators, comparator);
}
Also used : BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) BitmapBasedFilterOperator(com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) MatchEntireSegmentOperator(com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator) EmptyFilterOperator(com.linkedin.pinot.core.operator.filter.EmptyFilterOperator) SortedInvertedIndexBasedFilterOperator(com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator) Operator(com.linkedin.pinot.core.common.Operator) FilterOperator(com.linkedin.pinot.common.request.FilterOperator) StarTreeIndexOperator(com.linkedin.pinot.core.operator.filter.StarTreeIndexOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) OrOperator(com.linkedin.pinot.core.operator.filter.OrOperator) OrOperator(com.linkedin.pinot.core.operator.filter.OrOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) HashMap(java.util.HashMap) BitmapBasedFilterOperator(com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) SortedInvertedIndexBasedFilterOperator(com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator) Comparator(java.util.Comparator)

Example 4 with ScanBasedFilterOperator

use of com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator in project pinot by linkedin.

the class FilterPlanNode method constructPhysicalOperator.

/**
   * Helper method to build the operator tree from the filter query tree.
   * @param filterQueryTree
   * @param segment Index segment
   * @param optimizeAlwaysFalse Optimize isResultEmpty predicates
   * @return Filter Operator created
   */
@VisibleForTesting
public static BaseFilterOperator constructPhysicalOperator(FilterQueryTree filterQueryTree, IndexSegment segment, boolean optimizeAlwaysFalse) {
    BaseFilterOperator ret;
    if (null == filterQueryTree) {
        return new MatchEntireSegmentOperator(segment.getSegmentMetadata().getTotalRawDocs());
    }
    final List<FilterQueryTree> childFilters = filterQueryTree.getChildren();
    final boolean isLeaf = (childFilters == null) || childFilters.isEmpty();
    if (!isLeaf) {
        int numChildrenAlwaysFalse = 0;
        int numChildren = childFilters.size();
        List<BaseFilterOperator> operators = new ArrayList<>();
        final FilterOperator filterType = filterQueryTree.getOperator();
        for (final FilterQueryTree query : childFilters) {
            BaseFilterOperator childOperator = constructPhysicalOperator(query, segment, optimizeAlwaysFalse);
            // Count number of always false children.
            if (optimizeAlwaysFalse && childOperator.isResultEmpty()) {
                numChildrenAlwaysFalse++;
                // Early bailout for 'AND' as soon as one of the children always evaluates to false.
                if (filterType == FilterOperator.AND) {
                    break;
                }
            }
            operators.add(childOperator);
        }
        ret = buildNonLeafOperator(filterType, operators, numChildrenAlwaysFalse, numChildren, optimizeAlwaysFalse);
    } else {
        final FilterOperator filterType = filterQueryTree.getOperator();
        final String column = filterQueryTree.getColumn();
        Predicate predicate = Predicate.newPredicate(filterQueryTree);
        DataSource ds;
        ds = segment.getDataSource(column);
        DataSourceMetadata dataSourceMetadata = ds.getDataSourceMetadata();
        BaseFilterOperator baseFilterOperator;
        int startDocId = 0;
        //end is inclusive
        int endDocId = segment.getSegmentMetadata().getTotalRawDocs() - 1;
        if (dataSourceMetadata.hasInvertedIndex()) {
            // range evaluation based on inv index is inefficient, so do this only if is NOT range.
            if (!filterType.equals(FilterOperator.RANGE)) {
                if (dataSourceMetadata.isSingleValue() && dataSourceMetadata.isSorted()) {
                    // if the column is sorted use sorted inverted index based implementation
                    baseFilterOperator = new SortedInvertedIndexBasedFilterOperator(predicate, ds, startDocId, endDocId);
                } else {
                    baseFilterOperator = new BitmapBasedFilterOperator(predicate, ds, startDocId, endDocId);
                }
            } else {
                baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
            }
        } else {
            baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
        }
        ret = baseFilterOperator;
    }
    // If operator evaluates to false, then just return an empty operator.
    if (ret.isResultEmpty()) {
        ret = new EmptyFilterOperator();
    }
    return ret;
}
Also used : FilterQueryTree(com.linkedin.pinot.common.utils.request.FilterQueryTree) ArrayList(java.util.ArrayList) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) Predicate(com.linkedin.pinot.core.common.Predicate) DataSource(com.linkedin.pinot.core.common.DataSource) MatchEntireSegmentOperator(com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator) BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) BitmapBasedFilterOperator(com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) EmptyFilterOperator(com.linkedin.pinot.core.operator.filter.EmptyFilterOperator) SortedInvertedIndexBasedFilterOperator(com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator) FilterOperator(com.linkedin.pinot.common.request.FilterOperator) BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) DataSourceMetadata(com.linkedin.pinot.core.common.DataSourceMetadata) BitmapBasedFilterOperator(com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator) EmptyFilterOperator(com.linkedin.pinot.core.operator.filter.EmptyFilterOperator) SortedInvertedIndexBasedFilterOperator(com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 5 with ScanBasedFilterOperator

use of com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator in project pinot by linkedin.

the class RealtimeSegmentTest method testNoMatchFilteringMetricPredicateWithoutInvIdx.

@Test
public void testNoMatchFilteringMetricPredicateWithoutInvIdx() throws Exception {
    DataSource ds1 = segmentWithoutInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("890662862");
    Predicate predicate = new NEqPredicate("count", rhs);
    ScanBasedFilterOperator op = new ScanBasedFilterOperator(predicate, ds1, 0, segmentWithoutInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    int counter = 0;
    int docId = iterator.next();
    while (docId != Constants.EOF) {
        // shouldn't reach here.
        Assert.assertTrue(false);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 0);
}
Also used : NEqPredicate(com.linkedin.pinot.core.common.predicate.NEqPredicate) ArrayList(java.util.ArrayList) Block(com.linkedin.pinot.core.common.Block) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) DataSource(com.linkedin.pinot.core.common.DataSource) RangePredicate(com.linkedin.pinot.core.common.predicate.RangePredicate) NEqPredicate(com.linkedin.pinot.core.common.predicate.NEqPredicate) EqPredicate(com.linkedin.pinot.core.common.predicate.EqPredicate) Predicate(com.linkedin.pinot.core.common.Predicate) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test) RealtimeSegmentImplTest(com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)

Aggregations

ScanBasedFilterOperator (com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator)6 DataSource (com.linkedin.pinot.core.common.DataSource)5 Predicate (com.linkedin.pinot.core.common.Predicate)5 ArrayList (java.util.ArrayList)5 Block (com.linkedin.pinot.core.common.Block)4 BlockDocIdIterator (com.linkedin.pinot.core.common.BlockDocIdIterator)4 EqPredicate (com.linkedin.pinot.core.common.predicate.EqPredicate)4 NEqPredicate (com.linkedin.pinot.core.common.predicate.NEqPredicate)4 RangePredicate (com.linkedin.pinot.core.common.predicate.RangePredicate)4 RealtimeSegmentImplTest (com.linkedin.pinot.core.realtime.impl.kafka.RealtimeSegmentImplTest)4 Test (org.testng.annotations.Test)4 FilterOperator (com.linkedin.pinot.common.request.FilterOperator)2 BlockSingleValIterator (com.linkedin.pinot.core.common.BlockSingleValIterator)2 BaseFilterOperator (com.linkedin.pinot.core.operator.filter.BaseFilterOperator)2 BitmapBasedFilterOperator (com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator)2 EmptyFilterOperator (com.linkedin.pinot.core.operator.filter.EmptyFilterOperator)2 MatchEntireSegmentOperator (com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator)2 SortedInvertedIndexBasedFilterOperator (com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 FilterQueryTree (com.linkedin.pinot.common.utils.request.FilterQueryTree)1