Search in sources :

Example 1 with AndOperator

use of com.linkedin.pinot.core.operator.filter.AndOperator in project pinot by linkedin.

the class AndOperatorTest method testIntersectionForThreeLists.

@Test
public void testIntersectionForThreeLists() {
    int[] list1 = new int[] { 2, 3, 6, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    int[] list3 = new int[] { 1, 2, 3, 6, 30 };
    List<BaseFilterOperator> operators = new ArrayList<BaseFilterOperator>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    operators.add(makeFilterOperator(list3));
    final AndOperator andOperator = new AndOperator(operators);
    andOperator.open();
    BaseFilterBlock block;
    while ((block = andOperator.getNextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
        //        System.out.println(docId);
        }
    }
    andOperator.close();
}
Also used : BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) BlockDocIdSet(com.linkedin.pinot.core.common.BlockDocIdSet) ArrayList(java.util.ArrayList) BaseFilterBlock(com.linkedin.pinot.core.operator.blocks.BaseFilterBlock) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test)

Example 2 with AndOperator

use of com.linkedin.pinot.core.operator.filter.AndOperator in project pinot by linkedin.

the class AndOperatorTest method testComplexWithOr.

@Test
public void testComplexWithOr() {
    int[] list1 = new int[] { 2, 3, 6, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    int[] list3 = new int[] { 1, 2, 3, 6, 30 };
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list3));
    operators.add(makeFilterOperator(list2));
    final OrOperator orOperator = new OrOperator(operators);
    List<BaseFilterOperator> operators1 = new ArrayList<>();
    operators1.add(orOperator);
    operators1.add(makeFilterOperator(list1));
    final AndOperator andOperator = new AndOperator(operators1);
    andOperator.open();
    BaseFilterBlock block;
    while ((block = andOperator.getNextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
        //        System.out.println(docId);
        }
    }
    andOperator.close();
}
Also used : OrOperator(com.linkedin.pinot.core.operator.filter.OrOperator) BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) BlockDocIdSet(com.linkedin.pinot.core.common.BlockDocIdSet) ArrayList(java.util.ArrayList) BaseFilterBlock(com.linkedin.pinot.core.operator.blocks.BaseFilterBlock) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test)

Example 3 with AndOperator

use of com.linkedin.pinot.core.operator.filter.AndOperator in project pinot by linkedin.

the class AndOperatorTest method testComplex.

@Test
public void testComplex() {
    int[] list1 = new int[] { 2, 3, 6, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    int[] list3 = new int[] { 1, 2, 3, 6, 30 };
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    final AndOperator andOperator1 = new AndOperator(operators);
    List<BaseFilterOperator> operators1 = new ArrayList<>();
    operators1.add(andOperator1);
    operators1.add(makeFilterOperator(list3));
    final AndOperator andOperator = new AndOperator(operators1);
    andOperator.open();
    BaseFilterBlock block;
    while ((block = andOperator.getNextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
        //        System.out.println(docId);
        }
    }
    andOperator.close();
}
Also used : BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) BlockDocIdSet(com.linkedin.pinot.core.common.BlockDocIdSet) ArrayList(java.util.ArrayList) BaseFilterBlock(com.linkedin.pinot.core.operator.blocks.BaseFilterBlock) BlockDocIdIterator(com.linkedin.pinot.core.common.BlockDocIdIterator) Test(org.testng.annotations.Test)

Example 4 with AndOperator

use of com.linkedin.pinot.core.operator.filter.AndOperator in project pinot by linkedin.

the class FilterPlanNode method reorder.

/**
   * Re orders operators, puts Sorted -> Inverted and then Raw scan. TODO: With Inverted, we can
   * further optimize based on cardinality
   * @param operators
   */
private static void reorder(List<BaseFilterOperator> operators) {
    final Map<Operator, Integer> operatorPriorityMap = new HashMap<Operator, Integer>();
    for (Operator operator : operators) {
        Integer priority = Integer.MAX_VALUE;
        if (operator instanceof SortedInvertedIndexBasedFilterOperator) {
            priority = 0;
        } else if (operator instanceof AndOperator) {
            priority = 1;
        } else if (operator instanceof BitmapBasedFilterOperator) {
            priority = 2;
        } else if (operator instanceof ScanBasedFilterOperator) {
            priority = 3;
        } else if (operator instanceof OrOperator) {
            priority = 4;
        }
        operatorPriorityMap.put(operator, priority);
    }
    Comparator<? super Operator> comparator = new Comparator<Operator>() {

        @Override
        public int compare(Operator o1, Operator o2) {
            return Integer.compare(operatorPriorityMap.get(o1), operatorPriorityMap.get(o2));
        }
    };
    Collections.sort(operators, comparator);
}
Also used : BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) BitmapBasedFilterOperator(com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) MatchEntireSegmentOperator(com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator) EmptyFilterOperator(com.linkedin.pinot.core.operator.filter.EmptyFilterOperator) SortedInvertedIndexBasedFilterOperator(com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator) Operator(com.linkedin.pinot.core.common.Operator) FilterOperator(com.linkedin.pinot.common.request.FilterOperator) StarTreeIndexOperator(com.linkedin.pinot.core.operator.filter.StarTreeIndexOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) OrOperator(com.linkedin.pinot.core.operator.filter.OrOperator) OrOperator(com.linkedin.pinot.core.operator.filter.OrOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) HashMap(java.util.HashMap) BitmapBasedFilterOperator(com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator) ScanBasedFilterOperator(com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator) SortedInvertedIndexBasedFilterOperator(com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator) Comparator(java.util.Comparator)

Example 5 with AndOperator

use of com.linkedin.pinot.core.operator.filter.AndOperator in project pinot by linkedin.

the class FilterPlanNode method buildNonLeafOperator.

/**
   * Helper method to build AND/OR operators.
   * <ul>
   *   <li> Returns {@link EmptyFilterOperator} if at least on child always evaluates to false for AND. </li>
   *   <li> Returns {@link EmptyFilterOperator} if all children always evaluates to false for OR. </li>
   *   <li> Returns {@link AndOperator} or {@link OrOperator} based on filterType, otherwise. </li>
   * </ul>
   * @param filterType AND/OR
   * @param nonFalseChildren Children that are not alwaysFalse.
   * @param numChildrenAlwaysFalse Number of children that are always false.
   * @param numChildren Total number of children.
   * @param optimizeAlwaysFalse Optimize alwaysFalse predicates
   * @return Filter Operator created
   */
private static BaseFilterOperator buildNonLeafOperator(FilterOperator filterType, List<BaseFilterOperator> nonFalseChildren, int numChildrenAlwaysFalse, int numChildren, boolean optimizeAlwaysFalse) {
    BaseFilterOperator operator;
    switch(filterType) {
        case AND:
            if (optimizeAlwaysFalse && numChildrenAlwaysFalse > 0) {
                operator = new EmptyFilterOperator();
            } else {
                reorder(nonFalseChildren);
                operator = new AndOperator(nonFalseChildren);
            }
            break;
        case OR:
            if (optimizeAlwaysFalse && numChildrenAlwaysFalse == numChildren) {
                operator = new EmptyFilterOperator();
            } else {
                reorder(nonFalseChildren);
                operator = new OrOperator(nonFalseChildren);
            }
            break;
        default:
            throw new UnsupportedOperationException("Not support filter type - " + filterType + " with children operators");
    }
    return operator;
}
Also used : OrOperator(com.linkedin.pinot.core.operator.filter.OrOperator) BaseFilterOperator(com.linkedin.pinot.core.operator.filter.BaseFilterOperator) AndOperator(com.linkedin.pinot.core.operator.filter.AndOperator) EmptyFilterOperator(com.linkedin.pinot.core.operator.filter.EmptyFilterOperator)

Aggregations

AndOperator (com.linkedin.pinot.core.operator.filter.AndOperator)6 BaseFilterOperator (com.linkedin.pinot.core.operator.filter.BaseFilterOperator)6 BlockDocIdIterator (com.linkedin.pinot.core.common.BlockDocIdIterator)4 BlockDocIdSet (com.linkedin.pinot.core.common.BlockDocIdSet)4 BaseFilterBlock (com.linkedin.pinot.core.operator.blocks.BaseFilterBlock)4 ArrayList (java.util.ArrayList)4 Test (org.testng.annotations.Test)4 OrOperator (com.linkedin.pinot.core.operator.filter.OrOperator)3 EmptyFilterOperator (com.linkedin.pinot.core.operator.filter.EmptyFilterOperator)2 FilterOperator (com.linkedin.pinot.common.request.FilterOperator)1 Operator (com.linkedin.pinot.core.common.Operator)1 BitmapBasedFilterOperator (com.linkedin.pinot.core.operator.filter.BitmapBasedFilterOperator)1 MatchEntireSegmentOperator (com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator)1 ScanBasedFilterOperator (com.linkedin.pinot.core.operator.filter.ScanBasedFilterOperator)1 SortedInvertedIndexBasedFilterOperator (com.linkedin.pinot.core.operator.filter.SortedInvertedIndexBasedFilterOperator)1 StarTreeIndexOperator (com.linkedin.pinot.core.operator.filter.StarTreeIndexOperator)1 Comparator (java.util.Comparator)1 HashMap (java.util.HashMap)1