Examples with BaseFilterOperator com.linkedin.pinot.core.operator.filter.BaseFilterOperator used on opensource projects

Example 6 with BaseFilterOperator

use of com.linkedin.pinot.core.operator.filter.BaseFilterOperator in project pinot by linkedin.

the class FilterPlanNode method reorder.

/**
   * Re orders operators, puts Sorted -> Inverted and then Raw scan. TODO: With Inverted, we can
   * further optimize based on cardinality
   * @param operators
   */
private static void reorder(List<BaseFilterOperator> operators) {
    final Map<Operator, Integer> operatorPriorityMap = new HashMap<Operator, Integer>();
    for (Operator operator : operators) {
        Integer priority = Integer.MAX_VALUE;
        if (operator instanceof SortedInvertedIndexBasedFilterOperator) {
            priority = 0;
        } else if (operator instanceof AndOperator) {
            priority = 1;
        } else if (operator instanceof BitmapBasedFilterOperator) {
            priority = 2;
        } else if (operator instanceof ScanBasedFilterOperator) {
            priority = 3;
        } else if (operator instanceof OrOperator) {
            priority = 4;
        }
        operatorPriorityMap.put(operator, priority);
    }
    Comparator<? super Operator> comparator = new Comparator<Operator>() {

        @Override
        public int compare(Operator o1, Operator o2) {
            return Integer.compare(operatorPriorityMap.get(o1), operatorPriorityMap.get(o2));
        }
    };
    Collections.sort(operators, comparator);
}

Example 7 with BaseFilterOperator

use of com.linkedin.pinot.core.operator.filter.BaseFilterOperator in project pinot by linkedin.

the class FilterPlanNode method buildNonLeafOperator.

/**
   * Helper method to build AND/OR operators.
   * <ul>
   *   <li> Returns {@link EmptyFilterOperator} if at least on child always evaluates to false for AND. </li>
   *   <li> Returns {@link EmptyFilterOperator} if all children always evaluates to false for OR. </li>
   *   <li> Returns {@link AndOperator} or {@link OrOperator} based on filterType, otherwise. </li>
   * </ul>
   * @param filterType AND/OR
   * @param nonFalseChildren Children that are not alwaysFalse.
   * @param numChildrenAlwaysFalse Number of children that are always false.
   * @param numChildren Total number of children.
   * @param optimizeAlwaysFalse Optimize alwaysFalse predicates
   * @return Filter Operator created
   */
private static BaseFilterOperator buildNonLeafOperator(FilterOperator filterType, List<BaseFilterOperator> nonFalseChildren, int numChildrenAlwaysFalse, int numChildren, boolean optimizeAlwaysFalse) {
    BaseFilterOperator operator;
    switch(filterType) {
        case AND:
            if (optimizeAlwaysFalse && numChildrenAlwaysFalse > 0) {
                operator = new EmptyFilterOperator();
            } else {
                reorder(nonFalseChildren);
                operator = new AndOperator(nonFalseChildren);
            }
            break;
        case OR:
            if (optimizeAlwaysFalse && numChildrenAlwaysFalse == numChildren) {
                operator = new EmptyFilterOperator();
            } else {
                reorder(nonFalseChildren);
                operator = new OrOperator(nonFalseChildren);
            }
            break;
        default:
            throw new UnsupportedOperationException("Not support filter type - " + filterType + " with children operators");
    }
    return operator;
}

Example 8 with BaseFilterOperator

use of com.linkedin.pinot.core.operator.filter.BaseFilterOperator in project pinot by linkedin.

the class FilterPlanNode method constructPhysicalOperator.

/**
   * Helper method to build the operator tree from the filter query tree.
   * @param filterQueryTree
   * @param segment Index segment
   * @param optimizeAlwaysFalse Optimize isResultEmpty predicates
   * @return Filter Operator created
   */
@VisibleForTesting
public static BaseFilterOperator constructPhysicalOperator(FilterQueryTree filterQueryTree, IndexSegment segment, boolean optimizeAlwaysFalse) {
    BaseFilterOperator ret;
    if (null == filterQueryTree) {
        return new MatchEntireSegmentOperator(segment.getSegmentMetadata().getTotalRawDocs());
    }
    final List<FilterQueryTree> childFilters = filterQueryTree.getChildren();
    final boolean isLeaf = (childFilters == null) || childFilters.isEmpty();
    if (!isLeaf) {
        int numChildrenAlwaysFalse = 0;
        int numChildren = childFilters.size();
        List<BaseFilterOperator> operators = new ArrayList<>();
        final FilterOperator filterType = filterQueryTree.getOperator();
        for (final FilterQueryTree query : childFilters) {
            BaseFilterOperator childOperator = constructPhysicalOperator(query, segment, optimizeAlwaysFalse);
            // Count number of always false children.
            if (optimizeAlwaysFalse && childOperator.isResultEmpty()) {
                numChildrenAlwaysFalse++;
                // Early bailout for 'AND' as soon as one of the children always evaluates to false.
                if (filterType == FilterOperator.AND) {
                    break;
                }
            }
            operators.add(childOperator);
        }
        ret = buildNonLeafOperator(filterType, operators, numChildrenAlwaysFalse, numChildren, optimizeAlwaysFalse);
    } else {
        final FilterOperator filterType = filterQueryTree.getOperator();
        final String column = filterQueryTree.getColumn();
        Predicate predicate = Predicate.newPredicate(filterQueryTree);
        DataSource ds;
        ds = segment.getDataSource(column);
        DataSourceMetadata dataSourceMetadata = ds.getDataSourceMetadata();
        BaseFilterOperator baseFilterOperator;
        int startDocId = 0;
        //end is inclusive
        int endDocId = segment.getSegmentMetadata().getTotalRawDocs() - 1;
        if (dataSourceMetadata.hasInvertedIndex()) {
            // range evaluation based on inv index is inefficient, so do this only if is NOT range.
            if (!filterType.equals(FilterOperator.RANGE)) {
                if (dataSourceMetadata.isSingleValue() && dataSourceMetadata.isSorted()) {
                    // if the column is sorted use sorted inverted index based implementation
                    baseFilterOperator = new SortedInvertedIndexBasedFilterOperator(predicate, ds, startDocId, endDocId);
                } else {
                    baseFilterOperator = new BitmapBasedFilterOperator(predicate, ds, startDocId, endDocId);
                }
            } else {
                baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
            }
        } else {
            baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
        }
        ret = baseFilterOperator;
    }
    // If operator evaluates to false, then just return an empty operator.
    if (ret.isResultEmpty()) {
        ret = new EmptyFilterOperator();
    }
    return ret;
}

Example 9 with BaseFilterOperator

use of com.linkedin.pinot.core.operator.filter.BaseFilterOperator in project pinot by linkedin.

the class RawIndexBenchmark method profileLookups.

/**
   * Profiles the lookup time for a given column, for the given docIds.
   *
   * @param segment Segment to profile
   * @param column Column to profile
   * @param docIds DocIds to lookup on the column
   * @return Time take in millis for the lookups
   */
private long profileLookups(IndexSegment segment, String column, int[] docIds) {
    BaseFilterOperator filterOperator = new TestFilterOperator(docIds);
    BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(filterOperator, docIds.length, DocIdSetPlanNode.MAX_DOC_PER_CALL);
    ProjectionBlock projectionBlock;
    MProjectionOperator projectionOperator = new MProjectionOperator(buildDataSourceMap(segment), docIdSetOperator);
    long start = System.currentTimeMillis();
    while ((projectionBlock = (ProjectionBlock) projectionOperator.nextBlock()) != null) {
        ProjectionBlockValSet blockValueSet = (ProjectionBlockValSet) projectionBlock.getBlockValueSet(column);
        blockValueSet.getDoubleValuesSV();
    }
    return (System.currentTimeMillis() - start);
}

Example 10 with BaseFilterOperator

use of com.linkedin.pinot.core.operator.filter.BaseFilterOperator in project pinot by linkedin.

the class AndOperatorTest method testIntersectionForTwoLists.

@Test
public void testIntersectionForTwoLists() {
    int[] list1 = new int[] { 2, 3, 10, 15, 16, 28 };
    int[] list2 = new int[] { 3, 6, 8, 20, 28 };
    List<BaseFilterOperator> operators = new ArrayList<>();
    operators.add(makeFilterOperator(list1));
    operators.add(makeFilterOperator(list2));
    final AndOperator andOperator = new AndOperator(operators);
    andOperator.open();
    BaseFilterBlock block;
    while ((block = andOperator.getNextBlock()) != null) {
        final BlockDocIdSet blockDocIdSet = block.getBlockDocIdSet();
        final BlockDocIdIterator iterator = blockDocIdSet.iterator();
        int docId;
        while ((docId = iterator.next()) != Constants.EOF) {
        //        System.out.println(docId);
        }
    }
    andOperator.close();
}