Examples with FilterQueryTree com.linkedin.pinot.common.utils.request.FilterQueryTree used on opensource projects

Example 16 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class FilterPlanNode method run.

@Override
public Operator run() {
    long start = System.currentTimeMillis();
    Operator operator;
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(_brokerRequest);
    if (_segment.getSegmentMetadata().hasStarTree() && RequestUtils.isFitForStarTreeIndex(_segment.getSegmentMetadata(), filterQueryTree, _brokerRequest)) {
        operator = new StarTreeIndexOperator(_segment, _brokerRequest);
    } else {
        operator = constructPhysicalOperator(filterQueryTree, _segment, _optimizeAlwaysFalse);
    }
    long end = System.currentTimeMillis();
    LOGGER.debug("FilterPlanNode.run took:{}", (end - start));
    return operator;
}

Example 17 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class FilterPlanNode method constructPhysicalOperator.

/**
   * Helper method to build the operator tree from the filter query tree.
   * @param filterQueryTree
   * @param segment Index segment
   * @param optimizeAlwaysFalse Optimize isResultEmpty predicates
   * @return Filter Operator created
   */
@VisibleForTesting
public static BaseFilterOperator constructPhysicalOperator(FilterQueryTree filterQueryTree, IndexSegment segment, boolean optimizeAlwaysFalse) {
    BaseFilterOperator ret;
    if (null == filterQueryTree) {
        return new MatchEntireSegmentOperator(segment.getSegmentMetadata().getTotalRawDocs());
    }
    final List<FilterQueryTree> childFilters = filterQueryTree.getChildren();
    final boolean isLeaf = (childFilters == null) || childFilters.isEmpty();
    if (!isLeaf) {
        int numChildrenAlwaysFalse = 0;
        int numChildren = childFilters.size();
        List<BaseFilterOperator> operators = new ArrayList<>();
        final FilterOperator filterType = filterQueryTree.getOperator();
        for (final FilterQueryTree query : childFilters) {
            BaseFilterOperator childOperator = constructPhysicalOperator(query, segment, optimizeAlwaysFalse);
            // Count number of always false children.
            if (optimizeAlwaysFalse && childOperator.isResultEmpty()) {
                numChildrenAlwaysFalse++;
                // Early bailout for 'AND' as soon as one of the children always evaluates to false.
                if (filterType == FilterOperator.AND) {
                    break;
                }
            }
            operators.add(childOperator);
        }
        ret = buildNonLeafOperator(filterType, operators, numChildrenAlwaysFalse, numChildren, optimizeAlwaysFalse);
    } else {
        final FilterOperator filterType = filterQueryTree.getOperator();
        final String column = filterQueryTree.getColumn();
        Predicate predicate = Predicate.newPredicate(filterQueryTree);
        DataSource ds;
        ds = segment.getDataSource(column);
        DataSourceMetadata dataSourceMetadata = ds.getDataSourceMetadata();
        BaseFilterOperator baseFilterOperator;
        int startDocId = 0;
        //end is inclusive
        int endDocId = segment.getSegmentMetadata().getTotalRawDocs() - 1;
        if (dataSourceMetadata.hasInvertedIndex()) {
            // range evaluation based on inv index is inefficient, so do this only if is NOT range.
            if (!filterType.equals(FilterOperator.RANGE)) {
                if (dataSourceMetadata.isSingleValue() && dataSourceMetadata.isSorted()) {
                    // if the column is sorted use sorted inverted index based implementation
                    baseFilterOperator = new SortedInvertedIndexBasedFilterOperator(predicate, ds, startDocId, endDocId);
                } else {
                    baseFilterOperator = new BitmapBasedFilterOperator(predicate, ds, startDocId, endDocId);
                }
            } else {
                baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
            }
        } else {
            baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
        }
        ret = baseFilterOperator;
    }
    // If operator evaluates to false, then just return an empty operator.
    if (ret.isResultEmpty()) {
        ret = new EmptyFilterOperator();
    }
    return ret;
}

Example 18 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class BrokerRequestOptimizer method optimizeFilterQueryTree.

/**
   * Optimizes the filter query tree of a broker request in place.
   * @param brokerRequest The broker request to optimize
   * @param timeColumn time column
   */
private void optimizeFilterQueryTree(BrokerRequest brokerRequest, String timeColumn, OptimizationFlags optimizationFlags) {
    FilterQueryTree filterQueryTree = null;
    FilterQuery q = brokerRequest.getFilterQuery();
    if (q == null || brokerRequest.getFilterSubQueryMap() == null) {
        return;
    }
    filterQueryTree = RequestUtils.buildFilterQuery(q.getId(), brokerRequest.getFilterSubQueryMap().getFilterQueryMap());
    FilterQueryOptimizerRequest.FilterQueryOptimizerRequestBuilder builder = new FilterQueryOptimizerRequest.FilterQueryOptimizerRequestBuilder();
    FilterQueryOptimizerRequest request = builder.setFilterQueryTree(filterQueryTree).setTimeColumn(timeColumn).build();
    if (optimizationFlags == null) {
        for (FilterQueryTreeOptimizer filterQueryTreeOptimizer : FILTER_QUERY_TREE_OPTIMIZERS) {
            filterQueryTree = filterQueryTreeOptimizer.optimize(request);
            // Optimizers may return a new tree instead of in-place optimization
            request.setFilterQueryTree(filterQueryTree);
        }
    } else {
        if (optimizationFlags.isOptimizationEnabled("filterQueryTree")) {
            for (FilterQueryTreeOptimizer filterQueryTreeOptimizer : FILTER_QUERY_TREE_OPTIMIZERS) {
                if (optimizationFlags.isOptimizationEnabled(filterQueryTreeOptimizer.getOptimizationName())) {
                    filterQueryTree = filterQueryTreeOptimizer.optimize(request);
                    // Optimizers may return a new tree instead of in-place optimization
                    request.setFilterQueryTree(filterQueryTree);
                }
            }
        }
    }
    RequestUtils.generateFilterFromTree(filterQueryTree, brokerRequest);
}

Example 19 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class FlattenNestedPredicatesFilterQueryTreeOptimizer method optimize.

@Override
public FilterQueryTree optimize(FilterQueryOptimizerRequest request) {
    FilterQueryTree filterQueryTree = request.getFilterQueryTree();
    flatten(filterQueryTree, null, MAX_OPTIMIZING_DEPTH);
    return filterQueryTree;
}

Example 20 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class FlattenNestedPredicatesFilterQueryTreeOptimizer method flatten.

/**
   * Flatten the operators if parent and child have the same AND or OR operator.
   * (e.g. AND( a, AND (b, c)) is the same as AND(a, b, c). This helps when we re-order
   * operators for performance.
   *
   * It does so by looking at the operator of the 'parent' and 'node'. If they are same, and
   * collapsible, then all the children of 'node' are moved one level up to be siblings of
   * 'node', rendering 'node' childless. 'node' is then removed from 'parent's children list.
   *
   * @param node The node whose children are to be moved up one level if criteria is satisfied.
   * @param parent Node's parent who will inherit node's children if criteria is satisfied.
   * @param maxDepth is the maximum depth to which we recurse
   */
private void flatten(FilterQueryTree node, FilterQueryTree parent, int maxDepth) {
    if (node == null || node.getChildren() == null || maxDepth == 0) {
        return;
    }
    // Flatten all the children first.
    List<FilterQueryTree> toFlatten = new ArrayList<>(node.getChildren().size());
    for (FilterQueryTree child : node.getChildren()) {
        if (child.getChildren() != null && !child.getChildren().isEmpty()) {
            toFlatten.add(child);
        }
    }
    for (FilterQueryTree child : toFlatten) {
        flatten(child, node, maxDepth - 1);
    }
    if (parent == null) {
        return;
    }
    if (node.getOperator() == parent.getOperator() && (node.getOperator() == FilterOperator.OR || node.getOperator() == FilterOperator.AND)) {
        // Move all of 'node's children one level up. If 'node' has no children left, remove it from parent's list.
        List<FilterQueryTree> children = node.getChildren();
        Iterator<FilterQueryTree> it = children.iterator();
        while (it.hasNext()) {
            parent.getChildren().add(it.next());
            it.remove();
        }
        // 'node' is now childless
        // Remove this node from its parent's list.
        parent.getChildren().remove(node);
    }
}