Examples with FilterOperator com.linkedin.pinot.common.request.FilterOperator used on opensource projects

Example 1 with FilterOperator

use of com.linkedin.pinot.common.request.FilterOperator in project pinot by linkedin.

the class RangeMergeOptimizer method optimizeRanges.

/**
   * Recursive method that performs the actual optimization of merging range predicates.
   *
   * @param current Current node being visited in the DFS of the filter query tree.
   * @param timeColumn Name of time column
   * @return Returns the optimized filter query tree
   */
@Nonnull
private static FilterQueryTree optimizeRanges(@Nonnull FilterQueryTree current, @Nullable String timeColumn) {
    if (timeColumn == null) {
        return current;
    }
    List<FilterQueryTree> children = current.getChildren();
    if (children == null || children.isEmpty()) {
        return current;
    }
    // For OR, we optimize all its children, but do not propagate up.
    FilterOperator operator = current.getOperator();
    if (operator == FilterOperator.OR) {
        int length = children.size();
        for (int i = 0; i < length; i++) {
            children.set(i, optimizeRanges(children.get(i), timeColumn));
        }
        return current;
    }
    // After this point, since the node has children, it can only be an 'AND' node (only OR/AND supported).
    assert operator == FilterOperator.AND;
    List<FilterQueryTree> newChildren = new ArrayList<>();
    List<String> intersect = null;
    for (FilterQueryTree child : children) {
        FilterQueryTree newChild = optimizeRanges(child, timeColumn);
        if (newChild.getOperator() == FilterOperator.RANGE && newChild.getColumn().equals(timeColumn)) {
            List<String> value = newChild.getValue();
            intersect = (intersect == null) ? value : intersectRanges(intersect, value);
        } else {
            newChildren.add(newChild);
        }
    }
    if (newChildren.isEmpty()) {
        return new FilterQueryTree(timeColumn, intersect, FilterOperator.RANGE, null);
    } else {
        if (intersect != null) {
            newChildren.add(new FilterQueryTree(timeColumn, intersect, FilterOperator.RANGE, null));
        }
        return new FilterQueryTree(null, null, FilterOperator.AND, newChildren);
    }
}

Example 2 with FilterOperator

use of com.linkedin.pinot.common.request.FilterOperator in project pinot by linkedin.

the class Predicate method newPredicate.

public static Predicate newPredicate(FilterQueryTree filterQueryTree) {
    assert (filterQueryTree.getChildren() == null) || filterQueryTree.getChildren().isEmpty();
    final FilterOperator filterType = filterQueryTree.getOperator();
    final String column = filterQueryTree.getColumn();
    final List<String> value = filterQueryTree.getValue();
    Predicate predicate = null;
    switch(filterType) {
        case EQUALITY:
            predicate = new EqPredicate(column, value);
            break;
        case RANGE:
            predicate = new RangePredicate(column, value);
            break;
        case REGEX:
            predicate = new RegexPredicate(column, value);
            break;
        case NOT:
            predicate = new NEqPredicate(column, value);
            break;
        case NOT_IN:
            predicate = new NotInPredicate(column, value);
            break;
        case IN:
            predicate = new InPredicate(column, value);
            break;
        default:
            throw new UnsupportedOperationException("Unsupported filterType:" + filterType);
    }
    return predicate;
}

Example 3 with FilterOperator

use of com.linkedin.pinot.common.request.FilterOperator in project pinot by linkedin.

the class ColumnValueSegmentPruner method pruneSegment.

/**
   * Helper method to determine if a segment can be pruned based on the column min/max value in segment metadata and
   * the predicates on time column. The algorithm is as follows:
   *
   * <ul>
   *   <li> For leaf node: Returns true if there is a predicate on the column and apply the predicate would result in
   *   filtering out all docs of the segment, false otherwise. </li>
   *   <li> For non-leaf AND node: True if any of its children returned true, false otherwise. </li>
   *   <li> For non-leaf OR node: True if all its children returned true, false otherwise. </li>
   * </ul>
   *
   * @param filterQueryTree Filter tree for the query.
   * @param columnMetadataMap Map from column name to column metadata.
   * @return True if segment can be pruned out, false otherwise.
   */
@SuppressWarnings("unchecked")
public static boolean pruneSegment(@Nonnull FilterQueryTree filterQueryTree, @Nonnull Map<String, ColumnMetadata> columnMetadataMap) {
    FilterOperator filterOperator = filterQueryTree.getOperator();
    List<FilterQueryTree> children = filterQueryTree.getChildren();
    if (children == null || children.isEmpty()) {
        // Skip operator other than EQUALITY and RANGE
        if ((filterOperator != FilterOperator.EQUALITY) && (filterOperator != FilterOperator.RANGE)) {
            return false;
        }
        ColumnMetadata columnMetadata = columnMetadataMap.get(filterQueryTree.getColumn());
        if (columnMetadata == null) {
            // Should not reach here after DataSchemaSegmentPruner
            return true;
        }
        Comparable minValue = columnMetadata.getMinValue();
        Comparable maxValue = columnMetadata.getMaxValue();
        if (filterOperator == FilterOperator.EQUALITY) {
            // Doesn't have min/max value set in metadata
            if ((minValue == null) || (maxValue == null)) {
                return false;
            }
            // Check if the value is in the min/max range
            FieldSpec.DataType dataType = columnMetadata.getDataType();
            Comparable value = getValue(filterQueryTree.getValue().get(0), dataType);
            return (value.compareTo(minValue) < 0) || (value.compareTo(maxValue) > 0);
        } else {
            // RANGE
            // Get lower/upper boundary value
            FieldSpec.DataType dataType = columnMetadata.getDataType();
            RangePredicate rangePredicate = new RangePredicate(null, filterQueryTree.getValue());
            String lowerBoundary = rangePredicate.getLowerBoundary();
            boolean includeLowerBoundary = rangePredicate.includeLowerBoundary();
            Comparable lowerBoundaryValue = null;
            if (!lowerBoundary.equals(RangePredicate.UNBOUNDED)) {
                lowerBoundaryValue = getValue(lowerBoundary, dataType);
            }
            String upperBoundary = rangePredicate.getUpperBoundary();
            boolean includeUpperBoundary = rangePredicate.includeUpperBoundary();
            Comparable upperBoundaryValue = null;
            if (!upperBoundary.equals(RangePredicate.UNBOUNDED)) {
                upperBoundaryValue = getValue(upperBoundary, dataType);
            }
            // Check if the range is valid
            if ((lowerBoundaryValue != null) && (upperBoundaryValue != null)) {
                if (includeLowerBoundary && includeUpperBoundary) {
                    if (lowerBoundaryValue.compareTo(upperBoundaryValue) > 0) {
                        return true;
                    }
                } else {
                    if (lowerBoundaryValue.compareTo(upperBoundaryValue) >= 0) {
                        return true;
                    }
                }
            }
            // Doesn't have min/max value set in metadata
            if ((minValue == null) || (maxValue == null)) {
                return false;
            }
            if (lowerBoundaryValue != null) {
                if (includeLowerBoundary) {
                    if (lowerBoundaryValue.compareTo(maxValue) > 0) {
                        return true;
                    }
                } else {
                    if (lowerBoundaryValue.compareTo(maxValue) >= 0) {
                        return true;
                    }
                }
            }
            if (upperBoundaryValue != null) {
                if (includeUpperBoundary) {
                    if (upperBoundaryValue.compareTo(minValue) < 0) {
                        return true;
                    }
                } else {
                    if (upperBoundaryValue.compareTo(minValue) <= 0) {
                        return true;
                    }
                }
            }
            return false;
        }
    } else {
        switch(filterOperator) {
            case AND:
                for (FilterQueryTree child : children) {
                    if (pruneSegment(child, columnMetadataMap)) {
                        return true;
                    }
                }
                return false;
            case OR:
                for (FilterQueryTree child : children) {
                    if (!pruneSegment(child, columnMetadataMap)) {
                        return false;
                    }
                }
                return true;
            default:
                throw new IllegalStateException("Unsupported filter operator: " + filterOperator);
        }
    }
}

Example 4 with FilterOperator

use of com.linkedin.pinot.common.request.FilterOperator in project pinot by linkedin.

the class SegmentQueryProcessor method filterDocIds.

private List<Integer> filterDocIds(FilterQueryTree filterQueryTree, List<Integer> inputDocIds) {
    // If no filter predicate, return the input without filtering.
    if (filterQueryTree == null) {
        List<Integer> allDocs = new ArrayList<>(_totalDocs);
        for (int i = 0; i < _totalDocs; ++i) {
            allDocs.add(i);
        }
        return allDocs;
    }
    final List<FilterQueryTree> childFilters = filterQueryTree.getChildren();
    final boolean isLeaf = (childFilters == null) || childFilters.isEmpty();
    if (isLeaf) {
        FilterOperator filterType = filterQueryTree.getOperator();
        String column = filterQueryTree.getColumn();
        final List<String> value = filterQueryTree.getValue();
        return getMatchingDocIds(inputDocIds, filterType, column, value);
    }
    List<Integer> result = filterDocIds(childFilters.get(0), inputDocIds);
    final FilterOperator operator = filterQueryTree.getOperator();
    for (int i = 1; i < childFilters.size(); ++i) {
        //      List<Integer> childResult = operator.equals(FilterOperator.AND) ? filterDocIds(childFilters.get(i), result)
        //          : filterDocIds(childFilters.get(i), inputDocIds);
        List<Integer> childResult = filterDocIds(childFilters.get(i), inputDocIds);
        result = combine(result, childResult, operator);
    }
    return result;
}

Example 5 with FilterOperator

use of com.linkedin.pinot.common.request.FilterOperator in project pinot by linkedin.

the class FilterPlanNode method constructPhysicalOperator.

/**
   * Helper method to build the operator tree from the filter query tree.
   * @param filterQueryTree
   * @param segment Index segment
   * @param optimizeAlwaysFalse Optimize isResultEmpty predicates
   * @return Filter Operator created
   */
@VisibleForTesting
public static BaseFilterOperator constructPhysicalOperator(FilterQueryTree filterQueryTree, IndexSegment segment, boolean optimizeAlwaysFalse) {
    BaseFilterOperator ret;
    if (null == filterQueryTree) {
        return new MatchEntireSegmentOperator(segment.getSegmentMetadata().getTotalRawDocs());
    }
    final List<FilterQueryTree> childFilters = filterQueryTree.getChildren();
    final boolean isLeaf = (childFilters == null) || childFilters.isEmpty();
    if (!isLeaf) {
        int numChildrenAlwaysFalse = 0;
        int numChildren = childFilters.size();
        List<BaseFilterOperator> operators = new ArrayList<>();
        final FilterOperator filterType = filterQueryTree.getOperator();
        for (final FilterQueryTree query : childFilters) {
            BaseFilterOperator childOperator = constructPhysicalOperator(query, segment, optimizeAlwaysFalse);
            // Count number of always false children.
            if (optimizeAlwaysFalse && childOperator.isResultEmpty()) {
                numChildrenAlwaysFalse++;
                // Early bailout for 'AND' as soon as one of the children always evaluates to false.
                if (filterType == FilterOperator.AND) {
                    break;
                }
            }
            operators.add(childOperator);
        }
        ret = buildNonLeafOperator(filterType, operators, numChildrenAlwaysFalse, numChildren, optimizeAlwaysFalse);
    } else {
        final FilterOperator filterType = filterQueryTree.getOperator();
        final String column = filterQueryTree.getColumn();
        Predicate predicate = Predicate.newPredicate(filterQueryTree);
        DataSource ds;
        ds = segment.getDataSource(column);
        DataSourceMetadata dataSourceMetadata = ds.getDataSourceMetadata();
        BaseFilterOperator baseFilterOperator;
        int startDocId = 0;
        //end is inclusive
        int endDocId = segment.getSegmentMetadata().getTotalRawDocs() - 1;
        if (dataSourceMetadata.hasInvertedIndex()) {
            // range evaluation based on inv index is inefficient, so do this only if is NOT range.
            if (!filterType.equals(FilterOperator.RANGE)) {
                if (dataSourceMetadata.isSingleValue() && dataSourceMetadata.isSorted()) {
                    // if the column is sorted use sorted inverted index based implementation
                    baseFilterOperator = new SortedInvertedIndexBasedFilterOperator(predicate, ds, startDocId, endDocId);
                } else {
                    baseFilterOperator = new BitmapBasedFilterOperator(predicate, ds, startDocId, endDocId);
                }
            } else {
                baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
            }
        } else {
            baseFilterOperator = new ScanBasedFilterOperator(predicate, ds, startDocId, endDocId);
        }
        ret = baseFilterOperator;
    }
    // If operator evaluates to false, then just return an empty operator.
    if (ret.isResultEmpty()) {
        ret = new EmptyFilterOperator();
    }
    return ret;
}