Examples with RangePredicate com.linkedin.pinot.core.common.predicate.RangePredicate used on opensource projects

Example 1 with RangePredicate

use of com.linkedin.pinot.core.common.predicate.RangePredicate in project pinot by linkedin.

the class Predicate method newPredicate.

public static Predicate newPredicate(FilterQueryTree filterQueryTree) {
    assert (filterQueryTree.getChildren() == null) || filterQueryTree.getChildren().isEmpty();
    final FilterOperator filterType = filterQueryTree.getOperator();
    final String column = filterQueryTree.getColumn();
    final List<String> value = filterQueryTree.getValue();
    Predicate predicate = null;
    switch(filterType) {
        case EQUALITY:
            predicate = new EqPredicate(column, value);
            break;
        case RANGE:
            predicate = new RangePredicate(column, value);
            break;
        case REGEX:
            predicate = new RegexPredicate(column, value);
            break;
        case NOT:
            predicate = new NEqPredicate(column, value);
            break;
        case NOT_IN:
            predicate = new NotInPredicate(column, value);
            break;
        case IN:
            predicate = new InPredicate(column, value);
            break;
        default:
            throw new UnsupportedOperationException("Unsupported filterType:" + filterType);
    }
    return predicate;
}

Example 2 with RangePredicate

use of com.linkedin.pinot.core.common.predicate.RangePredicate in project pinot by linkedin.

the class ColumnValueSegmentPruner method pruneSegment.

/**
   * Helper method to determine if a segment can be pruned based on the column min/max value in segment metadata and
   * the predicates on time column. The algorithm is as follows:
   *
   * <ul>
   *   <li> For leaf node: Returns true if there is a predicate on the column and apply the predicate would result in
   *   filtering out all docs of the segment, false otherwise. </li>
   *   <li> For non-leaf AND node: True if any of its children returned true, false otherwise. </li>
   *   <li> For non-leaf OR node: True if all its children returned true, false otherwise. </li>
   * </ul>
   *
   * @param filterQueryTree Filter tree for the query.
   * @param columnMetadataMap Map from column name to column metadata.
   * @return True if segment can be pruned out, false otherwise.
   */
@SuppressWarnings("unchecked")
public static boolean pruneSegment(@Nonnull FilterQueryTree filterQueryTree, @Nonnull Map<String, ColumnMetadata> columnMetadataMap) {
    FilterOperator filterOperator = filterQueryTree.getOperator();
    List<FilterQueryTree> children = filterQueryTree.getChildren();
    if (children == null || children.isEmpty()) {
        // Skip operator other than EQUALITY and RANGE
        if ((filterOperator != FilterOperator.EQUALITY) && (filterOperator != FilterOperator.RANGE)) {
            return false;
        }
        ColumnMetadata columnMetadata = columnMetadataMap.get(filterQueryTree.getColumn());
        if (columnMetadata == null) {
            // Should not reach here after DataSchemaSegmentPruner
            return true;
        }
        Comparable minValue = columnMetadata.getMinValue();
        Comparable maxValue = columnMetadata.getMaxValue();
        if (filterOperator == FilterOperator.EQUALITY) {
            // Doesn't have min/max value set in metadata
            if ((minValue == null) || (maxValue == null)) {
                return false;
            }
            // Check if the value is in the min/max range
            FieldSpec.DataType dataType = columnMetadata.getDataType();
            Comparable value = getValue(filterQueryTree.getValue().get(0), dataType);
            return (value.compareTo(minValue) < 0) || (value.compareTo(maxValue) > 0);
        } else {
            // RANGE
            // Get lower/upper boundary value
            FieldSpec.DataType dataType = columnMetadata.getDataType();
            RangePredicate rangePredicate = new RangePredicate(null, filterQueryTree.getValue());
            String lowerBoundary = rangePredicate.getLowerBoundary();
            boolean includeLowerBoundary = rangePredicate.includeLowerBoundary();
            Comparable lowerBoundaryValue = null;
            if (!lowerBoundary.equals(RangePredicate.UNBOUNDED)) {
                lowerBoundaryValue = getValue(lowerBoundary, dataType);
            }
            String upperBoundary = rangePredicate.getUpperBoundary();
            boolean includeUpperBoundary = rangePredicate.includeUpperBoundary();
            Comparable upperBoundaryValue = null;
            if (!upperBoundary.equals(RangePredicate.UNBOUNDED)) {
                upperBoundaryValue = getValue(upperBoundary, dataType);
            }
            // Check if the range is valid
            if ((lowerBoundaryValue != null) && (upperBoundaryValue != null)) {
                if (includeLowerBoundary && includeUpperBoundary) {
                    if (lowerBoundaryValue.compareTo(upperBoundaryValue) > 0) {
                        return true;
                    }
                } else {
                    if (lowerBoundaryValue.compareTo(upperBoundaryValue) >= 0) {
                        return true;
                    }
                }
            }
            // Doesn't have min/max value set in metadata
            if ((minValue == null) || (maxValue == null)) {
                return false;
            }
            if (lowerBoundaryValue != null) {
                if (includeLowerBoundary) {
                    if (lowerBoundaryValue.compareTo(maxValue) > 0) {
                        return true;
                    }
                } else {
                    if (lowerBoundaryValue.compareTo(maxValue) >= 0) {
                        return true;
                    }
                }
            }
            if (upperBoundaryValue != null) {
                if (includeUpperBoundary) {
                    if (upperBoundaryValue.compareTo(minValue) < 0) {
                        return true;
                    }
                } else {
                    if (upperBoundaryValue.compareTo(minValue) <= 0) {
                        return true;
                    }
                }
            }
            return false;
        }
    } else {
        switch(filterOperator) {
            case AND:
                for (FilterQueryTree child : children) {
                    if (pruneSegment(child, columnMetadataMap)) {
                        return true;
                    }
                }
                return false;
            case OR:
                for (FilterQueryTree child : children) {
                    if (!pruneSegment(child, columnMetadataMap)) {
                        return false;
                    }
                }
                return true;
            default:
                throw new IllegalStateException("Unsupported filter operator: " + filterOperator);
        }
    }
}

Example 3 with RangePredicate

use of com.linkedin.pinot.core.common.predicate.RangePredicate in project pinot by linkedin.

the class RangeOfflineDictionaryPredicateEvaluatorTest method createPredicate.

private RangePredicate createPredicate(int lower, boolean inclLower, int upper, boolean inclUpper) {
    RangePredicate predicate = mock(RangePredicate.class);
    when(predicate.includeLowerBoundary()).thenReturn(inclLower);
    when(predicate.includeUpperBoundary()).thenReturn(inclUpper);
    String lowerStr = "lower";
    if (lower == 0) {
        lowerStr = "*";
    }
    String upperStr = "upper";
    if (upper == DICT_LEN - 1) {
        upperStr = "*";
    }
    when(predicate.getLowerBoundary()).thenReturn(lowerStr);
    when(predicate.getUpperBoundary()).thenReturn(upperStr);
    return predicate;
}

Example 4 with RangePredicate

use of com.linkedin.pinot.core.common.predicate.RangePredicate in project pinot by linkedin.

the class RealtimeSegmentTest method testNoRangeMatchFilteringMetricPredicateWithoutInvIdx.

@Test
public void testNoRangeMatchFilteringMetricPredicateWithoutInvIdx() throws Exception {
    DataSource ds1 = segmentWithoutInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("[0\t\t100)");
    Predicate predicate = new RangePredicate("count", rhs);
    ScanBasedFilterOperator op = new ScanBasedFilterOperator(predicate, ds1, 0, segmentWithoutInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    int counter = 0;
    int docId = iterator.next();
    while (docId != Constants.EOF) {
        // shouldn't reach here.
        Assert.assertTrue(false);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 0);
}

Example 5 with RangePredicate

use of com.linkedin.pinot.core.common.predicate.RangePredicate in project pinot by linkedin.

the class RealtimeSegmentTest method testRangeMatchFilteringMetricPredicateWithoutInvIdx.

@Test
public void testRangeMatchFilteringMetricPredicateWithoutInvIdx() throws Exception {
    DataSource ds1 = segmentWithoutInvIdx.getDataSource("count");
    List<String> rhs = new ArrayList<String>();
    rhs.add("[0\t\t*)");
    Predicate predicate = new RangePredicate("count", rhs);
    ScanBasedFilterOperator op = new ScanBasedFilterOperator(predicate, ds1, 0, segmentWithoutInvIdx.getRawDocumentCount() - 1);
    Block b = op.nextBlock();
    BlockDocIdIterator iterator = b.getBlockDocIdSet().iterator();
    DataSource ds2 = segmentWithoutInvIdx.getDataSource("count");
    BlockSingleValIterator blockValIterator = (BlockSingleValIterator) ds2.nextBlock().getBlockValueSet().iterator();
    int docId = iterator.next();
    int counter = 0;
    while (docId != Constants.EOF) {
        blockValIterator.skipTo(docId);
        Assert.assertEquals(ds1.getDictionary().get(blockValIterator.nextIntVal()), 890662862);
        docId = iterator.next();
        counter++;
    }
    Assert.assertEquals(counter, 100000);
}