Examples with FilterQueryTree com.linkedin.pinot.common.utils.request.FilterQueryTree used on opensource projects

Example 6 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class BetweenPredicateAstNode method buildFilterQueryTree.

@Override
public FilterQueryTree buildFilterQueryTree() {
    if (getChildren().size() == 2) {
        try {
            LiteralAstNode left = (LiteralAstNode) getChildren().get(0);
            LiteralAstNode right = (LiteralAstNode) getChildren().get(1);
            return new FilterQueryTree(_identifier, Collections.singletonList("[" + left.getValueAsString() + "\t\t" + right.getValueAsString() + "]"), FilterOperator.RANGE, null);
        } catch (ClassCastException e) {
            throw new Pql2CompilationException("BETWEEN clause was expecting two literal AST nodes, got " + getChildren().get(0) + " and " + getChildren().get(1));
        }
    } else {
        throw new Pql2CompilationException("BETWEEN clause does not have two children nodes");
    }
}

Example 7 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class StarTreeIndexOperator method initPredicatesToEvaluate.

private void initPredicatesToEvaluate() {
    FilterQueryTree filterTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    // Find all filter columns
    if (filterTree != null) {
        if (filterTree.getChildren() != null && !filterTree.getChildren().isEmpty()) {
            for (FilterQueryTree childFilter : filterTree.getChildren()) {
                // Nested filters are not supported
                assert childFilter.getChildren() == null || childFilter.getChildren().isEmpty();
                processFilterTree(childFilter);
            }
        } else {
            processFilterTree(filterTree);
        }
    }
    // Group by columns, we cannot lose group by columns during traversal
    GroupBy groupBy = brokerRequest.getGroupBy();
    if (groupBy != null) {
        groupByColumns.addAll(groupBy.getColumns());
    }
}

Example 8 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class ColumnValueSegmentPruner method pruneSegment.

/**
   * Helper method to determine if a segment can be pruned based on the column min/max value in segment metadata and
   * the predicates on time column. The algorithm is as follows:
   *
   * <ul>
   *   <li> For leaf node: Returns true if there is a predicate on the column and apply the predicate would result in
   *   filtering out all docs of the segment, false otherwise. </li>
   *   <li> For non-leaf AND node: True if any of its children returned true, false otherwise. </li>
   *   <li> For non-leaf OR node: True if all its children returned true, false otherwise. </li>
   * </ul>
   *
   * @param filterQueryTree Filter tree for the query.
   * @param columnMetadataMap Map from column name to column metadata.
   * @return True if segment can be pruned out, false otherwise.
   */
@SuppressWarnings("unchecked")
public static boolean pruneSegment(@Nonnull FilterQueryTree filterQueryTree, @Nonnull Map<String, ColumnMetadata> columnMetadataMap) {
    FilterOperator filterOperator = filterQueryTree.getOperator();
    List<FilterQueryTree> children = filterQueryTree.getChildren();
    if (children == null || children.isEmpty()) {
        // Skip operator other than EQUALITY and RANGE
        if ((filterOperator != FilterOperator.EQUALITY) && (filterOperator != FilterOperator.RANGE)) {
            return false;
        }
        ColumnMetadata columnMetadata = columnMetadataMap.get(filterQueryTree.getColumn());
        if (columnMetadata == null) {
            // Should not reach here after DataSchemaSegmentPruner
            return true;
        }
        Comparable minValue = columnMetadata.getMinValue();
        Comparable maxValue = columnMetadata.getMaxValue();
        if (filterOperator == FilterOperator.EQUALITY) {
            // Doesn't have min/max value set in metadata
            if ((minValue == null) || (maxValue == null)) {
                return false;
            }
            // Check if the value is in the min/max range
            FieldSpec.DataType dataType = columnMetadata.getDataType();
            Comparable value = getValue(filterQueryTree.getValue().get(0), dataType);
            return (value.compareTo(minValue) < 0) || (value.compareTo(maxValue) > 0);
        } else {
            // RANGE
            // Get lower/upper boundary value
            FieldSpec.DataType dataType = columnMetadata.getDataType();
            RangePredicate rangePredicate = new RangePredicate(null, filterQueryTree.getValue());
            String lowerBoundary = rangePredicate.getLowerBoundary();
            boolean includeLowerBoundary = rangePredicate.includeLowerBoundary();
            Comparable lowerBoundaryValue = null;
            if (!lowerBoundary.equals(RangePredicate.UNBOUNDED)) {
                lowerBoundaryValue = getValue(lowerBoundary, dataType);
            }
            String upperBoundary = rangePredicate.getUpperBoundary();
            boolean includeUpperBoundary = rangePredicate.includeUpperBoundary();
            Comparable upperBoundaryValue = null;
            if (!upperBoundary.equals(RangePredicate.UNBOUNDED)) {
                upperBoundaryValue = getValue(upperBoundary, dataType);
            }
            // Check if the range is valid
            if ((lowerBoundaryValue != null) && (upperBoundaryValue != null)) {
                if (includeLowerBoundary && includeUpperBoundary) {
                    if (lowerBoundaryValue.compareTo(upperBoundaryValue) > 0) {
                        return true;
                    }
                } else {
                    if (lowerBoundaryValue.compareTo(upperBoundaryValue) >= 0) {
                        return true;
                    }
                }
            }
            // Doesn't have min/max value set in metadata
            if ((minValue == null) || (maxValue == null)) {
                return false;
            }
            if (lowerBoundaryValue != null) {
                if (includeLowerBoundary) {
                    if (lowerBoundaryValue.compareTo(maxValue) > 0) {
                        return true;
                    }
                } else {
                    if (lowerBoundaryValue.compareTo(maxValue) >= 0) {
                        return true;
                    }
                }
            }
            if (upperBoundaryValue != null) {
                if (includeUpperBoundary) {
                    if (upperBoundaryValue.compareTo(minValue) < 0) {
                        return true;
                    }
                } else {
                    if (upperBoundaryValue.compareTo(minValue) <= 0) {
                        return true;
                    }
                }
            }
            return false;
        }
    } else {
        switch(filterOperator) {
            case AND:
                for (FilterQueryTree child : children) {
                    if (pruneSegment(child, columnMetadataMap)) {
                        return true;
                    }
                }
                return false;
            case OR:
                for (FilterQueryTree child : children) {
                    if (!pruneSegment(child, columnMetadataMap)) {
                        return false;
                    }
                }
                return true;
            default:
                throw new IllegalStateException("Unsupported filter operator: " + filterOperator);
        }
    }
}

Example 9 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class ColumnValueSegmentPruner method prune.

@Override
public boolean prune(@Nonnull IndexSegment segment, @Nonnull BrokerRequest brokerRequest) {
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    if (filterQueryTree == null) {
        return false;
    }
    // For realtime segment, this map can be null.
    Map<String, ColumnMetadata> columnMetadataMap = ((SegmentMetadataImpl) segment.getSegmentMetadata()).getColumnMetadataMap();
    return (columnMetadataMap != null) && pruneSegment(filterQueryTree, columnMetadataMap);
}

Example 10 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class BaseHllStarTreeIndexTest method testHardCodedQueries.

void testHardCodedQueries(IndexSegment segment, Schema schema) throws Exception {
    // only use metric corresponding to columnsToDeriveHllFields
    List<String> metricNames = new ArrayList<>();
    for (String column : columnsToDeriveHllFields) {
        metricNames.add(column + HLL_CONFIG.getHllDeriveColumnSuffix());
    }
    SegmentMetadata segmentMetadata = segment.getSegmentMetadata();
    LOGGER.info("[Schema] Dim: {} Metric: {}", schema.getDimensionNames(), schema.getMetricNames());
    for (int i = 0; i < _hardCodedQueries.length; i++) {
        Pql2Compiler compiler = new Pql2Compiler();
        BrokerRequest brokerRequest = compiler.compileToBrokerRequest(_hardCodedQueries[i]);
        FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
        Assert.assertTrue(RequestUtils.isFitForStarTreeIndex(segmentMetadata, filterQueryTree, brokerRequest));
        // Group -> Projected values of each group
        Map<String, long[]> expectedResult = computeHllUsingRawDocs(segment, metricNames, brokerRequest);
        Map<String, long[]> actualResult = computeHllUsingAggregatedDocs(segment, metricNames, brokerRequest);
        Assert.assertEquals(expectedResult.size(), actualResult.size(), "Mis-match in number of groups");
        for (Map.Entry<String, long[]> entry : expectedResult.entrySet()) {
            String expectedKey = entry.getKey();
            Assert.assertTrue(actualResult.containsKey(expectedKey));
            long[] expectedSums = entry.getValue();
            long[] actualSums = actualResult.get(expectedKey);
            for (int j = 0; j < expectedSums.length; j++) {
                LOGGER.info("actual hll: {} ", actualSums[j]);
                LOGGER.info("expected hll: {} ", expectedSums[j]);
                Assert.assertEquals(actualSums[j], expectedSums[j], "Mis-match hll for key '" + expectedKey + "', Metric: " + metricNames.get(j) + ", Random Seed: " + _randomSeed);
            }
        }
    }
}