Examples with FilterQueryTree com.linkedin.pinot.common.utils.request.FilterQueryTree used on opensource projects

Example 11 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class FilterTreeOptimizationTest method testQuery.

/**
   * Helper method to perform the actual testing for the given query.
   * <ul>
   *   <li> Constructs the operator tree with and without alwaysFalse optimizations. </li>
   *   <li> Compares that all docIds filtered by the two operators are identical. </li>
   * </ul>
   * @param query Query to run.
   */
private void testQuery(String query) {
    BrokerRequest brokerRequest = _compiler.compileToBrokerRequest(query);
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    BaseFilterOperator expectedOperator = FilterPlanNode.constructPhysicalOperator(filterQueryTree, _indexSegment, false);
    BaseFilterOperator actualOperator = FilterPlanNode.constructPhysicalOperator(filterQueryTree, _indexSegment, true);
    BaseFilterBlock expectedBlock;
    while ((expectedBlock = expectedOperator.getNextBlock()) != null) {
        BaseFilterBlock actualBlock = actualOperator.getNextBlock();
        Assert.assertNotNull(actualBlock);
        final BlockDocIdSet expectedDocIdSet = expectedBlock.getBlockDocIdSet();
        final BlockDocIdIterator expectedIterator = expectedDocIdSet.iterator();
        final BlockDocIdSet actualDocIdSet = actualBlock.getBlockDocIdSet();
        final BlockDocIdIterator actualIterator = actualDocIdSet.iterator();
        int expectedDocId;
        int actualDocId;
        while (((expectedDocId = expectedIterator.next()) != Constants.EOF) && ((actualDocId = actualIterator.next()) != Constants.EOF)) {
            Assert.assertEquals(actualDocId, expectedDocId);
        }
        Assert.assertTrue(expectedIterator.next() == Constants.EOF);
        Assert.assertTrue(actualIterator.next() == Constants.EOF);
    }
}

Example 12 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class InPredicateAstNode method buildFilterQueryTree.

@Override
public FilterQueryTree buildFilterQueryTree() {
    if (_identifier == null) {
        throw new Pql2CompilationException("IN predicate has no identifier");
    }
    TreeSet<String> values = new TreeSet<>();
    for (AstNode astNode : getChildren()) {
        if (astNode instanceof LiteralAstNode) {
            LiteralAstNode node = (LiteralAstNode) astNode;
            values.add(node.getValueAsString());
        }
    }
    String[] valueArray = values.toArray(new String[values.size()]);
    FilterOperator filterOperator;
    if (_isNotInClause) {
        filterOperator = FilterOperator.NOT_IN;
    } else {
        filterOperator = FilterOperator.IN;
    }
    return new FilterQueryTree(_identifier, Collections.singletonList(StringUtil.join("\t\t", valueArray)), filterOperator, null);
}

Example 13 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class ColumnValueSegmentPrunerTest method runPruner.

private boolean runPruner(String query) {
    BrokerRequest brokerRequest = COMPILER.compileToBrokerRequest(query);
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    return ColumnValueSegmentPruner.pruneSegment(filterQueryTree, COLUMN_METADATA_MAP);
}

Example 14 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class SegmentQueryProcessor method process.

public ResultTable process(BrokerRequest brokerRequest) throws Exception {
    if (pruneSegment(brokerRequest)) {
        return null;
    }
    LOGGER.debug("Processing segment: {}", _segmentName);
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    List<Integer> filteredDocIds = filterDocIds(filterQueryTree, null);
    ResultTable result = null;
    if (brokerRequest.isSetAggregationsInfo()) {
        // Aggregation only
        if (!brokerRequest.isSetGroupBy()) {
            Aggregation aggregation = new Aggregation(_indexSegment, _metadata, filteredDocIds, brokerRequest.getAggregationsInfo(), null, 10);
            result = aggregation.run();
        } else {
            // Aggregation GroupBy
            GroupBy groupBy = brokerRequest.getGroupBy();
            Aggregation aggregation = new Aggregation(_indexSegment, _metadata, filteredDocIds, brokerRequest.getAggregationsInfo(), groupBy.getColumns(), groupBy.getTopN());
            result = aggregation.run();
        }
    } else {
        // Only Selection
        if (brokerRequest.isSetSelections()) {
            List<String> columns = brokerRequest.getSelections().getSelectionColumns();
            if (columns.contains("*")) {
                columns = Arrays.asList(_indexSegment.getColumnNames());
            }
            List<Pair> selectionColumns = new ArrayList<>();
            Set<String> columSet = new HashSet<>();
            // Collect a unique list of columns, in case input has duplicates.
            for (String column : columns) {
                if (!columSet.contains(column)) {
                    selectionColumns.add(new Pair(column, null));
                    columSet.add(column);
                }
            }
            Selection selection = new Selection(_indexSegment, _metadata, filteredDocIds, selectionColumns);
            result = selection.run();
        }
    }
    result.setNumDocsScanned(filteredDocIds.size());
    result.setTotalDocs(_totalDocs);
    return result;
}

Example 15 with FilterQueryTree

use of com.linkedin.pinot.common.utils.request.FilterQueryTree in project pinot by linkedin.

the class SegmentQueryProcessor method filterDocIds.

private List<Integer> filterDocIds(FilterQueryTree filterQueryTree, List<Integer> inputDocIds) {
    // If no filter predicate, return the input without filtering.
    if (filterQueryTree == null) {
        List<Integer> allDocs = new ArrayList<>(_totalDocs);
        for (int i = 0; i < _totalDocs; ++i) {
            allDocs.add(i);
        }
        return allDocs;
    }
    final List<FilterQueryTree> childFilters = filterQueryTree.getChildren();
    final boolean isLeaf = (childFilters == null) || childFilters.isEmpty();
    if (isLeaf) {
        FilterOperator filterType = filterQueryTree.getOperator();
        String column = filterQueryTree.getColumn();
        final List<String> value = filterQueryTree.getValue();
        return getMatchingDocIds(inputDocIds, filterType, column, value);
    }
    List<Integer> result = filterDocIds(childFilters.get(0), inputDocIds);
    final FilterOperator operator = filterQueryTree.getOperator();
    for (int i = 1; i < childFilters.size(); ++i) {
        //      List<Integer> childResult = operator.equals(FilterOperator.AND) ? filterDocIds(childFilters.get(i), result)
        //          : filterDocIds(childFilters.get(i), inputDocIds);
        List<Integer> childResult = filterDocIds(childFilters.get(i), inputDocIds);
        result = combine(result, childResult, operator);
    }
    return result;
}