Examples with GroupBy com.linkedin.pinot.common.request.GroupBy used on opensource projects

Example 6 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class Pql2CompilerTest method testTopZeroFor.

private void testTopZeroFor(Pql2Compiler compiler, String s, final int expectedTopN, boolean parseException) throws Exception {
    BrokerRequest req;
    try {
        req = compiler.compileToBrokerRequest(s);
    } catch (Pql2CompilationException e) {
        if (parseException) {
            return;
        }
        throw e;
    }
    Assert.assertTrue(req.isSetGroupBy());
    GroupBy groupBy = req.getGroupBy();
    Assert.assertTrue(groupBy.isSetTopN());
    Assert.assertEquals(expectedTopN, groupBy.getTopN());
}

Example 7 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class ScanBasedQueryProcessor method processQuery.

public QueryResponse processQuery(String query) throws Exception {
    long startTimeInMillis = System.currentTimeMillis();
    Pql2Compiler pql2Compiler = new Pql2Compiler();
    BrokerRequest brokerRequest = pql2Compiler.compileToBrokerRequest(query);
    ResultTable results = null;
    Aggregation aggregation = null;
    List<String> groupByColumns;
    List<AggregationInfo> aggregationsInfo = brokerRequest.getAggregationsInfo();
    if (aggregationsInfo != null) {
        GroupBy groupBy = brokerRequest.getGroupBy();
        groupByColumns = (brokerRequest.isSetGroupBy()) ? groupBy.getColumns() : null;
        long topN = (groupByColumns != null) ? groupBy.getTopN() : 10;
        aggregation = new Aggregation(brokerRequest.getAggregationsInfo(), groupByColumns, topN);
    }
    int numDocsScanned = 0;
    int totalDocs = 0;
    int numSegments = 0;
    LOGGER.info("Processing Query: {}", query);
    List<ResultTable> resultTables = processSegments(query, brokerRequest);
    for (ResultTable segmentResults : resultTables) {
        numDocsScanned += segmentResults.getNumDocsScanned();
        totalDocs += segmentResults.getTotalDocs();
        ++numSegments;
        results = (results == null) ? segmentResults : results.append(segmentResults);
    }
    if (aggregation != null && numSegments > 1 && numDocsScanned > 0) {
        results = aggregation.aggregate(results);
    }
    if (results != null) {
        results.setNumDocsScanned(numDocsScanned);
        results.setTotalDocs(totalDocs);
        long totalUsedMs = System.currentTimeMillis() - startTimeInMillis;
        results.setProcessingTime(totalUsedMs);
        results.seal();
    }
    QueryResponse queryResponse = new QueryResponse(results);
    return queryResponse;
}

Example 8 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class SegmentQueryProcessor method process.

public ResultTable process(BrokerRequest brokerRequest) throws Exception {
    if (pruneSegment(brokerRequest)) {
        return null;
    }
    LOGGER.debug("Processing segment: {}", _segmentName);
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    List<Integer> filteredDocIds = filterDocIds(filterQueryTree, null);
    ResultTable result = null;
    if (brokerRequest.isSetAggregationsInfo()) {
        // Aggregation only
        if (!brokerRequest.isSetGroupBy()) {
            Aggregation aggregation = new Aggregation(_indexSegment, _metadata, filteredDocIds, brokerRequest.getAggregationsInfo(), null, 10);
            result = aggregation.run();
        } else {
            // Aggregation GroupBy
            GroupBy groupBy = brokerRequest.getGroupBy();
            Aggregation aggregation = new Aggregation(_indexSegment, _metadata, filteredDocIds, brokerRequest.getAggregationsInfo(), groupBy.getColumns(), groupBy.getTopN());
            result = aggregation.run();
        }
    } else {
        // Only Selection
        if (brokerRequest.isSetSelections()) {
            List<String> columns = brokerRequest.getSelections().getSelectionColumns();
            if (columns.contains("*")) {
                columns = Arrays.asList(_indexSegment.getColumnNames());
            }
            List<Pair> selectionColumns = new ArrayList<>();
            Set<String> columSet = new HashSet<>();
            // Collect a unique list of columns, in case input has duplicates.
            for (String column : columns) {
                if (!columSet.contains(column)) {
                    selectionColumns.add(new Pair(column, null));
                    columSet.add(column);
                }
            }
            Selection selection = new Selection(_indexSegment, _metadata, filteredDocIds, selectionColumns);
            result = selection.run();
        }
    }
    result.setNumDocsScanned(filteredDocIds.size());
    result.setTotalDocs(_totalDocs);
    return result;
}

Example 9 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class TransformGroupByTest method executeGroupByQuery.

/**
   * Helper method that executes the group by query on the index and returns the group by result.
   *
   * @param query Query to execute
   * @return Group by result
   */
private AggregationGroupByResult executeGroupByQuery(IndexSegment indexSegment, String query) {
    Operator filterOperator = new MatchEntireSegmentOperator(indexSegment.getSegmentMetadata().getTotalDocs());
    final BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(filterOperator, indexSegment.getSegmentMetadata().getTotalDocs(), NUM_ROWS);
    final Map<String, BaseOperator> dataSourceMap = buildDataSourceMap(indexSegment.getSegmentMetadata().getSchema());
    final MProjectionOperator projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator);
    Pql2Compiler compiler = new Pql2Compiler();
    BrokerRequest brokerRequest = compiler.compileToBrokerRequest(query);
    List<AggregationInfo> aggregationsInfo = brokerRequest.getAggregationsInfo();
    int numAggFunctions = aggregationsInfo.size();
    AggregationFunctionContext[] aggrFuncContextArray = new AggregationFunctionContext[numAggFunctions];
    AggregationFunctionInitializer aggFuncInitializer = new AggregationFunctionInitializer(indexSegment.getSegmentMetadata());
    for (int i = 0; i < numAggFunctions; i++) {
        AggregationInfo aggregationInfo = aggregationsInfo.get(i);
        aggrFuncContextArray[i] = AggregationFunctionContext.instantiate(aggregationInfo);
        aggrFuncContextArray[i].getAggregationFunction().accept(aggFuncInitializer);
    }
    GroupBy groupBy = brokerRequest.getGroupBy();
    Set<String> expressions = new HashSet<>(groupBy.getExpressions());
    TransformExpressionOperator transformOperator = new TransformExpressionOperator(projectionOperator, TransformPlanNode.buildTransformExpressionTrees(expressions));
    AggregationGroupByOperator groupByOperator = new AggregationGroupByOperator(aggrFuncContextArray, groupBy, Integer.MAX_VALUE, transformOperator, NUM_ROWS);
    IntermediateResultsBlock block = (IntermediateResultsBlock) groupByOperator.nextBlock();
    return block.getAggregationGroupByResult();
}

Example 10 with GroupBy

use of com.linkedin.pinot.common.request.GroupBy in project pinot by linkedin.

the class GroupByAstNode method updateBrokerRequest.

@Override
public void updateBrokerRequest(BrokerRequest brokerRequest) {
    GroupBy groupBy = new GroupBy();
    for (AstNode astNode : getChildren()) {
        if (astNode instanceof IdentifierAstNode) {
            IdentifierAstNode node = (IdentifierAstNode) astNode;
            String groupByColumnName = node.getName();
            groupBy.addToColumns(groupByColumnName);
            // List of expression contains columns as well as expressions to maintain ordering of group by columns.
            groupBy.addToExpressions(groupByColumnName);
        } else {
            FunctionCallAstNode functionCallAstNode = (FunctionCallAstNode) astNode;
            // Remove all white-space until we start compiling expressions on broker side.
            groupBy.addToExpressions(functionCallAstNode.getExpression().replaceAll("\\s", ""));
        }
    }
    brokerRequest.setGroupBy(groupBy);
}