Search in sources :

Example 1 with AggregationExecutor

use of com.linkedin.pinot.core.query.aggregation.AggregationExecutor in project pinot by linkedin.

the class AggregationOperator method getNextBlock.

@Override
public Block getNextBlock() {
    int numDocsScanned = 0;
    // Perform aggregation on all the blocks.
    AggregationExecutor aggregationExecutor = new DefaultAggregationExecutor(_aggregationFunctionContexts);
    aggregationExecutor.init();
    TransformBlock transformBlock;
    while ((transformBlock = (TransformBlock) _transformOperator.nextBlock()) != null) {
        numDocsScanned += transformBlock.getNumDocs();
        aggregationExecutor.aggregate(transformBlock);
    }
    aggregationExecutor.finish();
    // Create execution statistics.
    long numEntriesScannedInFilter = _transformOperator.getExecutionStatistics().getNumEntriesScannedInFilter();
    long numEntriesScannedPostFilter = numDocsScanned * _transformOperator.getNumProjectionColumns();
    _executionStatistics = new ExecutionStatistics(numDocsScanned, numEntriesScannedInFilter, numEntriesScannedPostFilter, _numTotalRawDocs);
    // Build intermediate result block based on aggregation result from the executor.
    return new IntermediateResultsBlock(_aggregationFunctionContexts, aggregationExecutor.getResult(), false);
}
Also used : ExecutionStatistics(com.linkedin.pinot.core.operator.ExecutionStatistics) TransformBlock(com.linkedin.pinot.core.operator.blocks.TransformBlock) DefaultAggregationExecutor(com.linkedin.pinot.core.query.aggregation.DefaultAggregationExecutor) AggregationExecutor(com.linkedin.pinot.core.query.aggregation.AggregationExecutor) DefaultAggregationExecutor(com.linkedin.pinot.core.query.aggregation.DefaultAggregationExecutor) IntermediateResultsBlock(com.linkedin.pinot.core.operator.blocks.IntermediateResultsBlock)

Example 2 with AggregationExecutor

use of com.linkedin.pinot.core.query.aggregation.AggregationExecutor in project pinot by linkedin.

the class DefaultAggregationExecutorTest method testAggregation.

/**
   * Runs 'sum', 'min' & 'max' aggregation functions on the DefaultAggregationExecutor.
   * Asserts that the aggregation results returned by the executor are as expected.
   */
@Test
void testAggregation() {
    Map<String, BaseOperator> dataSourceMap = new HashMap<>();
    for (String column : _indexSegment.getColumnNames()) {
        dataSourceMap.put(column, _indexSegment.getDataSource(column));
    }
    int totalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
    MatchEntireSegmentOperator matchEntireSegmentOperator = new MatchEntireSegmentOperator(totalRawDocs);
    BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(matchEntireSegmentOperator, totalRawDocs, 10000);
    MProjectionOperator projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator);
    TransformExpressionOperator transformOperator = new TransformExpressionOperator(projectionOperator, Collections.<TransformExpressionTree>emptyList());
    TransformBlock transformBlock = (TransformBlock) transformOperator.nextBlock();
    int numAggFuncs = _aggregationInfoList.size();
    AggregationFunctionContext[] aggrFuncContextArray = new AggregationFunctionContext[numAggFuncs];
    AggregationFunctionInitializer aggFuncInitializer = new AggregationFunctionInitializer(_indexSegment.getSegmentMetadata());
    for (int i = 0; i < numAggFuncs; i++) {
        AggregationInfo aggregationInfo = _aggregationInfoList.get(i);
        aggrFuncContextArray[i] = AggregationFunctionContext.instantiate(aggregationInfo);
        aggrFuncContextArray[i].getAggregationFunction().accept(aggFuncInitializer);
    }
    AggregationExecutor aggregationExecutor = new DefaultAggregationExecutor(aggrFuncContextArray);
    aggregationExecutor.init();
    aggregationExecutor.aggregate(transformBlock);
    aggregationExecutor.finish();
    List<Object> result = aggregationExecutor.getResult();
    for (int i = 0; i < result.size(); i++) {
        double actual = (double) result.get(i);
        double expected = computeAggregation(AGGREGATION_FUNCTIONS[i], _inputData[i]);
        Assert.assertEquals(actual, expected, "Aggregation mis-match for function " + AGGREGATION_FUNCTIONS[i] + ", Expected: " + expected + " Actual: " + actual);
    }
}
Also used : BaseOperator(com.linkedin.pinot.core.operator.BaseOperator) TransformExpressionOperator(com.linkedin.pinot.core.operator.transform.TransformExpressionOperator) AggregationFunctionInitializer(com.linkedin.pinot.core.plan.AggregationFunctionInitializer) HashMap(java.util.HashMap) MProjectionOperator(com.linkedin.pinot.core.operator.MProjectionOperator) MatchEntireSegmentOperator(com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator) TransformBlock(com.linkedin.pinot.core.operator.blocks.TransformBlock) DefaultAggregationExecutor(com.linkedin.pinot.core.query.aggregation.DefaultAggregationExecutor) AggregationExecutor(com.linkedin.pinot.core.query.aggregation.AggregationExecutor) BReusableFilteredDocIdSetOperator(com.linkedin.pinot.core.operator.BReusableFilteredDocIdSetOperator) DefaultAggregationExecutor(com.linkedin.pinot.core.query.aggregation.DefaultAggregationExecutor) AggregationInfo(com.linkedin.pinot.common.request.AggregationInfo) AggregationFunctionContext(com.linkedin.pinot.core.query.aggregation.AggregationFunctionContext) Test(org.testng.annotations.Test)

Aggregations

TransformBlock (com.linkedin.pinot.core.operator.blocks.TransformBlock)2 AggregationExecutor (com.linkedin.pinot.core.query.aggregation.AggregationExecutor)2 DefaultAggregationExecutor (com.linkedin.pinot.core.query.aggregation.DefaultAggregationExecutor)2 AggregationInfo (com.linkedin.pinot.common.request.AggregationInfo)1 BReusableFilteredDocIdSetOperator (com.linkedin.pinot.core.operator.BReusableFilteredDocIdSetOperator)1 BaseOperator (com.linkedin.pinot.core.operator.BaseOperator)1 ExecutionStatistics (com.linkedin.pinot.core.operator.ExecutionStatistics)1 MProjectionOperator (com.linkedin.pinot.core.operator.MProjectionOperator)1 IntermediateResultsBlock (com.linkedin.pinot.core.operator.blocks.IntermediateResultsBlock)1 MatchEntireSegmentOperator (com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator)1 TransformExpressionOperator (com.linkedin.pinot.core.operator.transform.TransformExpressionOperator)1 AggregationFunctionInitializer (com.linkedin.pinot.core.plan.AggregationFunctionInitializer)1 AggregationFunctionContext (com.linkedin.pinot.core.query.aggregation.AggregationFunctionContext)1 HashMap (java.util.HashMap)1 Test (org.testng.annotations.Test)1