Examples with MProjectionOperator com.linkedin.pinot.core.operator.MProjectionOperator used on opensource projects

Example 1 with MProjectionOperator

use of com.linkedin.pinot.core.operator.MProjectionOperator in project pinot by linkedin.

the class NoDictionaryGroupKeyGeneratorTest method testGroupKeyGenerator.

private void testGroupKeyGenerator(String[] groupByColumns, FieldSpec.DataType[] dataTypes) throws Exception {
    // Build the projection operator.
    MatchEntireSegmentOperator matchEntireSegmentOperator = new MatchEntireSegmentOperator(NUM_ROWS);
    BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(matchEntireSegmentOperator, NUM_ROWS, 10000);
    MProjectionOperator projectionOperator = new MProjectionOperator(_dataSourceMap, docIdSetOperator);
    TransformExpressionOperator transformOperator = new TransformExpressionOperator(projectionOperator, new ArrayList<TransformExpressionTree>());
    // Iterator over all projection blocks and generate group keys.
    TransformBlock transformBlock;
    int[] docIdToGroupKeys = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];
    GroupKeyGenerator groupKeyGenerator = null;
    while ((transformBlock = (TransformBlock) transformOperator.nextBlock()) != null) {
        if (groupKeyGenerator == null) {
            // Build the group key generator.
            groupKeyGenerator = (groupByColumns.length == 1) ? new NoDictionarySingleColumnGroupKeyGenerator(groupByColumns[0], dataTypes[0]) : new NoDictionaryMultiColumnGroupKeyGenerator(transformBlock, groupByColumns);
        }
        groupKeyGenerator.generateKeysForBlock(transformBlock, docIdToGroupKeys);
    }
    // Assert total number of group keys is as expected
    Assert.assertTrue(groupKeyGenerator != null);
    Set<String> expectedGroupKeys = getExpectedGroupKeys(_recordReader, groupByColumns);
    Assert.assertEquals(groupKeyGenerator.getCurrentGroupKeyUpperBound(), expectedGroupKeys.size(), "Number of group keys mis-match.");
    // Assert all group key values are as expected
    Iterator<GroupKeyGenerator.GroupKey> uniqueGroupKeys = groupKeyGenerator.getUniqueGroupKeys();
    while (uniqueGroupKeys.hasNext()) {
        GroupKeyGenerator.GroupKey groupKey = uniqueGroupKeys.next();
        String actual = groupKey.getStringKey();
        Assert.assertTrue(expectedGroupKeys.contains(actual), "Unexpected group key: " + actual);
    }
}

Example 2 with MProjectionOperator

use of com.linkedin.pinot.core.operator.MProjectionOperator in project pinot by linkedin.

the class ProjectionPlanNode method run.

@Override
public Operator run() {
    long start = System.currentTimeMillis();
    if (_projectionOperator == null) {
        Map<String, BaseOperator> dataSourceMap = new HashMap<String, BaseOperator>();
        BReusableFilteredDocIdSetOperator docIdSetOperator = (BReusableFilteredDocIdSetOperator) _docIdSetPlanNode.run();
        for (String column : _dataSourcePlanNodeMap.keySet()) {
            ColumnarDataSourcePlanNode columnarDataSourcePlanNode = _dataSourcePlanNodeMap.get(column);
            BaseOperator operator = columnarDataSourcePlanNode.run();
            dataSourceMap.put(column, operator);
        }
        _projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator);
    }
    long end = System.currentTimeMillis();
    LOGGER.debug("Time take in ProjectionPlanNode: " + (end - start));
    return _projectionOperator;
}

Example 3 with MProjectionOperator

use of com.linkedin.pinot.core.operator.MProjectionOperator in project pinot by linkedin.

the class DefaultAggregationExecutorTest method testAggregation.

/**
   * Runs 'sum', 'min' & 'max' aggregation functions on the DefaultAggregationExecutor.
   * Asserts that the aggregation results returned by the executor are as expected.
   */
@Test
void testAggregation() {
    Map<String, BaseOperator> dataSourceMap = new HashMap<>();
    for (String column : _indexSegment.getColumnNames()) {
        dataSourceMap.put(column, _indexSegment.getDataSource(column));
    }
    int totalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
    MatchEntireSegmentOperator matchEntireSegmentOperator = new MatchEntireSegmentOperator(totalRawDocs);
    BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(matchEntireSegmentOperator, totalRawDocs, 10000);
    MProjectionOperator projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator);
    TransformExpressionOperator transformOperator = new TransformExpressionOperator(projectionOperator, Collections.<TransformExpressionTree>emptyList());
    TransformBlock transformBlock = (TransformBlock) transformOperator.nextBlock();
    int numAggFuncs = _aggregationInfoList.size();
    AggregationFunctionContext[] aggrFuncContextArray = new AggregationFunctionContext[numAggFuncs];
    AggregationFunctionInitializer aggFuncInitializer = new AggregationFunctionInitializer(_indexSegment.getSegmentMetadata());
    for (int i = 0; i < numAggFuncs; i++) {
        AggregationInfo aggregationInfo = _aggregationInfoList.get(i);
        aggrFuncContextArray[i] = AggregationFunctionContext.instantiate(aggregationInfo);
        aggrFuncContextArray[i].getAggregationFunction().accept(aggFuncInitializer);
    }
    AggregationExecutor aggregationExecutor = new DefaultAggregationExecutor(aggrFuncContextArray);
    aggregationExecutor.init();
    aggregationExecutor.aggregate(transformBlock);
    aggregationExecutor.finish();
    List<Object> result = aggregationExecutor.getResult();
    for (int i = 0; i < result.size(); i++) {
        double actual = (double) result.get(i);
        double expected = computeAggregation(AGGREGATION_FUNCTIONS[i], _inputData[i]);
        Assert.assertEquals(actual, expected, "Aggregation mis-match for function " + AGGREGATION_FUNCTIONS[i] + ", Expected: " + expected + " Actual: " + actual);
    }
}

Example 4 with MProjectionOperator

use of com.linkedin.pinot.core.operator.MProjectionOperator in project pinot by linkedin.

the class RawIndexBenchmark method profileLookups.

/**
   * Profiles the lookup time for a given column, for the given docIds.
   *
   * @param segment Segment to profile
   * @param column Column to profile
   * @param docIds DocIds to lookup on the column
   * @return Time take in millis for the lookups
   */
private long profileLookups(IndexSegment segment, String column, int[] docIds) {
    BaseFilterOperator filterOperator = new TestFilterOperator(docIds);
    BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(filterOperator, docIds.length, DocIdSetPlanNode.MAX_DOC_PER_CALL);
    ProjectionBlock projectionBlock;
    MProjectionOperator projectionOperator = new MProjectionOperator(buildDataSourceMap(segment), docIdSetOperator);
    long start = System.currentTimeMillis();
    while ((projectionBlock = (ProjectionBlock) projectionOperator.nextBlock()) != null) {
        ProjectionBlockValSet blockValueSet = (ProjectionBlockValSet) projectionBlock.getBlockValueSet(column);
        blockValueSet.getDoubleValuesSV();
    }
    return (System.currentTimeMillis() - start);
}

Example 5 with MProjectionOperator

use of com.linkedin.pinot.core.operator.MProjectionOperator in project pinot by linkedin.

the class TransformGroupByTest method executeGroupByQuery.

/**
   * Helper method that executes the group by query on the index and returns the group by result.
   *
   * @param query Query to execute
   * @return Group by result
   */
private AggregationGroupByResult executeGroupByQuery(IndexSegment indexSegment, String query) {
    Operator filterOperator = new MatchEntireSegmentOperator(indexSegment.getSegmentMetadata().getTotalDocs());
    final BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(filterOperator, indexSegment.getSegmentMetadata().getTotalDocs(), NUM_ROWS);
    final Map<String, BaseOperator> dataSourceMap = buildDataSourceMap(indexSegment.getSegmentMetadata().getSchema());
    final MProjectionOperator projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator);
    Pql2Compiler compiler = new Pql2Compiler();
    BrokerRequest brokerRequest = compiler.compileToBrokerRequest(query);
    List<AggregationInfo> aggregationsInfo = brokerRequest.getAggregationsInfo();
    int numAggFunctions = aggregationsInfo.size();
    AggregationFunctionContext[] aggrFuncContextArray = new AggregationFunctionContext[numAggFunctions];
    AggregationFunctionInitializer aggFuncInitializer = new AggregationFunctionInitializer(indexSegment.getSegmentMetadata());
    for (int i = 0; i < numAggFunctions; i++) {
        AggregationInfo aggregationInfo = aggregationsInfo.get(i);
        aggrFuncContextArray[i] = AggregationFunctionContext.instantiate(aggregationInfo);
        aggrFuncContextArray[i].getAggregationFunction().accept(aggFuncInitializer);
    }
    GroupBy groupBy = brokerRequest.getGroupBy();
    Set<String> expressions = new HashSet<>(groupBy.getExpressions());
    TransformExpressionOperator transformOperator = new TransformExpressionOperator(projectionOperator, TransformPlanNode.buildTransformExpressionTrees(expressions));
    AggregationGroupByOperator groupByOperator = new AggregationGroupByOperator(aggrFuncContextArray, groupBy, Integer.MAX_VALUE, transformOperator, NUM_ROWS);
    IntermediateResultsBlock block = (IntermediateResultsBlock) groupByOperator.nextBlock();
    return block.getAggregationGroupByResult();
}