Search in sources :

Example 41 with AggregationInfo

use of com.linkedin.pinot.common.request.AggregationInfo in project pinot by linkedin.

the class BrokerRequestPreProcessor method rewriteFastHllColumnName.

/**
   * Rewrite 'fasthll' column name.
   *
   * @param indexSegments list of index segments.
   * @param aggregationsInfo list of aggregation info.
   */
private static void rewriteFastHllColumnName(List<IndexSegment> indexSegments, List<AggregationInfo> aggregationsInfo) {
    // Consistent check.
    for (AggregationInfo aggregationInfo : aggregationsInfo) {
        if (aggregationInfo.getAggregationType().equalsIgnoreCase("fasthll")) {
            String column = aggregationInfo.getAggregationParams().get("column").trim();
            boolean isFirstSegment = true;
            String firstSegmentName = null;
            String hllDerivedColumn = null;
            for (IndexSegment indexSegment : indexSegments) {
                SegmentMetadata segmentMetadata = indexSegment.getSegmentMetadata();
                if (isFirstSegment) {
                    // Use metadata from first index segment to perform rewrite.
                    isFirstSegment = false;
                    firstSegmentName = segmentMetadata.getName();
                    hllDerivedColumn = segmentMetadata.getDerivedColumn(column, MetricFieldSpec.DerivedMetricType.HLL);
                    if (hllDerivedColumn != null) {
                        aggregationInfo.getAggregationParams().put("column", hllDerivedColumn);
                    }
                } else {
                    // Perform consistency check on other index segments.
                    String hllDerivedColumnToCheck = segmentMetadata.getDerivedColumn(column, MetricFieldSpec.DerivedMetricType.HLL);
                    if (!Objects.equals(hllDerivedColumn, hllDerivedColumnToCheck)) {
                        throw new RuntimeException("Found inconsistency HLL derived column name. In segment " + firstSegmentName + ": " + hllDerivedColumn + "; In segment " + segmentMetadata.getName() + ": " + hllDerivedColumnToCheck);
                    }
                }
            }
        }
    }
}
Also used : SegmentMetadata(com.linkedin.pinot.common.segment.SegmentMetadata) IndexSegment(com.linkedin.pinot.core.indexsegment.IndexSegment) AggregationInfo(com.linkedin.pinot.common.request.AggregationInfo)

Example 42 with AggregationInfo

use of com.linkedin.pinot.common.request.AggregationInfo in project pinot by linkedin.

the class DefaultAggregationExecutorTest method testAggregation.

/**
   * Runs 'sum', 'min' & 'max' aggregation functions on the DefaultAggregationExecutor.
   * Asserts that the aggregation results returned by the executor are as expected.
   */
@Test
void testAggregation() {
    Map<String, BaseOperator> dataSourceMap = new HashMap<>();
    for (String column : _indexSegment.getColumnNames()) {
        dataSourceMap.put(column, _indexSegment.getDataSource(column));
    }
    int totalRawDocs = _indexSegment.getSegmentMetadata().getTotalRawDocs();
    MatchEntireSegmentOperator matchEntireSegmentOperator = new MatchEntireSegmentOperator(totalRawDocs);
    BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(matchEntireSegmentOperator, totalRawDocs, 10000);
    MProjectionOperator projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator);
    TransformExpressionOperator transformOperator = new TransformExpressionOperator(projectionOperator, Collections.<TransformExpressionTree>emptyList());
    TransformBlock transformBlock = (TransformBlock) transformOperator.nextBlock();
    int numAggFuncs = _aggregationInfoList.size();
    AggregationFunctionContext[] aggrFuncContextArray = new AggregationFunctionContext[numAggFuncs];
    AggregationFunctionInitializer aggFuncInitializer = new AggregationFunctionInitializer(_indexSegment.getSegmentMetadata());
    for (int i = 0; i < numAggFuncs; i++) {
        AggregationInfo aggregationInfo = _aggregationInfoList.get(i);
        aggrFuncContextArray[i] = AggregationFunctionContext.instantiate(aggregationInfo);
        aggrFuncContextArray[i].getAggregationFunction().accept(aggFuncInitializer);
    }
    AggregationExecutor aggregationExecutor = new DefaultAggregationExecutor(aggrFuncContextArray);
    aggregationExecutor.init();
    aggregationExecutor.aggregate(transformBlock);
    aggregationExecutor.finish();
    List<Object> result = aggregationExecutor.getResult();
    for (int i = 0; i < result.size(); i++) {
        double actual = (double) result.get(i);
        double expected = computeAggregation(AGGREGATION_FUNCTIONS[i], _inputData[i]);
        Assert.assertEquals(actual, expected, "Aggregation mis-match for function " + AGGREGATION_FUNCTIONS[i] + ", Expected: " + expected + " Actual: " + actual);
    }
}
Also used : BaseOperator(com.linkedin.pinot.core.operator.BaseOperator) TransformExpressionOperator(com.linkedin.pinot.core.operator.transform.TransformExpressionOperator) AggregationFunctionInitializer(com.linkedin.pinot.core.plan.AggregationFunctionInitializer) HashMap(java.util.HashMap) MProjectionOperator(com.linkedin.pinot.core.operator.MProjectionOperator) MatchEntireSegmentOperator(com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator) TransformBlock(com.linkedin.pinot.core.operator.blocks.TransformBlock) DefaultAggregationExecutor(com.linkedin.pinot.core.query.aggregation.DefaultAggregationExecutor) AggregationExecutor(com.linkedin.pinot.core.query.aggregation.AggregationExecutor) BReusableFilteredDocIdSetOperator(com.linkedin.pinot.core.operator.BReusableFilteredDocIdSetOperator) DefaultAggregationExecutor(com.linkedin.pinot.core.query.aggregation.DefaultAggregationExecutor) AggregationInfo(com.linkedin.pinot.common.request.AggregationInfo) AggregationFunctionContext(com.linkedin.pinot.core.query.aggregation.AggregationFunctionContext) Test(org.testng.annotations.Test)

Example 43 with AggregationInfo

use of com.linkedin.pinot.common.request.AggregationInfo in project pinot by linkedin.

the class DefaultAggregationExecutorTest method init.

/**
   * Initializations prior to the test:
   * - Build a segment with metric columns (that will be aggregated) containing
   *  randomly generated data.
   *
   * @throws Exception
   */
@BeforeSuite
void init() throws Exception {
    _random = new Random(System.currentTimeMillis());
    _docIdSet = new int[NUM_ROWS];
    int numColumns = AGGREGATION_FUNCTIONS.length;
    _inputData = new double[numColumns][NUM_ROWS];
    _columns = new String[numColumns];
    setupSegment();
    _aggregationInfoList = new ArrayList<>();
    for (int i = 0; i < _columns.length; i++) {
        AggregationInfo aggregationInfo = new AggregationInfo();
        aggregationInfo.setAggregationType(AGGREGATION_FUNCTIONS[i]);
        Map<String, String> params = new HashMap<String, String>();
        params.put("column", _columns[i]);
        aggregationInfo.setAggregationParams(params);
        _aggregationInfoList.add(aggregationInfo);
    }
}
Also used : Random(java.util.Random) HashMap(java.util.HashMap) AggregationInfo(com.linkedin.pinot.common.request.AggregationInfo) BeforeSuite(org.testng.annotations.BeforeSuite)

Example 44 with AggregationInfo

use of com.linkedin.pinot.common.request.AggregationInfo in project pinot by linkedin.

the class InstanceServerStarter method getMaxQuery.

private static BrokerRequest getMaxQuery() {
    BrokerRequest query = new BrokerRequest();
    AggregationInfo aggregationInfo = getMaxAggregationInfo();
    List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>();
    aggregationsInfo.add(aggregationInfo);
    query.setAggregationsInfo(aggregationsInfo);
    FilterQuery filterQuery = getFilterQuery();
    query.setFilterQuery(filterQuery);
    return query;
}
Also used : ArrayList(java.util.ArrayList) FilterQuery(com.linkedin.pinot.common.request.FilterQuery) BrokerRequest(com.linkedin.pinot.common.request.BrokerRequest) AggregationInfo(com.linkedin.pinot.common.request.AggregationInfo)

Example 45 with AggregationInfo

use of com.linkedin.pinot.common.request.AggregationInfo in project pinot by linkedin.

the class InstanceServerStarter method getMinQuery.

private static BrokerRequest getMinQuery() {
    BrokerRequest query = new BrokerRequest();
    AggregationInfo aggregationInfo = getMinAggregationInfo();
    List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>();
    aggregationsInfo.add(aggregationInfo);
    query.setAggregationsInfo(aggregationsInfo);
    FilterQuery filterQuery = getFilterQuery();
    query.setFilterQuery(filterQuery);
    return query;
}
Also used : ArrayList(java.util.ArrayList) FilterQuery(com.linkedin.pinot.common.request.FilterQuery) BrokerRequest(com.linkedin.pinot.common.request.BrokerRequest) AggregationInfo(com.linkedin.pinot.common.request.AggregationInfo)

Aggregations

AggregationInfo (com.linkedin.pinot.common.request.AggregationInfo)51 BrokerRequest (com.linkedin.pinot.common.request.BrokerRequest)22 ArrayList (java.util.ArrayList)21 HashMap (java.util.HashMap)21 FilterQuery (com.linkedin.pinot.common.request.FilterQuery)9 GroupBy (com.linkedin.pinot.common.request.GroupBy)6 AggregationFunctionContext (com.linkedin.pinot.core.query.aggregation.AggregationFunctionContext)5 AggregationFunctionInitializer (com.linkedin.pinot.core.plan.AggregationFunctionInitializer)3 Selection (com.linkedin.pinot.common.request.Selection)2 Operator (com.linkedin.pinot.core.common.Operator)2 BReusableFilteredDocIdSetOperator (com.linkedin.pinot.core.operator.BReusableFilteredDocIdSetOperator)2 BaseOperator (com.linkedin.pinot.core.operator.BaseOperator)2 MProjectionOperator (com.linkedin.pinot.core.operator.MProjectionOperator)2 IntermediateResultsBlock (com.linkedin.pinot.core.operator.blocks.IntermediateResultsBlock)2 MatchEntireSegmentOperator (com.linkedin.pinot.core.operator.filter.MatchEntireSegmentOperator)2 TransformExpressionOperator (com.linkedin.pinot.core.operator.transform.TransformExpressionOperator)2 Pql2Compiler (com.linkedin.pinot.pql.parsers.Pql2Compiler)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 Test (org.testng.annotations.Test)2