Examples with AggregationResult com.linkedin.pinot.common.response.broker.AggregationResult used on opensource projects

Example 1 with AggregationResult

use of com.linkedin.pinot.common.response.broker.AggregationResult in project pinot by linkedin.

the class BrokerReduceService method setGroupByResults.

/**
   * Reduce group-by results from multiple servers and set them into BrokerResponseNative passed in.
   *
   * @param brokerResponseNative broker response.
   * @param aggregationFunctions array of aggregation functions.
   * @param groupBy group-by information.
   * @param dataTableMap map from server to data table.
   */
@SuppressWarnings("unchecked")
private void setGroupByResults(@Nonnull BrokerResponseNative brokerResponseNative, @Nonnull AggregationFunction[] aggregationFunctions, @Nonnull GroupBy groupBy, @Nonnull Map<ServerInstance, DataTable> dataTableMap) {
    int numAggregationFunctions = aggregationFunctions.length;
    // Merge results from all data tables.
    String[] columnNames = new String[numAggregationFunctions];
    Map<String, Object>[] intermediateResultMaps = new Map[numAggregationFunctions];
    for (DataTable dataTable : dataTableMap.values()) {
        for (int i = 0; i < numAggregationFunctions; i++) {
            if (columnNames[i] == null) {
                columnNames[i] = dataTable.getString(i, 0);
                intermediateResultMaps[i] = dataTable.getObject(i, 1);
            } else {
                Map<String, Object> mergedIntermediateResultMap = intermediateResultMaps[i];
                Map<String, Object> intermediateResultMapToMerge = dataTable.getObject(i, 1);
                for (Map.Entry<String, Object> entry : intermediateResultMapToMerge.entrySet()) {
                    String groupKey = entry.getKey();
                    Object intermediateResultToMerge = entry.getValue();
                    if (mergedIntermediateResultMap.containsKey(groupKey)) {
                        Object mergedIntermediateResult = mergedIntermediateResultMap.get(groupKey);
                        mergedIntermediateResultMap.put(groupKey, aggregationFunctions[i].merge(mergedIntermediateResult, intermediateResultToMerge));
                    } else {
                        mergedIntermediateResultMap.put(groupKey, intermediateResultToMerge);
                    }
                }
            }
        }
    }
    // Extract final result maps from the merged intermediate result maps.
    Map<String, Comparable>[] finalResultMaps = new Map[numAggregationFunctions];
    for (int i = 0; i < numAggregationFunctions; i++) {
        Map<String, Object> intermediateResultMap = intermediateResultMaps[i];
        Map<String, Comparable> finalResultMap = new HashMap<>();
        for (String groupKey : intermediateResultMap.keySet()) {
            Object intermediateResult = intermediateResultMap.get(groupKey);
            finalResultMap.put(groupKey, aggregationFunctions[i].extractFinalResult(intermediateResult));
        }
        finalResultMaps[i] = finalResultMap;
    }
    // Trim the final result maps to topN and set them into the broker response.
    AggregationGroupByTrimmingService aggregationGroupByTrimmingService = new AggregationGroupByTrimmingService(aggregationFunctions, (int) groupBy.getTopN());
    List<GroupByResult>[] groupByResultLists = aggregationGroupByTrimmingService.trimFinalResults(finalResultMaps);
    List<AggregationResult> aggregationResults = new ArrayList<>(numAggregationFunctions);
    for (int i = 0; i < numAggregationFunctions; i++) {
        List<GroupByResult> groupByResultList = groupByResultLists[i];
        List<String> groupByColumns = groupBy.getExpressions();
        if (groupByColumns == null) {
            groupByColumns = groupBy.getColumns();
        }
        aggregationResults.add(new AggregationResult(groupByResultList, groupByColumns, columnNames[i]));
    }
    brokerResponseNative.setAggregationResults(aggregationResults);
}

Example 2 with AggregationResult

use of com.linkedin.pinot.common.response.broker.AggregationResult in project pinot by linkedin.

the class BrokerReduceServiceTest method testDistinctCountQuery1.

@Test
public void testDistinctCountQuery1() {
    BrokerRequest brokerRequest = getDistinctCountQuery("dim1");
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.addToSearchSegments(segment.getSegmentName());
    }
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    try {
        QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
        DataTable instanceResponse1 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse1);
        DataTable instanceResponse2 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse2);
        BrokerResponseNative brokerResponse = _reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
        AggregationResult aggregationResult = brokerResponse.getAggregationResults().get(0);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "distinctCount_dim1", 100.0);
        LOGGER.info("Time used for BrokerResponse is " + brokerResponse.getTimeUsedMs());
    } catch (Exception e) {
        e.printStackTrace();
        // Should never happen
        throw new RuntimeException(e.toString(), e);
    }
}

Example 3 with AggregationResult

use of com.linkedin.pinot.common.response.broker.AggregationResult in project pinot by linkedin.

the class BrokerReduceServiceTest method testAvgQuery.

@Test
public void testAvgQuery() {
    BrokerRequest brokerRequest = getAvgQuery();
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.addToSearchSegments(segment.getSegmentName());
    }
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    try {
        QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
        DataTable instanceResponse1 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse1);
        DataTable instanceResponse2 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse2);
        BrokerResponseNative brokerResponse = _reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
        AggregationResult aggregationResult = brokerResponse.getAggregationResults().get(0);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "avg_met", 100000.0);
        LOGGER.info("Time used for BrokerResponse is " + brokerResponse.getTimeUsedMs());
    } catch (Exception e) {
        e.printStackTrace();
        // Should never happen
        throw new RuntimeException(e.toString(), e);
    }
}

Example 4 with AggregationResult

use of com.linkedin.pinot.common.response.broker.AggregationResult in project pinot by linkedin.

the class BrokerReduceServiceTest method testCountQuery.

@Test
public void testCountQuery() {
    BrokerRequest brokerRequest = getCountQuery();
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.addToSearchSegments(segment.getSegmentName());
    }
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
    DataTable instanceResponse1 = _queryExecutor.processQuery(queryRequest, queryRunners);
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse1);
    DataTable instanceResponse2 = _queryExecutor.processQuery(queryRequest, queryRunners);
    instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse2);
    BrokerResponseNative brokerResponse = _reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
    AggregationResult aggregationResult = brokerResponse.getAggregationResults().get(0);
    LOGGER.info("BrokerResponse is " + aggregationResult);
    checkAggregationResult(aggregationResult, "count_star", 800004.0);
    LOGGER.info("Time used for BrokerResponse is " + brokerResponse.getTimeUsedMs());
}

Example 5 with AggregationResult

use of com.linkedin.pinot.common.response.broker.AggregationResult in project pinot by linkedin.

the class BrokerReduceServiceTest method testDistinctCountQuery0.

@Test
public void testDistinctCountQuery0() {
    BrokerRequest brokerRequest = getDistinctCountQuery("dim0");
    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
        instanceRequest.addToSearchSegments(segment.getSegmentName());
    }
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    try {
        QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
        DataTable instanceResponse1 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse1);
        DataTable instanceResponse2 = _queryExecutor.processQuery(queryRequest, queryRunners);
        instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse2);
        BrokerResponseNative brokerResponse = _reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
        AggregationResult aggregationResult = brokerResponse.getAggregationResults().get(0);
        LOGGER.info("BrokerResponse is " + aggregationResult);
        checkAggregationResult(aggregationResult, "distinctCount_dim0", 10.0);
        LOGGER.info("Time used for BrokerResponse is " + brokerResponse.getTimeUsedMs());
    } catch (Exception e) {
        e.printStackTrace();
        // Should never happen
        throw new RuntimeException(e.toString(), e);
    }
}