Examples with GroupByResult com.linkedin.pinot.common.response.broker.GroupByResult used on opensource projects

Example 1 with GroupByResult

use of com.linkedin.pinot.common.response.broker.GroupByResult in project pinot by linkedin.

the class BrokerReduceService method setGroupByResults.

/**
   * Reduce group-by results from multiple servers and set them into BrokerResponseNative passed in.
   *
   * @param brokerResponseNative broker response.
   * @param aggregationFunctions array of aggregation functions.
   * @param groupBy group-by information.
   * @param dataTableMap map from server to data table.
   */
@SuppressWarnings("unchecked")
private void setGroupByResults(@Nonnull BrokerResponseNative brokerResponseNative, @Nonnull AggregationFunction[] aggregationFunctions, @Nonnull GroupBy groupBy, @Nonnull Map<ServerInstance, DataTable> dataTableMap) {
    int numAggregationFunctions = aggregationFunctions.length;
    // Merge results from all data tables.
    String[] columnNames = new String[numAggregationFunctions];
    Map<String, Object>[] intermediateResultMaps = new Map[numAggregationFunctions];
    for (DataTable dataTable : dataTableMap.values()) {
        for (int i = 0; i < numAggregationFunctions; i++) {
            if (columnNames[i] == null) {
                columnNames[i] = dataTable.getString(i, 0);
                intermediateResultMaps[i] = dataTable.getObject(i, 1);
            } else {
                Map<String, Object> mergedIntermediateResultMap = intermediateResultMaps[i];
                Map<String, Object> intermediateResultMapToMerge = dataTable.getObject(i, 1);
                for (Map.Entry<String, Object> entry : intermediateResultMapToMerge.entrySet()) {
                    String groupKey = entry.getKey();
                    Object intermediateResultToMerge = entry.getValue();
                    if (mergedIntermediateResultMap.containsKey(groupKey)) {
                        Object mergedIntermediateResult = mergedIntermediateResultMap.get(groupKey);
                        mergedIntermediateResultMap.put(groupKey, aggregationFunctions[i].merge(mergedIntermediateResult, intermediateResultToMerge));
                    } else {
                        mergedIntermediateResultMap.put(groupKey, intermediateResultToMerge);
                    }
                }
            }
        }
    }
    // Extract final result maps from the merged intermediate result maps.
    Map<String, Comparable>[] finalResultMaps = new Map[numAggregationFunctions];
    for (int i = 0; i < numAggregationFunctions; i++) {
        Map<String, Object> intermediateResultMap = intermediateResultMaps[i];
        Map<String, Comparable> finalResultMap = new HashMap<>();
        for (String groupKey : intermediateResultMap.keySet()) {
            Object intermediateResult = intermediateResultMap.get(groupKey);
            finalResultMap.put(groupKey, aggregationFunctions[i].extractFinalResult(intermediateResult));
        }
        finalResultMaps[i] = finalResultMap;
    }
    // Trim the final result maps to topN and set them into the broker response.
    AggregationGroupByTrimmingService aggregationGroupByTrimmingService = new AggregationGroupByTrimmingService(aggregationFunctions, (int) groupBy.getTopN());
    List<GroupByResult>[] groupByResultLists = aggregationGroupByTrimmingService.trimFinalResults(finalResultMaps);
    List<AggregationResult> aggregationResults = new ArrayList<>(numAggregationFunctions);
    for (int i = 0; i < numAggregationFunctions; i++) {
        List<GroupByResult> groupByResultList = groupByResultLists[i];
        List<String> groupByColumns = groupBy.getExpressions();
        if (groupByColumns == null) {
            groupByColumns = groupBy.getColumns();
        }
        aggregationResults.add(new AggregationResult(groupByResultList, groupByColumns, columnNames[i]));
    }
    brokerResponseNative.setAggregationResults(aggregationResults);
}

Example 2 with GroupByResult

use of com.linkedin.pinot.common.response.broker.GroupByResult in project pinot by linkedin.

the class AggregationGroupByTrimmingServiceTest method testTrimming.

@SuppressWarnings("unchecked")
@Test
public void testTrimming() {
    // Test server side trimming.
    Map<String, Object[]> intermediateResultsMap = new HashMap<>(NUM_GROUPS);
    for (int i = 0; i < NUM_GROUPS; i++) {
        intermediateResultsMap.put(_groups.get(i), new Double[] { (double) i });
    }
    Map<String, Object> trimmedIntermediateResultsMap = _serverTrimmingService.trimIntermediateResultsMap(intermediateResultsMap).get(0);
    int trimSize = trimmedIntermediateResultsMap.size();
    for (int i = NUM_GROUPS - trimSize; i < NUM_GROUPS; i++) {
        Assert.assertEquals(trimmedIntermediateResultsMap.get(_groups.get(i)), (double) i, ERROR_MESSAGE);
    }
    // Test broker side trimming.
    List<GroupByResult> groupByResults = _brokerTrimmingService.trimFinalResults(new Map[] { trimmedIntermediateResultsMap })[0];
    for (int i = 0; i < GROUP_BY_TOP_N; i++) {
        int expectedGroupIndex = NUM_GROUPS - 1 - i;
        GroupByResult groupByResult = groupByResults.get(i);
        List<String> group = groupByResult.getGroup();
        Assert.assertEquals(group.size(), NUM_GROUP_KEYS, ERROR_MESSAGE);
        String groupString = "";
        for (int j = 0; j < NUM_GROUP_KEYS; j++) {
            if (j != 0) {
                groupString += '\t';
            }
            groupString += group.get(j);
        }
        Assert.assertEquals(groupString, _groups.get(expectedGroupIndex), ERROR_MESSAGE);
        Assert.assertEquals(Double.parseDouble((String) groupByResult.getValue()), (double) expectedGroupIndex, ERROR_MESSAGE);
    }
}

Example 3 with GroupByResult

use of com.linkedin.pinot.common.response.broker.GroupByResult in project pinot by linkedin.

the class AggregationGroupByTrimmingService method trimFinalResults.

/**
   * Given an array of maps from group key to final result for each aggregation function, trim the results to topN size.
   */
@SuppressWarnings("unchecked")
@Nonnull
public List<GroupByResult>[] trimFinalResults(@Nonnull Map<String, Comparable>[] finalResultMaps) {
    List<GroupByResult>[] trimmedResults = new List[_numAggregationFunctions];
    for (int i = 0; i < _numAggregationFunctions; i++) {
        LinkedList<GroupByResult> groupByResults = new LinkedList<>();
        trimmedResults[i] = groupByResults;
        Map<String, Comparable> finalResultMap = finalResultMaps[i];
        if (finalResultMap.isEmpty()) {
            continue;
        }
        // Construct the priority queues.
        PriorityQueue<GroupKeyResultPair> priorityQueue = new PriorityQueue<>(_groupByTopN + 1, getGroupKeyResultPairComparator(_minOrders[i]));
        // Fill results into the priority queues.
        for (Map.Entry<String, Comparable> entry : finalResultMap.entrySet()) {
            String groupKey = entry.getKey();
            Comparable finalResult = entry.getValue();
            priorityQueue.add(new GroupKeyResultPair(groupKey, finalResult));
            if (priorityQueue.size() > _groupByTopN) {
                priorityQueue.poll();
            }
        }
        // Fill trimmed results into the list.
        while (!priorityQueue.isEmpty()) {
            GroupKeyResultPair groupKeyResultPair = priorityQueue.poll();
            GroupByResult groupByResult = new GroupByResult();
            // Do not remove trailing empty strings.
            String[] groupKeys = groupKeyResultPair._groupKey.split(GROUP_KEY_DELIMITER, -1);
            groupByResult.setGroup(Arrays.asList(groupKeys));
            groupByResult.setValue(AggregationFunctionUtils.formatValue(groupKeyResultPair._result));
            groupByResults.addFirst(groupByResult);
        }
    }
    return trimmedResults;
}

Example 4 with GroupByResult

use of com.linkedin.pinot.common.response.broker.GroupByResult in project pinot by linkedin.

the class TestUtils method assertGroupByResultsApproximation.

public static void assertGroupByResultsApproximation(List<GroupByResult> estimateValues, List<GroupByResult> actualValues, double precision) {
    LOGGER.info("====== assertGroupByResultsApproximation ======");
    // estimation should not affect number of groups formed
    Assert.assertEquals(estimateValues.size(), actualValues.size());
    Map<List<String>, Double> mapEstimate = new HashMap<>();
    Map<List<String>, Double> mapActual = new HashMap<>();
    for (GroupByResult gby : estimateValues) {
        mapEstimate.put(gby.getGroup(), Double.parseDouble(gby.getValue().toString()));
    }
    for (GroupByResult gby : actualValues) {
        mapActual.put(gby.getGroup(), Double.parseDouble(gby.getValue().toString()));
    }
    LOGGER.info("estimate: " + mapEstimate.keySet());
    LOGGER.info("actual: " + mapActual.keySet());
    int cnt = 0;
    for (List<String> key : mapEstimate.keySet()) {
        // Assert.assertEquals(mapActual.keySet().contains(key), true);
        if (mapActual.keySet().contains(key)) {
            assertApproximation(mapEstimate.get(key), mapActual.get(key), precision);
            cnt += 1;
        }
    }
    LOGGER.info("group overlap rate: " + (cnt + 0.0) / mapEstimate.keySet().size());
}