use of com.linkedin.pinot.core.query.aggregation.function.AggregationFunction in project pinot by linkedin.
the class DefaultGroupByExecutor method aggregateColumn.
/**
* Helper method to perform aggregation for a given column.
*
* @param transformBlock Transform block to aggregate
* @param aggrFuncContext Aggregation function context
* @param resultHolder Holder for results of aggregation
*/
@SuppressWarnings("ConstantConditions")
private void aggregateColumn(TransformBlock transformBlock, AggregationFunctionContext aggrFuncContext, GroupByResultHolder resultHolder) {
AggregationFunction aggregationFunction = aggrFuncContext.getAggregationFunction();
String[] aggregationColumns = aggrFuncContext.getAggregationColumns();
Preconditions.checkState(aggregationColumns.length == 1);
int length = transformBlock.getNumDocs();
if (!aggregationFunction.getName().equals(AggregationFunctionFactory.AggregationFunctionType.COUNT.getName())) {
BlockValSet blockValueSet = transformBlock.getBlockValueSet(aggregationColumns[0]);
if (_hasMVGroupByColumns) {
aggregationFunction.aggregateGroupByMV(length, _docIdToMVGroupKey, resultHolder, blockValueSet);
} else {
aggregationFunction.aggregateGroupBySV(length, _docIdToSVGroupKey, resultHolder, blockValueSet);
}
} else {
if (_hasMVGroupByColumns) {
aggregationFunction.aggregateGroupByMV(length, _docIdToMVGroupKey, resultHolder);
} else {
aggregationFunction.aggregateGroupBySV(length, _docIdToSVGroupKey, resultHolder);
}
}
}
use of com.linkedin.pinot.core.query.aggregation.function.AggregationFunction in project pinot by linkedin.
the class BrokerReduceService method reduceOnDataTable.
@Nonnull
@Override
public BrokerResponseNative reduceOnDataTable(@Nonnull BrokerRequest brokerRequest, @Nonnull Map<ServerInstance, DataTable> dataTableMap, @Nullable BrokerMetrics brokerMetrics) {
if (dataTableMap.size() == 0) {
// Empty response.
return BrokerResponseNative.empty();
}
BrokerResponseNative brokerResponseNative = new BrokerResponseNative();
List<QueryProcessingException> processingExceptions = brokerResponseNative.getProcessingExceptions();
long numDocsScanned = 0L;
long numEntriesScannedInFilter = 0L;
long numEntriesScannedPostFilter = 0L;
long numTotalRawDocs = 0L;
// Cache a data schema from data tables (try to cache one with data rows associated with it).
DataSchema cachedDataSchema = null;
// Process server response metadata.
Iterator<Map.Entry<ServerInstance, DataTable>> iterator = dataTableMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<ServerInstance, DataTable> entry = iterator.next();
ServerInstance serverInstance = entry.getKey();
DataTable dataTable = entry.getValue();
Map<String, String> metadata = dataTable.getMetadata();
// Reduce on trace info.
if (brokerRequest.isEnableTrace()) {
brokerResponseNative.getTraceInfo().put(serverInstance.getHostname(), metadata.get(DataTable.TRACE_INFO_METADATA_KEY));
}
// Reduce on exceptions.
for (String key : metadata.keySet()) {
if (key.startsWith(DataTable.EXCEPTION_METADATA_KEY)) {
processingExceptions.add(new QueryProcessingException(Integer.parseInt(key.substring(9)), metadata.get(key)));
}
}
// Reduce on execution statistics.
String numDocsScannedString = metadata.get(DataTable.NUM_DOCS_SCANNED_METADATA_KEY);
if (numDocsScannedString != null) {
numDocsScanned += Long.parseLong(numDocsScannedString);
}
String numEntriesScannedInFilterString = metadata.get(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY);
if (numEntriesScannedInFilterString != null) {
numEntriesScannedInFilter += Long.parseLong(numEntriesScannedInFilterString);
}
String numEntriesScannedPostFilterString = metadata.get(DataTable.NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY);
if (numEntriesScannedPostFilterString != null) {
numEntriesScannedPostFilter += Long.parseLong(numEntriesScannedPostFilterString);
}
String numTotalRawDocsString = metadata.get(DataTable.TOTAL_DOCS_METADATA_KEY);
if (numTotalRawDocsString != null) {
numTotalRawDocs += Long.parseLong(numTotalRawDocsString);
}
// After processing the metadata, remove data tables without data rows inside.
DataSchema dataSchema = dataTable.getDataSchema();
if (dataSchema == null) {
iterator.remove();
} else {
// Try to cache a data table with data rows inside, or cache one with data schema inside.
if (dataTable.getNumberOfRows() == 0) {
if (cachedDataSchema == null) {
cachedDataSchema = dataSchema;
}
iterator.remove();
} else {
cachedDataSchema = dataSchema;
}
}
}
// Set execution statistics.
brokerResponseNative.setNumDocsScanned(numDocsScanned);
brokerResponseNative.setNumEntriesScannedInFilter(numEntriesScannedInFilter);
brokerResponseNative.setNumEntriesScannedPostFilter(numEntriesScannedPostFilter);
brokerResponseNative.setTotalDocs(numTotalRawDocs);
// Update broker metrics.
String tableName = brokerRequest.getQuerySource().getTableName();
if (brokerMetrics != null) {
brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.DOCUMENTS_SCANNED, numDocsScanned);
brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ENTRIES_SCANNED_IN_FILTER, numEntriesScannedInFilter);
brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ENTRIES_SCANNED_POST_FILTER, numEntriesScannedPostFilter);
}
if (dataTableMap.isEmpty()) {
// This will only happen to selection query.
if (cachedDataSchema != null) {
List<String> selectionColumns = SelectionOperatorUtils.getSelectionColumns(brokerRequest.getSelections().getSelectionColumns(), cachedDataSchema);
brokerResponseNative.setSelectionResults(new SelectionResults(selectionColumns, new ArrayList<Serializable[]>(0)));
}
} else {
// Reduce server responses data and set query results into the broker response.
assert cachedDataSchema != null;
if (brokerRequest.isSetSelections()) {
// Selection query.
// For data table map with more than one data tables, remove conflicting data tables.
DataSchema masterDataSchema = cachedDataSchema.clone();
if (dataTableMap.size() > 1) {
List<String> droppedServers = removeConflictingResponses(masterDataSchema, dataTableMap);
if (!droppedServers.isEmpty()) {
String errorMessage = QueryException.MERGE_RESPONSE_ERROR.getMessage() + ": responses for table: " + tableName + " from servers: " + droppedServers + " got dropped due to data schema inconsistency.";
LOGGER.error(errorMessage);
if (brokerMetrics != null) {
brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.RESPONSE_MERGE_EXCEPTIONS, 1);
}
brokerResponseNative.addToExceptions(new QueryProcessingException(QueryException.MERGE_RESPONSE_ERROR_CODE, errorMessage));
}
}
setSelectionResults(brokerResponseNative, brokerRequest.getSelections(), dataTableMap, masterDataSchema);
} else {
// Aggregation query.
AggregationFunction[] aggregationFunctions = AggregationFunctionUtils.getAggregationFunctions(brokerRequest.getAggregationsInfo());
if (!brokerRequest.isSetGroupBy()) {
// Aggregation only query.
setAggregationResults(brokerResponseNative, aggregationFunctions, dataTableMap, cachedDataSchema);
} else {
// Aggregation group-by query.
setGroupByResults(brokerResponseNative, aggregationFunctions, brokerRequest.getGroupBy(), dataTableMap);
}
}
}
return brokerResponseNative;
}
use of com.linkedin.pinot.core.query.aggregation.function.AggregationFunction in project pinot by linkedin.
the class AggregationFunctionContext method instantiate.
public static AggregationFunctionContext instantiate(AggregationInfo aggregationInfo) {
String[] aggrColumns = aggregationInfo.getAggregationParams().get("column").trim().split(",");
String functionName = aggregationInfo.getAggregationType();
AggregationFunction aggregationFunction = AggregationFunctionFactory.getAggregationFunction(functionName);
return new AggregationFunctionContext(aggrColumns, aggregationFunction);
}
use of com.linkedin.pinot.core.query.aggregation.function.AggregationFunction in project pinot by linkedin.
the class DefaultAggregationExecutor method aggregateColumn.
/**
* Helper method to perform aggregation for a given column.
*
* @param aggrFuncContext aggregation function context.
* @param resultHolder result holder.
*/
@SuppressWarnings("ConstantConditions")
private void aggregateColumn(TransformBlock transformBlock, AggregationFunctionContext aggrFuncContext, AggregationResultHolder resultHolder) {
AggregationFunction aggregationFunction = aggrFuncContext.getAggregationFunction();
String[] aggregationColumns = aggrFuncContext.getAggregationColumns();
Preconditions.checkState(aggregationColumns.length == 1);
int length = transformBlock.getNumDocs();
if (!aggregationFunction.getName().equals(AggregationFunctionFactory.AggregationFunctionType.COUNT.getName())) {
BlockValSet blockValSet = transformBlock.getBlockValueSet(aggregationColumns[0]);
aggregationFunction.aggregate(length, resultHolder, blockValSet);
} else {
aggregationFunction.aggregate(length, resultHolder);
}
}
use of com.linkedin.pinot.core.query.aggregation.function.AggregationFunction in project pinot by linkedin.
the class DefaultAggregationExecutor method getResult.
@Override
public List<Object> getResult() {
Preconditions.checkState(_finished, "Method 'getResult' cannot be called before 'finish' for class " + getClass().getName());
List<Object> aggregationResults = new ArrayList<>(_numAggrFunc);
for (int i = 0; i < _numAggrFunc; i++) {
AggregationFunction aggregationFunction = _aggrFuncContextArray[i].getAggregationFunction();
aggregationResults.add(aggregationFunction.extractAggregationResult(_resultHolderArray[i]));
}
return aggregationResults;
}
Aggregations