Examples with DataTable com.linkedin.pinot.common.utils.DataTable used on opensource projects

Example 31 with DataTable

use of com.linkedin.pinot.common.utils.DataTable in project pinot by linkedin.

the class BrokerReduceService method reduceOnDataTable.

@Nonnull
@Override
public BrokerResponseNative reduceOnDataTable(@Nonnull BrokerRequest brokerRequest, @Nonnull Map<ServerInstance, DataTable> dataTableMap, @Nullable BrokerMetrics brokerMetrics) {
    if (dataTableMap.size() == 0) {
        // Empty response.
        return BrokerResponseNative.empty();
    }
    BrokerResponseNative brokerResponseNative = new BrokerResponseNative();
    List<QueryProcessingException> processingExceptions = brokerResponseNative.getProcessingExceptions();
    long numDocsScanned = 0L;
    long numEntriesScannedInFilter = 0L;
    long numEntriesScannedPostFilter = 0L;
    long numTotalRawDocs = 0L;
    // Cache a data schema from data tables (try to cache one with data rows associated with it).
    DataSchema cachedDataSchema = null;
    // Process server response metadata.
    Iterator<Map.Entry<ServerInstance, DataTable>> iterator = dataTableMap.entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<ServerInstance, DataTable> entry = iterator.next();
        ServerInstance serverInstance = entry.getKey();
        DataTable dataTable = entry.getValue();
        Map<String, String> metadata = dataTable.getMetadata();
        // Reduce on trace info.
        if (brokerRequest.isEnableTrace()) {
            brokerResponseNative.getTraceInfo().put(serverInstance.getHostname(), metadata.get(DataTable.TRACE_INFO_METADATA_KEY));
        }
        // Reduce on exceptions.
        for (String key : metadata.keySet()) {
            if (key.startsWith(DataTable.EXCEPTION_METADATA_KEY)) {
                processingExceptions.add(new QueryProcessingException(Integer.parseInt(key.substring(9)), metadata.get(key)));
            }
        }
        // Reduce on execution statistics.
        String numDocsScannedString = metadata.get(DataTable.NUM_DOCS_SCANNED_METADATA_KEY);
        if (numDocsScannedString != null) {
            numDocsScanned += Long.parseLong(numDocsScannedString);
        }
        String numEntriesScannedInFilterString = metadata.get(DataTable.NUM_ENTRIES_SCANNED_IN_FILTER_METADATA_KEY);
        if (numEntriesScannedInFilterString != null) {
            numEntriesScannedInFilter += Long.parseLong(numEntriesScannedInFilterString);
        }
        String numEntriesScannedPostFilterString = metadata.get(DataTable.NUM_ENTRIES_SCANNED_POST_FILTER_METADATA_KEY);
        if (numEntriesScannedPostFilterString != null) {
            numEntriesScannedPostFilter += Long.parseLong(numEntriesScannedPostFilterString);
        }
        String numTotalRawDocsString = metadata.get(DataTable.TOTAL_DOCS_METADATA_KEY);
        if (numTotalRawDocsString != null) {
            numTotalRawDocs += Long.parseLong(numTotalRawDocsString);
        }
        // After processing the metadata, remove data tables without data rows inside.
        DataSchema dataSchema = dataTable.getDataSchema();
        if (dataSchema == null) {
            iterator.remove();
        } else {
            // Try to cache a data table with data rows inside, or cache one with data schema inside.
            if (dataTable.getNumberOfRows() == 0) {
                if (cachedDataSchema == null) {
                    cachedDataSchema = dataSchema;
                }
                iterator.remove();
            } else {
                cachedDataSchema = dataSchema;
            }
        }
    }
    // Set execution statistics.
    brokerResponseNative.setNumDocsScanned(numDocsScanned);
    brokerResponseNative.setNumEntriesScannedInFilter(numEntriesScannedInFilter);
    brokerResponseNative.setNumEntriesScannedPostFilter(numEntriesScannedPostFilter);
    brokerResponseNative.setTotalDocs(numTotalRawDocs);
    // Update broker metrics.
    String tableName = brokerRequest.getQuerySource().getTableName();
    if (brokerMetrics != null) {
        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.DOCUMENTS_SCANNED, numDocsScanned);
        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ENTRIES_SCANNED_IN_FILTER, numEntriesScannedInFilter);
        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.ENTRIES_SCANNED_POST_FILTER, numEntriesScannedPostFilter);
    }
    if (dataTableMap.isEmpty()) {
        // This will only happen to selection query.
        if (cachedDataSchema != null) {
            List<String> selectionColumns = SelectionOperatorUtils.getSelectionColumns(brokerRequest.getSelections().getSelectionColumns(), cachedDataSchema);
            brokerResponseNative.setSelectionResults(new SelectionResults(selectionColumns, new ArrayList<Serializable[]>(0)));
        }
    } else {
        // Reduce server responses data and set query results into the broker response.
        assert cachedDataSchema != null;
        if (brokerRequest.isSetSelections()) {
            // Selection query.
            // For data table map with more than one data tables, remove conflicting data tables.
            DataSchema masterDataSchema = cachedDataSchema.clone();
            if (dataTableMap.size() > 1) {
                List<String> droppedServers = removeConflictingResponses(masterDataSchema, dataTableMap);
                if (!droppedServers.isEmpty()) {
                    String errorMessage = QueryException.MERGE_RESPONSE_ERROR.getMessage() + ": responses for table: " + tableName + " from servers: " + droppedServers + " got dropped due to data schema inconsistency.";
                    LOGGER.error(errorMessage);
                    if (brokerMetrics != null) {
                        brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.RESPONSE_MERGE_EXCEPTIONS, 1);
                    }
                    brokerResponseNative.addToExceptions(new QueryProcessingException(QueryException.MERGE_RESPONSE_ERROR_CODE, errorMessage));
                }
            }
            setSelectionResults(brokerResponseNative, brokerRequest.getSelections(), dataTableMap, masterDataSchema);
        } else {
            // Aggregation query.
            AggregationFunction[] aggregationFunctions = AggregationFunctionUtils.getAggregationFunctions(brokerRequest.getAggregationsInfo());
            if (!brokerRequest.isSetGroupBy()) {
                // Aggregation only query.
                setAggregationResults(brokerResponseNative, aggregationFunctions, dataTableMap, cachedDataSchema);
            } else {
                // Aggregation group-by query.
                setGroupByResults(brokerResponseNative, aggregationFunctions, brokerRequest.getGroupBy(), dataTableMap);
            }
        }
    }
    return brokerResponseNative;
}

Example 32 with DataTable

use of com.linkedin.pinot.common.utils.DataTable in project pinot by linkedin.

the class BrokerReduceService method setAggregationResults.

/**
   * Reduce aggregation results from multiple servers and set them into BrokerResponseNative passed in.
   *
   * @param brokerResponseNative broker response.
   * @param aggregationFunctions array of aggregation functions.
   * @param dataTableMap map from server to data table.
   * @param dataSchema data schema.
   */
@SuppressWarnings("unchecked")
private void setAggregationResults(@Nonnull BrokerResponseNative brokerResponseNative, @Nonnull AggregationFunction[] aggregationFunctions, @Nonnull Map<ServerInstance, DataTable> dataTableMap, @Nonnull DataSchema dataSchema) {
    int numAggregationFunctions = aggregationFunctions.length;
    // Merge results from all data tables.
    Object[] intermediateResults = new Object[numAggregationFunctions];
    for (DataTable dataTable : dataTableMap.values()) {
        for (int i = 0; i < numAggregationFunctions; i++) {
            Object intermediateResultToMerge;
            FieldSpec.DataType columnType = dataSchema.getColumnType(i);
            switch(columnType) {
                case LONG:
                    intermediateResultToMerge = dataTable.getLong(0, i);
                    break;
                case DOUBLE:
                    intermediateResultToMerge = dataTable.getDouble(0, i);
                    break;
                case OBJECT:
                    intermediateResultToMerge = dataTable.getObject(0, i);
                    break;
                default:
                    throw new IllegalStateException("Illegal column type in aggregation results: " + columnType);
            }
            Object mergedIntermediateResult = intermediateResults[i];
            if (mergedIntermediateResult == null) {
                intermediateResults[i] = intermediateResultToMerge;
            } else {
                intermediateResults[i] = aggregationFunctions[i].merge(mergedIntermediateResult, intermediateResultToMerge);
            }
        }
    }
    // Extract final results and set them into the broker response.
    List<AggregationResult> reducedAggregationResults = new ArrayList<>(numAggregationFunctions);
    for (int i = 0; i < numAggregationFunctions; i++) {
        String formattedResult = AggregationFunctionUtils.formatValue(aggregationFunctions[i].extractFinalResult(intermediateResults[i]));
        reducedAggregationResults.add(new AggregationResult(dataSchema.getColumnName(i), formattedResult));
    }
    brokerResponseNative.setAggregationResults(reducedAggregationResults);
}

Example 33 with DataTable

use of com.linkedin.pinot.common.utils.DataTable in project pinot by linkedin.

the class BrokerReduceService method removeConflictingResponses.

/**
   * Given a data schema, remove data tables that are not compatible with this data schema.
   * <p>Upgrade the data schema passed in to cover all remaining data schemas.
   *
   * @param dataSchema data schema.
   * @param dataTableMap map from server to data table.
   * @return list of server names where the data table got removed.
   */
@Nonnull
private List<String> removeConflictingResponses(@Nonnull DataSchema dataSchema, @Nonnull Map<ServerInstance, DataTable> dataTableMap) {
    List<String> droppedServers = new ArrayList<>();
    Iterator<Map.Entry<ServerInstance, DataTable>> iterator = dataTableMap.entrySet().iterator();
    while (iterator.hasNext()) {
        Map.Entry<ServerInstance, DataTable> entry = iterator.next();
        DataSchema dataSchemaToCompare = entry.getValue().getDataSchema();
        assert dataSchemaToCompare != null;
        if (!dataSchema.isTypeCompatibleWith(dataSchemaToCompare)) {
            droppedServers.add(entry.getKey().toString());
            iterator.remove();
        } else {
            dataSchema.upgradeToCover(dataSchemaToCompare);
        }
    }
    return droppedServers;
}

Example 34 with DataTable

use of com.linkedin.pinot.common.utils.DataTable in project pinot by linkedin.

the class ApproximateQueryTestUtil method runQuery.

public static Object runQuery(QueryExecutor queryExecutor, List<String> segments, AvroQueryGenerator.TestAggreationQuery query, ServerMetrics metrics) {
    LOGGER.info("\nRunning: " + query.getPql());
    final BrokerRequest brokerRequest = REQUEST_COMPILER.compileToBrokerRequest(query.getPql());
    InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (String segment : segments) {
        instanceRequest.getSearchSegments().add(segment);
    }
    QueryRequest queryRequest = new QueryRequest(instanceRequest, metrics);
    final DataTable instanceResponse = queryExecutor.processQuery(queryRequest, queryRunners);
    final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    final BrokerResponseNative brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
    AggregationResult result = brokerResponse.getAggregationResults().get(0);
    Assert.assertNotNull(result);
    if (result.getValue() != null) {
        LOGGER.info("Aggregation Result is " + result.getValue().toString());
    } else if (result.getGroupByResult() != null) {
        LOGGER.info("GroupBy Result is " + result.getGroupByResult().toString());
    } else {
        throw new RuntimeException("Aggregation and GroupBy Results both null.");
    }
    // compute value
    Object val;
    if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
        val = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getValue().toString());
    } else {
        val = brokerResponse.getAggregationResults().get(0).getGroupByResult();
    }
    return val;
}

Example 35 with DataTable

use of com.linkedin.pinot.common.utils.DataTable in project pinot by linkedin.

the class BrokerRequestHandler method processOptimizedBrokerRequests.

/**
   * Process the optimized broker requests for both OFFLINE and REALTIME table.
   *
   * @param originalBrokerRequest original broker request.
   * @param offlineBrokerRequest broker request for OFFLINE table.
   * @param realtimeBrokerRequest broker request for REALTIME table.
   * @param reduceService reduce service.
   * @param bucketingSelection customized bucketing selection.
   * @param scatterGatherStats scatter-gather statistics.
   * @param requestId request ID.
   * @return broker response.
   * @throws InterruptedException
   */
@Nonnull
private BrokerResponse processOptimizedBrokerRequests(@Nonnull BrokerRequest originalBrokerRequest, @Nullable BrokerRequest offlineBrokerRequest, @Nullable BrokerRequest realtimeBrokerRequest, @Nonnull ReduceService reduceService, @Nonnull ScatterGatherStats scatterGatherStats, @Nullable BucketingSelection bucketingSelection, long requestId) throws InterruptedException {
    String originalTableName = originalBrokerRequest.getQuerySource().getTableName();
    ResponseType serverResponseType = BrokerResponseFactory.getResponseType(originalBrokerRequest.getResponseFormat());
    PhaseTimes phaseTimes = new PhaseTimes();
    // Step 1: find the candidate servers to be queried for each set of segments from the routing table.
    // Step 2: select servers for each segment set and scatter request to the servers.
    String offlineTableName = null;
    CompositeFuture<ServerInstance, ByteBuf> offlineCompositeFuture = null;
    if (offlineBrokerRequest != null) {
        offlineTableName = offlineBrokerRequest.getQuerySource().getTableName();
        offlineCompositeFuture = routeAndScatterBrokerRequest(offlineBrokerRequest, phaseTimes, scatterGatherStats, true, bucketingSelection, requestId);
    }
    String realtimeTableName = null;
    CompositeFuture<ServerInstance, ByteBuf> realtimeCompositeFuture = null;
    if (realtimeBrokerRequest != null) {
        realtimeTableName = realtimeBrokerRequest.getQuerySource().getTableName();
        realtimeCompositeFuture = routeAndScatterBrokerRequest(realtimeBrokerRequest, phaseTimes, scatterGatherStats, false, bucketingSelection, requestId);
    }
    if ((offlineCompositeFuture == null) && (realtimeCompositeFuture == null)) {
        // No server found in either OFFLINE or REALTIME table.
        return BrokerResponseFactory.getStaticEmptyBrokerResponse(serverResponseType);
    }
    // Step 3: gather response from the servers.
    int numServersQueried = 0;
    long gatherStartTime = System.nanoTime();
    List<ProcessingException> processingExceptions = new ArrayList<>();
    Map<ServerInstance, ByteBuf> offlineServerResponseMap = null;
    Map<ServerInstance, ByteBuf> realtimeServerResponseMap = null;
    if (offlineCompositeFuture != null) {
        numServersQueried += offlineCompositeFuture.getNumFutures();
        offlineServerResponseMap = gatherServerResponses(offlineCompositeFuture, scatterGatherStats, true, offlineTableName, processingExceptions);
    }
    if (realtimeCompositeFuture != null) {
        numServersQueried += realtimeCompositeFuture.getNumFutures();
        realtimeServerResponseMap = gatherServerResponses(realtimeCompositeFuture, scatterGatherStats, false, realtimeTableName, processingExceptions);
    }
    phaseTimes.addToGatherTime(System.nanoTime() - gatherStartTime);
    if ((offlineServerResponseMap == null) && (realtimeServerResponseMap == null)) {
        // No response gathered.
        return BrokerResponseFactory.getBrokerResponseWithExceptions(serverResponseType, processingExceptions);
    }
    //Step 4: deserialize the server responses.
    int numServersResponded = 0;
    long deserializationStartTime = System.nanoTime();
    Map<ServerInstance, DataTable> dataTableMap = new HashMap<>();
    if (offlineServerResponseMap != null) {
        numServersResponded += offlineServerResponseMap.size();
        deserializeServerResponses(offlineServerResponseMap, true, dataTableMap, offlineTableName, processingExceptions);
    }
    if (realtimeServerResponseMap != null) {
        numServersResponded += realtimeServerResponseMap.size();
        deserializeServerResponses(realtimeServerResponseMap, false, dataTableMap, realtimeTableName, processingExceptions);
    }
    phaseTimes.addToDeserializationTime(System.nanoTime() - deserializationStartTime);
    // Step 5: reduce (merge) the server responses and create a broker response to be returned.
    long reduceStartTime = System.nanoTime();
    BrokerResponse brokerResponse = reduceService.reduceOnDataTable(originalBrokerRequest, dataTableMap, _brokerMetrics);
    phaseTimes.addToReduceTime(System.nanoTime() - reduceStartTime);
    // Set processing exceptions and number of servers queried/responded.
    brokerResponse.setExceptions(processingExceptions);
    brokerResponse.setNumServersQueried(numServersQueried);
    brokerResponse.setNumServersResponded(numServersResponded);
    // Update broker metrics.
    phaseTimes.addPhaseTimesToBrokerMetrics(_brokerMetrics, originalTableName);
    if (brokerResponse.getExceptionsSize() > 0) {
        _brokerMetrics.addMeteredTableValue(originalTableName, BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, 1);
    }
    if (numServersQueried > numServersResponded) {
        _brokerMetrics.addMeteredTableValue(originalTableName, BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1);
    }
    return brokerResponse;
}