Examples with ResponseType com.linkedin.pinot.common.response.BrokerResponseFactory.ResponseType used on opensource projects

Example 1 with ResponseType

use of com.linkedin.pinot.common.response.BrokerResponseFactory.ResponseType in project pinot by linkedin.

the class BrokerRequestHandler method processBrokerRequest.

/**
   * Main method to process the request.
   * <p>Following lifecycle stages:
   * <ul>
   *   <li>1. Find the candidate servers to be queried for each set of segments from the routing table.</li>
   *   <li>2. Select servers for each segment set and scatter request to the servers.</li>
   *   <li>3. Gather responses from the servers.</li>
   *   <li>4. Deserialize the server responses.</li>
   *   <li>5. Reduce (merge) the server responses and create a broker response to be returned.</li>
   * </ul>
   *
   * @param brokerRequest broker request to be processed.
   * @param scatterGatherStats scatter-gather statistics.
   * @param requestId broker request ID.
   * @return broker response.
   * @throws InterruptedException
   */
@Nonnull
public BrokerResponse processBrokerRequest(@Nonnull BrokerRequest brokerRequest, @Nonnull ScatterGatherStats scatterGatherStats, long requestId) throws InterruptedException {
    String tableName = brokerRequest.getQuerySource().getTableName();
    ResponseType responseType = BrokerResponseFactory.getResponseType(brokerRequest.getResponseFormat());
    LOGGER.debug("Broker Response Type: {}", responseType.name());
    String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName);
    if (!_routingTable.routingTableExists(offlineTableName)) {
        offlineTableName = null;
    }
    String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName);
    if (!_routingTable.routingTableExists(realtimeTableName)) {
        realtimeTableName = null;
    }
    if ((offlineTableName == null) && (realtimeTableName == null)) {
        // No table matches the broker request.
        LOGGER.warn("No table matches the name: {}", tableName);
        _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.RESOURCE_MISSING_EXCEPTIONS, 1);
        return BrokerResponseFactory.getStaticNoTableHitBrokerResponse(responseType);
    } else {
        // At least one table matches the broker request.
        BrokerRequest offlineBrokerRequest = null;
        BrokerRequest realtimeBrokerRequest = null;
        // TODO: get time column name from schema or table config so that we can apply it in realtime only use case.
        // We get timeColumnName from time boundary service currently, which only exists for offline table.
        String timeColumnName = (offlineTableName != null) ? getTimeColumnName(offlineTableName) : null;
        if ((offlineTableName != null) && (realtimeTableName != null)) {
            // Hybrid table.
            offlineBrokerRequest = _optimizer.optimize(getOfflineBrokerRequest(brokerRequest), timeColumnName);
            realtimeBrokerRequest = _optimizer.optimize(getRealtimeBrokerRequest(brokerRequest), timeColumnName);
        } else if (offlineTableName != null) {
            // Offline table only.
            brokerRequest.getQuerySource().setTableName(offlineTableName);
            offlineBrokerRequest = _optimizer.optimize(brokerRequest, timeColumnName);
        } else {
            // Realtime table only.
            brokerRequest.getQuerySource().setTableName(realtimeTableName);
            realtimeBrokerRequest = _optimizer.optimize(brokerRequest, timeColumnName);
        }
        ReduceService reduceService = _reduceServiceRegistry.get(responseType);
        // TODO: wire up the customized BucketingSelection.
        return processOptimizedBrokerRequests(brokerRequest, offlineBrokerRequest, realtimeBrokerRequest, reduceService, scatterGatherStats, null, requestId);
    }
}

Example 2 with ResponseType

use of com.linkedin.pinot.common.response.BrokerResponseFactory.ResponseType in project pinot by linkedin.

the class BrokerRequestHandler method processOptimizedBrokerRequests.

/**
   * Process the optimized broker requests for both OFFLINE and REALTIME table.
   *
   * @param originalBrokerRequest original broker request.
   * @param offlineBrokerRequest broker request for OFFLINE table.
   * @param realtimeBrokerRequest broker request for REALTIME table.
   * @param reduceService reduce service.
   * @param bucketingSelection customized bucketing selection.
   * @param scatterGatherStats scatter-gather statistics.
   * @param requestId request ID.
   * @return broker response.
   * @throws InterruptedException
   */
@Nonnull
private BrokerResponse processOptimizedBrokerRequests(@Nonnull BrokerRequest originalBrokerRequest, @Nullable BrokerRequest offlineBrokerRequest, @Nullable BrokerRequest realtimeBrokerRequest, @Nonnull ReduceService reduceService, @Nonnull ScatterGatherStats scatterGatherStats, @Nullable BucketingSelection bucketingSelection, long requestId) throws InterruptedException {
    String originalTableName = originalBrokerRequest.getQuerySource().getTableName();
    ResponseType serverResponseType = BrokerResponseFactory.getResponseType(originalBrokerRequest.getResponseFormat());
    PhaseTimes phaseTimes = new PhaseTimes();
    // Step 1: find the candidate servers to be queried for each set of segments from the routing table.
    // Step 2: select servers for each segment set and scatter request to the servers.
    String offlineTableName = null;
    CompositeFuture<ServerInstance, ByteBuf> offlineCompositeFuture = null;
    if (offlineBrokerRequest != null) {
        offlineTableName = offlineBrokerRequest.getQuerySource().getTableName();
        offlineCompositeFuture = routeAndScatterBrokerRequest(offlineBrokerRequest, phaseTimes, scatterGatherStats, true, bucketingSelection, requestId);
    }
    String realtimeTableName = null;
    CompositeFuture<ServerInstance, ByteBuf> realtimeCompositeFuture = null;
    if (realtimeBrokerRequest != null) {
        realtimeTableName = realtimeBrokerRequest.getQuerySource().getTableName();
        realtimeCompositeFuture = routeAndScatterBrokerRequest(realtimeBrokerRequest, phaseTimes, scatterGatherStats, false, bucketingSelection, requestId);
    }
    if ((offlineCompositeFuture == null) && (realtimeCompositeFuture == null)) {
        // No server found in either OFFLINE or REALTIME table.
        return BrokerResponseFactory.getStaticEmptyBrokerResponse(serverResponseType);
    }
    // Step 3: gather response from the servers.
    int numServersQueried = 0;
    long gatherStartTime = System.nanoTime();
    List<ProcessingException> processingExceptions = new ArrayList<>();
    Map<ServerInstance, ByteBuf> offlineServerResponseMap = null;
    Map<ServerInstance, ByteBuf> realtimeServerResponseMap = null;
    if (offlineCompositeFuture != null) {
        numServersQueried += offlineCompositeFuture.getNumFutures();
        offlineServerResponseMap = gatherServerResponses(offlineCompositeFuture, scatterGatherStats, true, offlineTableName, processingExceptions);
    }
    if (realtimeCompositeFuture != null) {
        numServersQueried += realtimeCompositeFuture.getNumFutures();
        realtimeServerResponseMap = gatherServerResponses(realtimeCompositeFuture, scatterGatherStats, false, realtimeTableName, processingExceptions);
    }
    phaseTimes.addToGatherTime(System.nanoTime() - gatherStartTime);
    if ((offlineServerResponseMap == null) && (realtimeServerResponseMap == null)) {
        // No response gathered.
        return BrokerResponseFactory.getBrokerResponseWithExceptions(serverResponseType, processingExceptions);
    }
    //Step 4: deserialize the server responses.
    int numServersResponded = 0;
    long deserializationStartTime = System.nanoTime();
    Map<ServerInstance, DataTable> dataTableMap = new HashMap<>();
    if (offlineServerResponseMap != null) {
        numServersResponded += offlineServerResponseMap.size();
        deserializeServerResponses(offlineServerResponseMap, true, dataTableMap, offlineTableName, processingExceptions);
    }
    if (realtimeServerResponseMap != null) {
        numServersResponded += realtimeServerResponseMap.size();
        deserializeServerResponses(realtimeServerResponseMap, false, dataTableMap, realtimeTableName, processingExceptions);
    }
    phaseTimes.addToDeserializationTime(System.nanoTime() - deserializationStartTime);
    // Step 5: reduce (merge) the server responses and create a broker response to be returned.
    long reduceStartTime = System.nanoTime();
    BrokerResponse brokerResponse = reduceService.reduceOnDataTable(originalBrokerRequest, dataTableMap, _brokerMetrics);
    phaseTimes.addToReduceTime(System.nanoTime() - reduceStartTime);
    // Set processing exceptions and number of servers queried/responded.
    brokerResponse.setExceptions(processingExceptions);
    brokerResponse.setNumServersQueried(numServersQueried);
    brokerResponse.setNumServersResponded(numServersResponded);
    // Update broker metrics.
    phaseTimes.addPhaseTimesToBrokerMetrics(_brokerMetrics, originalTableName);
    if (brokerResponse.getExceptionsSize() > 0) {
        _brokerMetrics.addMeteredTableValue(originalTableName, BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, 1);
    }
    if (numServersQueried > numServersResponded) {
        _brokerMetrics.addMeteredTableValue(originalTableName, BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1);
    }
    return brokerResponse;
}