Examples with BrokerResponse com.linkedin.pinot.common.response.BrokerResponse used on opensource projects

Example 1 with BrokerResponse

use of com.linkedin.pinot.common.response.BrokerResponse in project pinot by linkedin.

the class BrokerRequestHandler method handleRequest.

/**
   * Process a JSON format request.
   *
   * @param request JSON format request to be processed.
   * @return broker response.
   * @throws Exception
   */
@Nonnull
public BrokerResponse handleRequest(@Nonnull JSONObject request) throws Exception {
    long requestId = _requestIdGenerator.incrementAndGet();
    String pql = request.getString("pql");
    LOGGER.debug("Query string for requestId {}: {}", requestId, pql);
    boolean isTraceEnabled = false;
    if (request.has("trace")) {
        isTraceEnabled = Boolean.parseBoolean(request.getString("trace"));
        LOGGER.debug("Trace is set to: {} for requestId {}: {}", isTraceEnabled, requestId, pql);
    }
    Map<String, String> debugOptions = null;
    if (request.has("debugOptions")) {
        String routingOptionParameter = request.getString("debugOptions");
        debugOptions = Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('=').split(routingOptionParameter);
        LOGGER.debug("Debug options are set to: {} for requestId {}: {}", debugOptions, requestId, pql);
    }
    // Compile and validate the request.
    long compilationStartTime = System.nanoTime();
    BrokerRequest brokerRequest;
    try {
        brokerRequest = REQUEST_COMPILER.compileToBrokerRequest(pql);
    } catch (Exception e) {
        LOGGER.warn("Parsing error on requestId {}: {}", requestId, pql, e);
        _brokerMetrics.addMeteredGlobalValue(BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, 1);
        return BrokerResponseFactory.getBrokerResponseWithException(DEFAULT_BROKER_RESPONSE_TYPE, QueryException.getException(QueryException.PQL_PARSING_ERROR, e));
    }
    String tableName = brokerRequest.getQuerySource().getTableName();
    try {
        validateRequest(brokerRequest);
    } catch (Exception e) {
        LOGGER.warn("Validation error on requestId {}: {}", requestId, pql, e);
        _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.QUERY_VALIDATION_EXCEPTIONS, 1);
        return BrokerResponseFactory.getBrokerResponseWithException(DEFAULT_BROKER_RESPONSE_TYPE, QueryException.getException(QueryException.QUERY_VALIDATION_ERROR, e));
    }
    if (isTraceEnabled) {
        brokerRequest.setEnableTrace(true);
    }
    if (debugOptions != null) {
        brokerRequest.setDebugOptions(debugOptions);
    }
    brokerRequest.setResponseFormat(ResponseType.BROKER_RESPONSE_TYPE_NATIVE.name());
    _brokerMetrics.addPhaseTiming(tableName, BrokerQueryPhase.REQUEST_COMPILATION, System.nanoTime() - compilationStartTime);
    _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.QUERIES, 1);
    // Execute the query.
    long executionStartTime = System.nanoTime();
    ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerResponse brokerResponse = processBrokerRequest(brokerRequest, scatterGatherStats, requestId);
    _brokerMetrics.addPhaseTiming(tableName, BrokerQueryPhase.QUERY_EXECUTION, System.nanoTime() - executionStartTime);
    // Set total query processing time.
    long totalTimeMs = TimeUnit.MILLISECONDS.convert(System.nanoTime() - compilationStartTime, TimeUnit.NANOSECONDS);
    brokerResponse.setTimeUsedMs(totalTimeMs);
    LOGGER.debug("Broker Response: {}", brokerResponse);
    // Table name might have been changed (with suffix _OFFLINE/_REALTIME appended).
    LOGGER.info("RequestId: {}, table: {}, totalTimeMs: {}, numDocsScanned: {}, numEntriesScannedInFilter: {}, " + "numEntriesScannedPostFilter: {}, totalDocs: {}, scatterGatherStats: {}, query: {}", requestId, brokerRequest.getQuerySource().getTableName(), totalTimeMs, brokerResponse.getNumDocsScanned(), brokerResponse.getNumEntriesScannedInFilter(), brokerResponse.getNumEntriesScannedPostFilter(), brokerResponse.getTotalDocs(), scatterGatherStats, pql);
    return brokerResponse;
}

Example 2 with BrokerResponse

use of com.linkedin.pinot.common.response.BrokerResponse in project pinot by linkedin.

the class BrokerRequestHandler method processOptimizedBrokerRequests.

/**
   * Process the optimized broker requests for both OFFLINE and REALTIME table.
   *
   * @param originalBrokerRequest original broker request.
   * @param offlineBrokerRequest broker request for OFFLINE table.
   * @param realtimeBrokerRequest broker request for REALTIME table.
   * @param reduceService reduce service.
   * @param bucketingSelection customized bucketing selection.
   * @param scatterGatherStats scatter-gather statistics.
   * @param requestId request ID.
   * @return broker response.
   * @throws InterruptedException
   */
@Nonnull
private BrokerResponse processOptimizedBrokerRequests(@Nonnull BrokerRequest originalBrokerRequest, @Nullable BrokerRequest offlineBrokerRequest, @Nullable BrokerRequest realtimeBrokerRequest, @Nonnull ReduceService reduceService, @Nonnull ScatterGatherStats scatterGatherStats, @Nullable BucketingSelection bucketingSelection, long requestId) throws InterruptedException {
    String originalTableName = originalBrokerRequest.getQuerySource().getTableName();
    ResponseType serverResponseType = BrokerResponseFactory.getResponseType(originalBrokerRequest.getResponseFormat());
    PhaseTimes phaseTimes = new PhaseTimes();
    // Step 1: find the candidate servers to be queried for each set of segments from the routing table.
    // Step 2: select servers for each segment set and scatter request to the servers.
    String offlineTableName = null;
    CompositeFuture<ServerInstance, ByteBuf> offlineCompositeFuture = null;
    if (offlineBrokerRequest != null) {
        offlineTableName = offlineBrokerRequest.getQuerySource().getTableName();
        offlineCompositeFuture = routeAndScatterBrokerRequest(offlineBrokerRequest, phaseTimes, scatterGatherStats, true, bucketingSelection, requestId);
    }
    String realtimeTableName = null;
    CompositeFuture<ServerInstance, ByteBuf> realtimeCompositeFuture = null;
    if (realtimeBrokerRequest != null) {
        realtimeTableName = realtimeBrokerRequest.getQuerySource().getTableName();
        realtimeCompositeFuture = routeAndScatterBrokerRequest(realtimeBrokerRequest, phaseTimes, scatterGatherStats, false, bucketingSelection, requestId);
    }
    if ((offlineCompositeFuture == null) && (realtimeCompositeFuture == null)) {
        // No server found in either OFFLINE or REALTIME table.
        return BrokerResponseFactory.getStaticEmptyBrokerResponse(serverResponseType);
    }
    // Step 3: gather response from the servers.
    int numServersQueried = 0;
    long gatherStartTime = System.nanoTime();
    List<ProcessingException> processingExceptions = new ArrayList<>();
    Map<ServerInstance, ByteBuf> offlineServerResponseMap = null;
    Map<ServerInstance, ByteBuf> realtimeServerResponseMap = null;
    if (offlineCompositeFuture != null) {
        numServersQueried += offlineCompositeFuture.getNumFutures();
        offlineServerResponseMap = gatherServerResponses(offlineCompositeFuture, scatterGatherStats, true, offlineTableName, processingExceptions);
    }
    if (realtimeCompositeFuture != null) {
        numServersQueried += realtimeCompositeFuture.getNumFutures();
        realtimeServerResponseMap = gatherServerResponses(realtimeCompositeFuture, scatterGatherStats, false, realtimeTableName, processingExceptions);
    }
    phaseTimes.addToGatherTime(System.nanoTime() - gatherStartTime);
    if ((offlineServerResponseMap == null) && (realtimeServerResponseMap == null)) {
        // No response gathered.
        return BrokerResponseFactory.getBrokerResponseWithExceptions(serverResponseType, processingExceptions);
    }
    //Step 4: deserialize the server responses.
    int numServersResponded = 0;
    long deserializationStartTime = System.nanoTime();
    Map<ServerInstance, DataTable> dataTableMap = new HashMap<>();
    if (offlineServerResponseMap != null) {
        numServersResponded += offlineServerResponseMap.size();
        deserializeServerResponses(offlineServerResponseMap, true, dataTableMap, offlineTableName, processingExceptions);
    }
    if (realtimeServerResponseMap != null) {
        numServersResponded += realtimeServerResponseMap.size();
        deserializeServerResponses(realtimeServerResponseMap, false, dataTableMap, realtimeTableName, processingExceptions);
    }
    phaseTimes.addToDeserializationTime(System.nanoTime() - deserializationStartTime);
    // Step 5: reduce (merge) the server responses and create a broker response to be returned.
    long reduceStartTime = System.nanoTime();
    BrokerResponse brokerResponse = reduceService.reduceOnDataTable(originalBrokerRequest, dataTableMap, _brokerMetrics);
    phaseTimes.addToReduceTime(System.nanoTime() - reduceStartTime);
    // Set processing exceptions and number of servers queried/responded.
    brokerResponse.setExceptions(processingExceptions);
    brokerResponse.setNumServersQueried(numServersQueried);
    brokerResponse.setNumServersResponded(numServersResponded);
    // Update broker metrics.
    phaseTimes.addPhaseTimesToBrokerMetrics(_brokerMetrics, originalTableName);
    if (brokerResponse.getExceptionsSize() > 0) {
        _brokerMetrics.addMeteredTableValue(originalTableName, BrokerMeter.BROKER_RESPONSES_WITH_PROCESSING_EXCEPTIONS, 1);
    }
    if (numServersQueried > numServersResponded) {
        _brokerMetrics.addMeteredTableValue(originalTableName, BrokerMeter.BROKER_RESPONSES_WITH_PARTIAL_SERVERS_RESPONDED, 1);
    }
    return brokerResponse;
}

Example 3 with BrokerResponse

use of com.linkedin.pinot.common.response.BrokerResponse in project pinot by linkedin.

the class PinotClientRequestServlet method doGet.

@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
    resp.setCharacterEncoding("UTF-8");
    PrintWriter writer = resp.getWriter();
    try {
        BrokerResponse brokerResponse = broker.handleRequest(new JSONObject(req.getParameter("bql")));
        String jsonString = brokerResponse.toJsonString();
        writer.print(jsonString);
        writer.flush();
        writer.close();
    } catch (final Exception e) {
        writer.print(e.getMessage());
        writer.flush();
        writer.close();
        LOGGER.error("Caught exception while processing GET request", e);
        brokerMetrics.addMeteredGlobalValue(BrokerMeter.UNCAUGHT_GET_EXCEPTIONS, 1);
    }
}

Example 4 with BrokerResponse

use of com.linkedin.pinot.common.response.BrokerResponse in project pinot by linkedin.

the class PinotClientRequestServlet method doPost.

@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
    resp.setCharacterEncoding("UTF-8");
    PrintWriter writer = resp.getWriter();
    try {
        BrokerResponse brokerResponse = broker.handleRequest(extractJSON(req));
        String jsonString = brokerResponse.toJsonString();
        writer.print(jsonString);
        writer.flush();
        writer.close();
    } catch (final Exception e) {
        writer.print(e.getMessage());
        writer.flush();
        writer.close();
        LOGGER.error("Caught exception while processing POST request", e);
        brokerMetrics.addMeteredGlobalValue(BrokerMeter.UNCAUGHT_POST_EXCEPTIONS, 1);
    }
}