Examples with ScatterGatherStats com.linkedin.pinot.transport.scattergather.ScatterGatherStats used on opensource projects

Example 1 with ScatterGatherStats

use of com.linkedin.pinot.transport.scattergather.ScatterGatherStats in project pinot by linkedin.

the class BrokerRequestHandler method handleRequest.

/**
   * Process a JSON format request.
   *
   * @param request JSON format request to be processed.
   * @return broker response.
   * @throws Exception
   */
@Nonnull
public BrokerResponse handleRequest(@Nonnull JSONObject request) throws Exception {
    long requestId = _requestIdGenerator.incrementAndGet();
    String pql = request.getString("pql");
    LOGGER.debug("Query string for requestId {}: {}", requestId, pql);
    boolean isTraceEnabled = false;
    if (request.has("trace")) {
        isTraceEnabled = Boolean.parseBoolean(request.getString("trace"));
        LOGGER.debug("Trace is set to: {} for requestId {}: {}", isTraceEnabled, requestId, pql);
    }
    Map<String, String> debugOptions = null;
    if (request.has("debugOptions")) {
        String routingOptionParameter = request.getString("debugOptions");
        debugOptions = Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('=').split(routingOptionParameter);
        LOGGER.debug("Debug options are set to: {} for requestId {}: {}", debugOptions, requestId, pql);
    }
    // Compile and validate the request.
    long compilationStartTime = System.nanoTime();
    BrokerRequest brokerRequest;
    try {
        brokerRequest = REQUEST_COMPILER.compileToBrokerRequest(pql);
    } catch (Exception e) {
        LOGGER.warn("Parsing error on requestId {}: {}", requestId, pql, e);
        _brokerMetrics.addMeteredGlobalValue(BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, 1);
        return BrokerResponseFactory.getBrokerResponseWithException(DEFAULT_BROKER_RESPONSE_TYPE, QueryException.getException(QueryException.PQL_PARSING_ERROR, e));
    }
    String tableName = brokerRequest.getQuerySource().getTableName();
    try {
        validateRequest(brokerRequest);
    } catch (Exception e) {
        LOGGER.warn("Validation error on requestId {}: {}", requestId, pql, e);
        _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.QUERY_VALIDATION_EXCEPTIONS, 1);
        return BrokerResponseFactory.getBrokerResponseWithException(DEFAULT_BROKER_RESPONSE_TYPE, QueryException.getException(QueryException.QUERY_VALIDATION_ERROR, e));
    }
    if (isTraceEnabled) {
        brokerRequest.setEnableTrace(true);
    }
    if (debugOptions != null) {
        brokerRequest.setDebugOptions(debugOptions);
    }
    brokerRequest.setResponseFormat(ResponseType.BROKER_RESPONSE_TYPE_NATIVE.name());
    _brokerMetrics.addPhaseTiming(tableName, BrokerQueryPhase.REQUEST_COMPILATION, System.nanoTime() - compilationStartTime);
    _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.QUERIES, 1);
    // Execute the query.
    long executionStartTime = System.nanoTime();
    ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerResponse brokerResponse = processBrokerRequest(brokerRequest, scatterGatherStats, requestId);
    _brokerMetrics.addPhaseTiming(tableName, BrokerQueryPhase.QUERY_EXECUTION, System.nanoTime() - executionStartTime);
    // Set total query processing time.
    long totalTimeMs = TimeUnit.MILLISECONDS.convert(System.nanoTime() - compilationStartTime, TimeUnit.NANOSECONDS);
    brokerResponse.setTimeUsedMs(totalTimeMs);
    LOGGER.debug("Broker Response: {}", brokerResponse);
    // Table name might have been changed (with suffix _OFFLINE/_REALTIME appended).
    LOGGER.info("RequestId: {}, table: {}, totalTimeMs: {}, numDocsScanned: {}, numEntriesScannedInFilter: {}, " + "numEntriesScannedPostFilter: {}, totalDocs: {}, scatterGatherStats: {}, query: {}", requestId, brokerRequest.getQuerySource().getTableName(), totalTimeMs, brokerResponse.getNumDocsScanned(), brokerResponse.getNumEntriesScannedInFilter(), brokerResponse.getNumEntriesScannedPostFilter(), brokerResponse.getTotalDocs(), scatterGatherStats, pql);
    return brokerResponse;
}

Example 2 with ScatterGatherStats

use of com.linkedin.pinot.transport.scattergather.ScatterGatherStats in project pinot by linkedin.

the class ScatterGatherPerfClient method run.

@Override
public void run() {
    System.out.println("Client starting !!");
    try {
        List<ServerInstance> s1 = new ArrayList<ServerInstance>();
        ServerInstance s = new ServerInstance("localhost", 9099);
        s1.add(s);
        SimpleScatterGatherRequest req = null;
        TimerContext tc = null;
        for (int i = 0; i < _numRequests; i++) {
            LOGGER.debug("Sending request number {}", i);
            do {
                req = getRequest();
            } while ((null == req));
            if (i == _numRequestsToSkipForMeasurement) {
                tc = MetricsHelper.startTimer();
                _beginFirstRequestTime = System.currentTimeMillis();
            }
            if (i >= _numRequestsToSkipForMeasurement) {
                _numRequestsMeasured++;
            }
            final ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
            if (!_asyncRequestSubmit) {
                sendRequestAndGetResponse(req, scatterGatherStats);
                _endLastResponseTime = System.currentTimeMillis();
            } else {
                CompositeFuture<ServerInstance, ByteBuf> future = asyncSendRequestAndGetResponse(req, scatterGatherStats);
                _queue.offer(new QueueEntry(false, i >= _numRequestsToSkipForMeasurement, System.currentTimeMillis(), future));
            }
            //System.out.println("Response is :" + r);
            //System.out.println("\n\n");
            req = null;
        }
        if (_asyncRequestSubmit) {
            int numTerminalEntries = _readerThreads.size();
            for (int i = 0; i < numTerminalEntries; i++) {
                _queue.offer(new QueueEntry(true, false, System.currentTimeMillis(), null));
            }
            for (AsyncReader r : _readerThreads) {
                r.join();
            }
        }
        if (null != tc) {
            tc.stop();
            _timerContext = tc;
            System.out.println("Num Requests :" + _numRequestsMeasured);
            System.out.println("Total time :" + tc.getLatencyMs());
            System.out.println("Throughput (Requests/Second) :" + ((_numRequestsMeasured * 1.0 * 1000) / tc.getLatencyMs()));
            System.out.println("Latency :" + new LatencyMetric<Histogram>(_latencyHistogram));
            System.out.println("Scatter-Gather Latency :" + new LatencyMetric<Histogram>(_scatterGather.getLatency()));
        }
    } catch (Exception ex) {
        System.err.println("Client stopped abnormally ");
        ex.printStackTrace();
    }
    shutdown();
    System.out.println("Client done !!");
}