Examples with BrokerRequest com.linkedin.pinot.common.request.BrokerRequest used on opensource projects

Example 11 with BrokerRequest

use of com.linkedin.pinot.common.request.BrokerRequest in project pinot by linkedin.

the class BrokerRequestHandler method processBrokerRequest.

/**
   * Main method to process the request.
   * <p>Following lifecycle stages:
   * <ul>
   *   <li>1. Find the candidate servers to be queried for each set of segments from the routing table.</li>
   *   <li>2. Select servers for each segment set and scatter request to the servers.</li>
   *   <li>3. Gather responses from the servers.</li>
   *   <li>4. Deserialize the server responses.</li>
   *   <li>5. Reduce (merge) the server responses and create a broker response to be returned.</li>
   * </ul>
   *
   * @param brokerRequest broker request to be processed.
   * @param scatterGatherStats scatter-gather statistics.
   * @param requestId broker request ID.
   * @return broker response.
   * @throws InterruptedException
   */
@Nonnull
public BrokerResponse processBrokerRequest(@Nonnull BrokerRequest brokerRequest, @Nonnull ScatterGatherStats scatterGatherStats, long requestId) throws InterruptedException {
    String tableName = brokerRequest.getQuerySource().getTableName();
    ResponseType responseType = BrokerResponseFactory.getResponseType(brokerRequest.getResponseFormat());
    LOGGER.debug("Broker Response Type: {}", responseType.name());
    String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName);
    if (!_routingTable.routingTableExists(offlineTableName)) {
        offlineTableName = null;
    }
    String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName);
    if (!_routingTable.routingTableExists(realtimeTableName)) {
        realtimeTableName = null;
    }
    if ((offlineTableName == null) && (realtimeTableName == null)) {
        // No table matches the broker request.
        LOGGER.warn("No table matches the name: {}", tableName);
        _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.RESOURCE_MISSING_EXCEPTIONS, 1);
        return BrokerResponseFactory.getStaticNoTableHitBrokerResponse(responseType);
    } else {
        // At least one table matches the broker request.
        BrokerRequest offlineBrokerRequest = null;
        BrokerRequest realtimeBrokerRequest = null;
        // TODO: get time column name from schema or table config so that we can apply it in realtime only use case.
        // We get timeColumnName from time boundary service currently, which only exists for offline table.
        String timeColumnName = (offlineTableName != null) ? getTimeColumnName(offlineTableName) : null;
        if ((offlineTableName != null) && (realtimeTableName != null)) {
            // Hybrid table.
            offlineBrokerRequest = _optimizer.optimize(getOfflineBrokerRequest(brokerRequest), timeColumnName);
            realtimeBrokerRequest = _optimizer.optimize(getRealtimeBrokerRequest(brokerRequest), timeColumnName);
        } else if (offlineTableName != null) {
            // Offline table only.
            brokerRequest.getQuerySource().setTableName(offlineTableName);
            offlineBrokerRequest = _optimizer.optimize(brokerRequest, timeColumnName);
        } else {
            // Realtime table only.
            brokerRequest.getQuerySource().setTableName(realtimeTableName);
            realtimeBrokerRequest = _optimizer.optimize(brokerRequest, timeColumnName);
        }
        ReduceService reduceService = _reduceServiceRegistry.get(responseType);
        // TODO: wire up the customized BucketingSelection.
        return processOptimizedBrokerRequests(brokerRequest, offlineBrokerRequest, realtimeBrokerRequest, reduceService, scatterGatherStats, null, requestId);
    }
}

Example 12 with BrokerRequest

use of com.linkedin.pinot.common.request.BrokerRequest in project pinot by linkedin.

the class BrokerRequestHandler method handleRequest.

/**
   * Process a JSON format request.
   *
   * @param request JSON format request to be processed.
   * @return broker response.
   * @throws Exception
   */
@Nonnull
public BrokerResponse handleRequest(@Nonnull JSONObject request) throws Exception {
    long requestId = _requestIdGenerator.incrementAndGet();
    String pql = request.getString("pql");
    LOGGER.debug("Query string for requestId {}: {}", requestId, pql);
    boolean isTraceEnabled = false;
    if (request.has("trace")) {
        isTraceEnabled = Boolean.parseBoolean(request.getString("trace"));
        LOGGER.debug("Trace is set to: {} for requestId {}: {}", isTraceEnabled, requestId, pql);
    }
    Map<String, String> debugOptions = null;
    if (request.has("debugOptions")) {
        String routingOptionParameter = request.getString("debugOptions");
        debugOptions = Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('=').split(routingOptionParameter);
        LOGGER.debug("Debug options are set to: {} for requestId {}: {}", debugOptions, requestId, pql);
    }
    // Compile and validate the request.
    long compilationStartTime = System.nanoTime();
    BrokerRequest brokerRequest;
    try {
        brokerRequest = REQUEST_COMPILER.compileToBrokerRequest(pql);
    } catch (Exception e) {
        LOGGER.warn("Parsing error on requestId {}: {}", requestId, pql, e);
        _brokerMetrics.addMeteredGlobalValue(BrokerMeter.REQUEST_COMPILATION_EXCEPTIONS, 1);
        return BrokerResponseFactory.getBrokerResponseWithException(DEFAULT_BROKER_RESPONSE_TYPE, QueryException.getException(QueryException.PQL_PARSING_ERROR, e));
    }
    String tableName = brokerRequest.getQuerySource().getTableName();
    try {
        validateRequest(brokerRequest);
    } catch (Exception e) {
        LOGGER.warn("Validation error on requestId {}: {}", requestId, pql, e);
        _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.QUERY_VALIDATION_EXCEPTIONS, 1);
        return BrokerResponseFactory.getBrokerResponseWithException(DEFAULT_BROKER_RESPONSE_TYPE, QueryException.getException(QueryException.QUERY_VALIDATION_ERROR, e));
    }
    if (isTraceEnabled) {
        brokerRequest.setEnableTrace(true);
    }
    if (debugOptions != null) {
        brokerRequest.setDebugOptions(debugOptions);
    }
    brokerRequest.setResponseFormat(ResponseType.BROKER_RESPONSE_TYPE_NATIVE.name());
    _brokerMetrics.addPhaseTiming(tableName, BrokerQueryPhase.REQUEST_COMPILATION, System.nanoTime() - compilationStartTime);
    _brokerMetrics.addMeteredTableValue(tableName, BrokerMeter.QUERIES, 1);
    // Execute the query.
    long executionStartTime = System.nanoTime();
    ScatterGatherStats scatterGatherStats = new ScatterGatherStats();
    BrokerResponse brokerResponse = processBrokerRequest(brokerRequest, scatterGatherStats, requestId);
    _brokerMetrics.addPhaseTiming(tableName, BrokerQueryPhase.QUERY_EXECUTION, System.nanoTime() - executionStartTime);
    // Set total query processing time.
    long totalTimeMs = TimeUnit.MILLISECONDS.convert(System.nanoTime() - compilationStartTime, TimeUnit.NANOSECONDS);
    brokerResponse.setTimeUsedMs(totalTimeMs);
    LOGGER.debug("Broker Response: {}", brokerResponse);
    // Table name might have been changed (with suffix _OFFLINE/_REALTIME appended).
    LOGGER.info("RequestId: {}, table: {}, totalTimeMs: {}, numDocsScanned: {}, numEntriesScannedInFilter: {}, " + "numEntriesScannedPostFilter: {}, totalDocs: {}, scatterGatherStats: {}, query: {}", requestId, brokerRequest.getQuerySource().getTableName(), totalTimeMs, brokerResponse.getNumDocsScanned(), brokerResponse.getNumEntriesScannedInFilter(), brokerResponse.getNumEntriesScannedPostFilter(), brokerResponse.getTotalDocs(), scatterGatherStats, pql);
    return brokerResponse;
}

Example 13 with BrokerRequest

use of com.linkedin.pinot.common.request.BrokerRequest in project pinot by linkedin.

the class OptimizationFlagsTest method testDisableOptimizations.

@Test
public void testDisableOptimizations() {
    BrokerRequest brokerRequest = new BrokerRequest();
    brokerRequest.setDebugOptions(Collections.singletonMap("optimizationFlags", "-foo"));
    OptimizationFlags optimizationFlags = OptimizationFlags.getOptimizationFlags(brokerRequest);
    Assert.assertFalse(optimizationFlags.isOptimizationEnabled("foo"));
    Assert.assertTrue(optimizationFlags.isOptimizationEnabled("bar"));
    Assert.assertTrue(optimizationFlags.isOptimizationEnabled("baz"));
}

Example 14 with BrokerRequest

use of com.linkedin.pinot.common.request.BrokerRequest in project pinot by linkedin.

the class RangeMergeOptimizerTest method buildFilterQueryTree.

/**
   * Helper method to build the filter query tree for the given query
   * @param query Query for which to build the filter query tree
   * @param optimize If true, then range optimization is performed
   * @return Filter query tree for the query
   */
private FilterQueryTree buildFilterQueryTree(String query, boolean optimize) {
    BrokerRequest brokerRequest = _compiler.compileToBrokerRequest(query);
    FilterQueryTree filterQueryTree = RequestUtils.generateFilterQueryTree(brokerRequest);
    if (optimize) {
        FilterQueryOptimizerRequest request = _builder.setFilterQueryTree(filterQueryTree).setTimeColumn(TIME_COLUMN).build();
        return _optimizer.optimize(request);
    } else {
        return filterQueryTree;
    }
}

Example 15 with BrokerRequest

use of com.linkedin.pinot.common.request.BrokerRequest in project pinot by linkedin.

the class Pql2Compiler method compileToBrokerRequest.

@Override
public BrokerRequest compileToBrokerRequest(String expression) throws Pql2CompilationException {
    try {
        //
        CharStream charStream = new ANTLRInputStream(expression);
        PQL2Lexer lexer = new PQL2Lexer(charStream);
        lexer.setTokenFactory(new CommonTokenFactory(true));
        TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
        PQL2Parser parser = new PQL2Parser(tokenStream);
        parser.setErrorHandler(new BailErrorStrategy());
        // Parse
        ParseTree parseTree = parser.root();
        ParseTreeWalker walker = new ParseTreeWalker();
        Pql2AstListener listener = new Pql2AstListener(expression);
        walker.walk(listener, parseTree);
        AstNode rootNode = listener.getRootNode();
        BrokerRequest brokerRequest = new BrokerRequest();
        rootNode.updateBrokerRequest(brokerRequest);
        return brokerRequest;
    } catch (Pql2CompilationException e) {
        throw e;
    } catch (Exception e) {
        throw new Pql2CompilationException(e.getMessage());
    }
}