Examples with QueryRequest com.linkedin.pinot.common.query.QueryRequest used on opensource projects

Example 31 with QueryRequest

use of com.linkedin.pinot.common.query.QueryRequest in project pinot by linkedin.

the class RealtimeQueriesSentinelTest method testAggregation.

@Test
public void testAggregation() throws Exception {
    int counter = 0;
    final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    final List<TestSimpleAggreationQuery> aggCalls = AVRO_QUERY_GENERATOR.giveMeNSimpleAggregationQueries(10000);
    for (final TestSimpleAggreationQuery aggCall : aggCalls) {
        LOGGER.info("running " + counter + " : " + aggCall.pql);
        final BrokerRequest brokerRequest = REQUEST_COMPILER.compileToBrokerRequest(aggCall.pql);
        InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
        instanceRequest.setSearchSegments(new ArrayList<String>());
        instanceRequest.getSearchSegments().add("testTable_testTable");
        QueryRequest queryRequest = new QueryRequest(instanceRequest, TableDataManagerProvider.getServerMetrics());
        DataTable instanceResponse = QUERY_EXECUTOR.processQuery(queryRequest, queryRunners);
        instanceResponseMap.clear();
        instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
        final BrokerResponseNative brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
        LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
        LOGGER.info("Result from avro is : " + aggCall.result);
        Double actual = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getValue().toString());
        Double expected = aggCall.result;
        try {
            Assert.assertEquals(actual, expected);
        } catch (AssertionError e) {
            System.out.println("********************************");
            System.out.println("query : " + aggCall.pql);
            System.out.println("actual : " + actual);
            System.out.println("expected : " + aggCall.result);
            System.out.println("********************************");
            throw e;
        }
    }
}

Example 32 with QueryRequest

use of com.linkedin.pinot.common.query.QueryRequest in project pinot by linkedin.

the class ScheduledRequestHandler method processRequest.

@Override
public ListenableFuture<byte[]> processRequest(ChannelHandlerContext channelHandlerContext, ByteBuf request) {
    final long queryStartTimeNs = System.nanoTime();
    serverMetrics.addMeteredGlobalValue(ServerMeter.QUERIES, 1);
    LOGGER.debug("Processing request : {}", request);
    byte[] byteArray = new byte[request.readableBytes()];
    request.readBytes(byteArray);
    SerDe serDe = new SerDe(new TCompactProtocol.Factory());
    final InstanceRequest instanceRequest = new InstanceRequest();
    if (!serDe.deserialize(instanceRequest, byteArray)) {
        LOGGER.error("Failed to deserialize query request from broker ip: {}", ((InetSocketAddress) channelHandlerContext.channel().remoteAddress()).getAddress().getHostAddress());
        DataTable result = new DataTableImplV2();
        result.addException(QueryException.INTERNAL_ERROR);
        serverMetrics.addMeteredGlobalValue(ServerMeter.REQUEST_DESERIALIZATION_EXCEPTIONS, 1);
        QueryRequest queryRequest = new QueryRequest(null, serverMetrics);
        queryRequest.getTimerContext().setQueryArrivalTimeNs(queryStartTimeNs);
        return Futures.immediateFuture(serializeDataTable(queryRequest, result));
    }
    final QueryRequest queryRequest = new QueryRequest(instanceRequest, serverMetrics);
    final TimerContext timerContext = queryRequest.getTimerContext();
    timerContext.setQueryArrivalTimeNs(queryStartTimeNs);
    TimerContext.Timer deserializationTimer = timerContext.startNewPhaseTimerAtNs(ServerQueryPhase.REQUEST_DESERIALIZATION, queryStartTimeNs);
    deserializationTimer.stopAndRecord();
    LOGGER.debug("Processing requestId:{},request={}", instanceRequest.getRequestId(), instanceRequest);
    ListenableFuture<DataTable> queryTask = queryScheduler.submit(queryRequest);
    // following future will provide default response in case of uncaught
    // exceptions from query processing
    ListenableFuture<DataTable> queryResponse = Futures.catching(queryTask, Throwable.class, new Function<Throwable, DataTable>() {

        @Nullable
        @Override
        public DataTable apply(@Nullable Throwable input) {
            // this is called iff queryTask fails with unhandled exception
            serverMetrics.addMeteredGlobalValue(ServerMeter.UNCAUGHT_EXCEPTIONS, 1);
            DataTable result = new DataTableImplV2();
            result.addException(QueryException.INTERNAL_ERROR);
            return result;
        }
    });
    // transform the DataTable to serialized byte[] to send back to broker
    ListenableFuture<byte[]> serializedQueryResponse = Futures.transform(queryResponse, new Function<DataTable, byte[]>() {

        @Nullable
        @Override
        public byte[] apply(@Nullable DataTable instanceResponse) {
            byte[] responseData = serializeDataTable(queryRequest, instanceResponse);
            LOGGER.info("Processed requestId {},reqSegments={},prunedToSegmentCount={},deserTimeMs={},planTimeMs={},planExecTimeMs={},totalExecMs={},serTimeMs={}TotalTimeMs={},broker={}", queryRequest.getInstanceRequest().getRequestId(), queryRequest.getInstanceRequest().getSearchSegments().size(), queryRequest.getSegmentCountAfterPruning(), timerContext.getPhaseDurationMs(ServerQueryPhase.REQUEST_DESERIALIZATION), timerContext.getPhaseDurationMs(ServerQueryPhase.BUILD_QUERY_PLAN), timerContext.getPhaseDurationMs(ServerQueryPhase.QUERY_PLAN_EXECUTION), timerContext.getPhaseDurationMs(ServerQueryPhase.QUERY_PROCESSING), timerContext.getPhaseDurationMs(ServerQueryPhase.RESPONSE_SERIALIZATION), timerContext.getPhaseDurationMs(ServerQueryPhase.TOTAL_QUERY_TIME), queryRequest.getBrokerId());
            return responseData;
        }
    });
    return serializedQueryResponse;
}

Example 33 with QueryRequest

use of com.linkedin.pinot.common.query.QueryRequest in project pinot by linkedin.

the class ScheduledRequestHandlerTest method testQueryProcessingException.

@Test
public void testQueryProcessingException() throws Exception {
    ScheduledRequestHandler handler = new ScheduledRequestHandler(new QueryScheduler(queryExecutor) {

        @Override
        public ListenableFuture<DataTable> submit(QueryRequest queryRequest) {
            return queryWorkers.submit(new Callable<DataTable>() {

                @Override
                public DataTable call() throws Exception {
                    throw new RuntimeException("query processing error");
                }
            });
        }
    }, serverMetrics);
    ByteBuf requestBuf = getSerializedInstanceRequest(getInstanceRequest());
    ListenableFuture<byte[]> responseFuture = handler.processRequest(channelHandlerContext, requestBuf);
    byte[] bytes = responseFuture.get(2, TimeUnit.SECONDS);
    // we get DataTable with exception information in case of query processing exception
    Assert.assertTrue(bytes.length > 0);
    DataTable expectedDT = new DataTableImplV2();
    expectedDT.addException(QueryException.INTERNAL_ERROR);
    Assert.assertEquals(bytes, expectedDT.toBytes());
}

Example 34 with QueryRequest

use of com.linkedin.pinot.common.query.QueryRequest in project pinot by linkedin.

the class ScheduledRequestHandlerTest method testValidQueryResponse.

@Test
public void testValidQueryResponse() throws InterruptedException, ExecutionException, TimeoutException, IOException {
    ScheduledRequestHandler handler = new ScheduledRequestHandler(new QueryScheduler(queryExecutor) {

        @Override
        public ListenableFuture<DataTable> submit(QueryRequest queryRequest) {
            return queryRunners.submit(new Callable<DataTable>() {

                @Override
                public DataTable call() throws Exception {
                    String[] columns = new String[] { "foo", "bar" };
                    FieldSpec.DataType[] columnTypes = new FieldSpec.DataType[] { FieldSpec.DataType.STRING, FieldSpec.DataType.INT };
                    DataSchema dataSchema = new DataSchema(columns, columnTypes);
                    DataTableBuilder dtBuilder = new DataTableBuilder(dataSchema);
                    dtBuilder.startRow();
                    dtBuilder.setColumn(0, "mars");
                    dtBuilder.setColumn(1, 10);
                    dtBuilder.finishRow();
                    dtBuilder.startRow();
                    dtBuilder.setColumn(0, "jupiter");
                    dtBuilder.setColumn(1, 100);
                    dtBuilder.finishRow();
                    return dtBuilder.build();
                }
            });
        }
    }, serverMetrics);
    ByteBuf requestBuf = getSerializedInstanceRequest(getInstanceRequest());
    ListenableFuture<byte[]> responseFuture = handler.processRequest(channelHandlerContext, requestBuf);
    byte[] responseBytes = responseFuture.get(2, TimeUnit.SECONDS);
    DataTable responseDT = DataTableFactory.getDataTable(responseBytes);
    Assert.assertEquals(responseDT.getNumberOfRows(), 2);
    Assert.assertEquals(responseDT.getString(0, 0), "mars");
    Assert.assertEquals(responseDT.getInt(0, 1), 10);
    Assert.assertEquals(responseDT.getString(1, 0), "jupiter");
    Assert.assertEquals(responseDT.getInt(1, 1), 100);
}

Example 35 with QueryRequest

use of com.linkedin.pinot.common.query.QueryRequest in project pinot by linkedin.

the class IntegrationTest method testMaxQuery.

@Test
public void testMaxQuery() {
    BrokerRequest brokerRequest = getMaxQuery();
    QuerySource querySource = new QuerySource();
    querySource.setTableName("testTable");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    addTestTableSearchSegmentsToInstanceRequest(instanceRequest);
    try {
        QueryRequest queryRequest = new QueryRequest(instanceRequest, _serverInstance.getServerMetrics());
        DataTable instanceResponse = _queryExecutor.processQuery(queryRequest, queryRunners);
    //      System.out.println(instanceResponse.getDouble(0, 0));
    //      System.out.println(instanceResponse.getMetadata().get(DataTable.TIME_USED_MS_METADATA_KEY));
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}