Search in sources :

Example 6 with QueryData

use of org.apache.drill.exec.proto.UserBitShared.QueryData in project drill by apache.

the class TestMergingReceiver method twoBitTwoExchange.

// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestMergingReceiver.class);
@Test
public void twoBitTwoExchange() throws Exception {
    final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
        final Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
        final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        bit1.run();
        bit2.run();
        client.connect();
        final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/mergerecv/merging_receiver.json"), Charsets.UTF_8).read());
        int count = 0;
        final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
        // print the results
        for (final QueryDataBatch b : results) {
            final QueryData queryData = b.getHeader();
            final int rowCount = queryData.getRowCount();
            count += rowCount;
            // loaded but not used, just to test
            batchLoader.load(queryData.getDef(), b.getData());
            b.release();
            batchLoader.clear();
        }
        assertEquals(200000, count);
    }
}
Also used : QueryDataBatch(org.apache.drill.exec.rpc.user.QueryDataBatch) Drillbit(org.apache.drill.exec.server.Drillbit) QueryData(org.apache.drill.exec.proto.UserBitShared.QueryData) RemoteServiceSet(org.apache.drill.exec.server.RemoteServiceSet) RecordBatchLoader(org.apache.drill.exec.record.RecordBatchLoader) DrillClient(org.apache.drill.exec.client.DrillClient) OperatorTest(org.apache.drill.categories.OperatorTest) Test(org.junit.Test) SlowTest(org.apache.drill.categories.SlowTest)

Example 7 with QueryData

use of org.apache.drill.exec.proto.UserBitShared.QueryData in project drill by apache.

the class TestMergingReceiver method testMultipleProvidersEmptyBatches.

@Test
public void testMultipleProvidersEmptyBatches() throws Exception {
    final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
        final Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
        final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        bit1.run();
        bit2.run();
        client.connect();
        final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/mergerecv/multiple_providers_empty_batches.json"), Charsets.UTF_8).read());
        int count = 0;
        final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
        // print the results
        Long lastBlueValue = null;
        for (final QueryDataBatch b : results) {
            final QueryData queryData = b.getHeader();
            final int batchRowCount = queryData.getRowCount();
            count += batchRowCount;
            batchLoader.load(queryData.getDef(), b.getData());
            for (final VectorWrapper<?> vw : batchLoader) {
                final ValueVector vv = vw.getValueVector();
                final ValueVector.Accessor va = vv.getAccessor();
                final MaterializedField materializedField = vv.getField();
                final int numValues = va.getValueCount();
                for (int valueIdx = 0; valueIdx < numValues; ++valueIdx) {
                    if (materializedField.getName().equals("blue")) {
                        final long longValue = (Long) va.getObject(valueIdx);
                        // check that order is ascending
                        if (lastBlueValue != null) {
                            assertTrue(longValue >= lastBlueValue);
                        }
                        lastBlueValue = longValue;
                    }
                }
            }
            b.release();
            batchLoader.clear();
        }
        assertEquals(300000, count);
    }
}
Also used : QueryData(org.apache.drill.exec.proto.UserBitShared.QueryData) RecordBatchLoader(org.apache.drill.exec.record.RecordBatchLoader) MaterializedField(org.apache.drill.exec.record.MaterializedField) ValueVector(org.apache.drill.exec.vector.ValueVector) QueryDataBatch(org.apache.drill.exec.rpc.user.QueryDataBatch) Drillbit(org.apache.drill.exec.server.Drillbit) RemoteServiceSet(org.apache.drill.exec.server.RemoteServiceSet) DrillClient(org.apache.drill.exec.client.DrillClient) OperatorTest(org.apache.drill.categories.OperatorTest) Test(org.junit.Test) SlowTest(org.apache.drill.categories.SlowTest)

Example 8 with QueryData

use of org.apache.drill.exec.proto.UserBitShared.QueryData in project drill by apache.

the class TestMergingReceiver method handleEmptyBatchNoSchema.

@Test
public void handleEmptyBatchNoSchema() throws Exception {
    final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
        final Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
        final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        bit1.run();
        bit2.run();
        client.connect();
        final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/mergerecv/empty_batch_noschema.json"), Charsets.UTF_8).read());
        int count = 0;
        final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
        // print the results
        for (final QueryDataBatch b : results) {
            final QueryData queryData = b.getHeader();
            // loaded but not used, for testing
            batchLoader.load(queryData.getDef(), b.getData());
            count += queryData.getRowCount();
            b.release();
            batchLoader.clear();
        }
        assertEquals(100000, count);
    }
}
Also used : QueryDataBatch(org.apache.drill.exec.rpc.user.QueryDataBatch) Drillbit(org.apache.drill.exec.server.Drillbit) QueryData(org.apache.drill.exec.proto.UserBitShared.QueryData) RemoteServiceSet(org.apache.drill.exec.server.RemoteServiceSet) RecordBatchLoader(org.apache.drill.exec.record.RecordBatchLoader) DrillClient(org.apache.drill.exec.client.DrillClient) OperatorTest(org.apache.drill.categories.OperatorTest) Test(org.junit.Test) SlowTest(org.apache.drill.categories.SlowTest)

Example 9 with QueryData

use of org.apache.drill.exec.proto.UserBitShared.QueryData in project drill by apache.

the class TestMergingReceiver method testMultipleProvidersMixedSizes.

@Test
public void testMultipleProvidersMixedSizes() throws Exception {
    final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
        final Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
        final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        bit1.run();
        bit2.run();
        client.connect();
        final List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/mergerecv/multiple_providers.json"), Charsets.UTF_8).read());
        int count = 0;
        final RecordBatchLoader batchLoader = new RecordBatchLoader(client.getAllocator());
        // print the results
        Long lastBlueValue = null;
        for (final QueryDataBatch b : results) {
            final QueryData queryData = b.getHeader();
            final int batchRowCount = queryData.getRowCount();
            count += batchRowCount;
            batchLoader.load(queryData.getDef(), b.getData());
            for (final VectorWrapper<?> vw : batchLoader) {
                final ValueVector vv = vw.getValueVector();
                final ValueVector.Accessor va = vv.getAccessor();
                final MaterializedField materializedField = vv.getField();
                final int numValues = va.getValueCount();
                for (int valueIdx = 0; valueIdx < numValues; ++valueIdx) {
                    if (materializedField.getName().equals("blue")) {
                        final long longValue = (Long) va.getObject(valueIdx);
                        // check that order is ascending
                        if (lastBlueValue != null) {
                            assertTrue(longValue >= lastBlueValue);
                        }
                        lastBlueValue = longValue;
                    }
                }
            }
            b.release();
            batchLoader.clear();
        }
        assertEquals(400000, count);
    }
}
Also used : QueryData(org.apache.drill.exec.proto.UserBitShared.QueryData) RecordBatchLoader(org.apache.drill.exec.record.RecordBatchLoader) MaterializedField(org.apache.drill.exec.record.MaterializedField) ValueVector(org.apache.drill.exec.vector.ValueVector) QueryDataBatch(org.apache.drill.exec.rpc.user.QueryDataBatch) Drillbit(org.apache.drill.exec.server.Drillbit) RemoteServiceSet(org.apache.drill.exec.server.RemoteServiceSet) DrillClient(org.apache.drill.exec.client.DrillClient) OperatorTest(org.apache.drill.categories.OperatorTest) Test(org.junit.Test) SlowTest(org.apache.drill.categories.SlowTest)

Example 10 with QueryData

use of org.apache.drill.exec.proto.UserBitShared.QueryData in project drill by apache.

the class QueryResultHandler method batchArrived.

/**
 * Maps internal low-level API protocol to {@link UserResultsListener}-level API protocol.
 * handles query data messages
 */
public void batchArrived(ConnectionThrottle throttle, ByteBuf pBody, ByteBuf dBody) throws RpcException {
    QueryData queryData = RpcBus.get(pBody, QueryData.PARSER);
    // Current batch coming in.
    DrillBuf drillBuf = (DrillBuf) dBody;
    QueryDataBatch batch = new QueryDataBatch(queryData, drillBuf);
    QueryId queryId = queryData.getQueryId();
    if (logger.isDebugEnabled()) {
        logger.debug("batchArrived: queryId = {}", QueryIdHelper.getQueryId(queryId));
    }
    logger.trace("batchArrived: batch = {}", batch);
    UserResultsListener resultsListener = newUserResultsListener(queryId);
    // A data case--pass on via dataArrived
    try {
        resultsListener.dataArrived(batch, throttle);
    // That releases batch if successful.
    } catch (Exception e) {
        batch.release();
        resultsListener.submissionFailed(UserException.systemError(e).build(logger));
    }
}
Also used : QueryData(org.apache.drill.exec.proto.UserBitShared.QueryData) QueryId(org.apache.drill.exec.proto.UserBitShared.QueryId) UserException(org.apache.drill.common.exceptions.UserException) RpcException(org.apache.drill.exec.rpc.RpcException) UserRemoteException(org.apache.drill.common.exceptions.UserRemoteException) DrillBuf(io.netty.buffer.DrillBuf)

Aggregations

QueryData (org.apache.drill.exec.proto.UserBitShared.QueryData)20 RecordBatchLoader (org.apache.drill.exec.record.RecordBatchLoader)12 QueryDataBatch (org.apache.drill.exec.rpc.user.QueryDataBatch)12 OperatorTest (org.apache.drill.categories.OperatorTest)10 DrillClient (org.apache.drill.exec.client.DrillClient)10 Drillbit (org.apache.drill.exec.server.Drillbit)10 RemoteServiceSet (org.apache.drill.exec.server.RemoteServiceSet)10 Test (org.junit.Test)10 MaterializedField (org.apache.drill.exec.record.MaterializedField)6 DrillBuf (io.netty.buffer.DrillBuf)5 SlowTest (org.apache.drill.categories.SlowTest)5 UserException (org.apache.drill.common.exceptions.UserException)4 RpcException (org.apache.drill.exec.rpc.RpcException)4 ValueVector (org.apache.drill.exec.vector.ValueVector)4 SchemaChangeException (org.apache.drill.exec.exception.SchemaChangeException)3 IOException (java.io.IOException)2 SingleRowListener (org.apache.drill.SingleRowListener)2 UserRemoteException (org.apache.drill.common.exceptions.UserRemoteException)2 MinorType (org.apache.drill.common.types.TypeProtos.MinorType)2 BufferAllocator (org.apache.drill.exec.memory.BufferAllocator)2