Examples with QueryDataBatch org.apache.drill.exec.rpc.user.QueryDataBatch used on opensource projects

Example 36 with QueryDataBatch

use of org.apache.drill.exec.rpc.user.QueryDataBatch in project drill by axbaretto.

the class TestSimpleFragmentRun method runJSONScanPopFragment.

@Test
public void runJSONScanPopFragment() throws Exception {
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(CONFIG, serviceSet);
        final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        // run query.
        bit.run();
        client.connect();
        final List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(DrillFileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8).replace("#{TEST_FILE}", DrillFileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString()));
        // look at records
        final RecordBatchLoader batchLoader = new RecordBatchLoader(RootAllocatorFactory.newRoot(CONFIG));
        int recordCount = 0;
        for (int i = 0; i < results.size(); ++i) {
            final QueryDataBatch batch = results.get(i);
            if (i == 0) {
                assertTrue(batch.hasData());
            } else {
                assertFalse(batch.hasData());
                batch.release();
                continue;
            }
            assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
            boolean firstColumn = true;
            // print headers.
            System.out.println("\n\n========NEW SCHEMA=========\n\n");
            for (final VectorWrapper<?> v : batchLoader) {
                if (firstColumn) {
                    firstColumn = false;
                } else {
                    System.out.print("\t");
                }
                System.out.print(v.getField().getName());
                System.out.print("[");
                System.out.print(v.getField().getType().getMinorType());
                System.out.print("]");
            }
            System.out.println();
            for (int r = 0; r < batchLoader.getRecordCount(); r++) {
                boolean first = true;
                recordCount++;
                for (final VectorWrapper<?> v : batchLoader) {
                    if (first) {
                        first = false;
                    } else {
                        System.out.print("\t");
                    }
                    final ValueVector.Accessor accessor = v.getValueVector().getAccessor();
                    System.out.print(accessor.getObject(r));
                }
                if (!first) {
                    System.out.println();
                }
            }
            batchLoader.clear();
            batch.release();
        }
        assertEquals(2, recordCount);
    }
}

Example 37 with QueryDataBatch

use of org.apache.drill.exec.rpc.user.QueryDataBatch in project drill by axbaretto.

the class TestSimpleExternalSort method validateResults.

private void validateResults(BufferAllocator allocator, List<QueryDataBatch> results) throws SchemaChangeException {
    long previousBigInt = Long.MAX_VALUE;
    int recordCount = 0;
    int batchCount = 0;
    for (QueryDataBatch b : results) {
        RecordBatchLoader loader = new RecordBatchLoader(allocator);
        if (b.getHeader().getRowCount() > 0) {
            batchCount++;
            loader.load(b.getHeader().getDef(), b.getData());
            @SuppressWarnings({ "deprecation", "resource" }) BigIntVector c1 = (BigIntVector) loader.getValueAccessorById(BigIntVector.class, loader.getValueVectorId(new SchemaPath("blue", ExpressionPosition.UNKNOWN)).getFieldIds()).getValueVector();
            BigIntVector.Accessor a1 = c1.getAccessor();
            for (int i = 0; i < c1.getAccessor().getValueCount(); i++) {
                recordCount++;
                assertTrue(String.format("%d > %d", previousBigInt, a1.get(i)), previousBigInt >= a1.get(i));
                previousBigInt = a1.get(i);
            }
        }
        loader.clear();
        b.release();
    }
    System.out.println(String.format("Sorted %,d records in %d batches.", recordCount, batchCount));
}

Example 38 with QueryDataBatch

use of org.apache.drill.exec.rpc.user.QueryDataBatch in project drill by axbaretto.

the class TestSimpleExternalSort method outOfMemoryExternalSort.

private void outOfMemoryExternalSort(boolean testLegacy) throws Throwable {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).configProperty("drill.memory.fragment.max", 50_000_000).configProperty("drill.memory.fragment.initial", 2_000_000).configProperty("drill.memory.operator.max", 30_000_000).configProperty("drill.memory.operator.initial", 2_000_000).configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, testLegacy);
    try (ClusterFixture cluster = builder.build();
        ClientFixture client = cluster.clientFixture()) {
        chooseImpl(client, testLegacy);
        List<QueryDataBatch> results = client.queryBuilder().physicalResource("/xsort/oom_sort_test.json").results();
        assertEquals(10_000_000, client.countResults(results));
        long previousBigInt = Long.MAX_VALUE;
        int recordCount = 0;
        int batchCount = 0;
        for (QueryDataBatch b : results) {
            RecordBatchLoader loader = new RecordBatchLoader(client.allocator());
            if (b.getHeader().getRowCount() > 0) {
                batchCount++;
                loader.load(b.getHeader().getDef(), b.getData());
                @SuppressWarnings("resource") BigIntVector c1 = (BigIntVector) loader.getValueAccessorById(BigIntVector.class, loader.getValueVectorId(new SchemaPath("blue", ExpressionPosition.UNKNOWN)).getFieldIds()).getValueVector();
                BigIntVector.Accessor a1 = c1.getAccessor();
                for (int i = 0; i < c1.getAccessor().getValueCount(); i++) {
                    recordCount++;
                    assertTrue(String.format("%d < %d", previousBigInt, a1.get(i)), previousBigInt >= a1.get(i));
                    previousBigInt = a1.get(i);
                }
                assertTrue(String.format("%d == %d", a1.get(0), a1.get(a1.getValueCount() - 1)), a1.get(0) != a1.get(a1.getValueCount() - 1));
            }
            loader.clear();
            b.release();
        }
        System.out.println(String.format("Sorted %,d records in %d batches.", recordCount, batchCount));
    }
}

Example 39 with QueryDataBatch

use of org.apache.drill.exec.rpc.user.QueryDataBatch in project drill by axbaretto.

the class TestSimpleExternalSort method mergeSortWithSv2.

/**
 * Tests the external sort using an in-memory sort. Relies on default memory
 * settings to be large enough to do the in-memory sort (there is,
 * unfortunately, no way to double-check that no spilling was done.)
 * This must be checked manually by setting a breakpoint in the in-memory
 * sort routine.
 *
 * @param testLegacy
 * @throws Exception
 */
private void mergeSortWithSv2(boolean testLegacy) throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false);
    try (ClusterFixture cluster = builder.build();
        ClientFixture client = cluster.clientFixture()) {
        chooseImpl(client, testLegacy);
        List<QueryDataBatch> results = client.queryBuilder().physicalResource("xsort/one_key_sort_descending_sv2.json").results();
        assertEquals(500_000, client.countResults(results));
        validateResults(client.allocator(), results);
    }
}

Example 40 with QueryDataBatch

use of org.apache.drill.exec.rpc.user.QueryDataBatch in project drill by axbaretto.

the class TestSimpleExternalSort method sortOneKeyDescendingMergeSort.

private void sortOneKeyDescendingMergeSort(boolean testLegacy) throws Throwable {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).configProperty(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED, false);
    try (ClusterFixture cluster = builder.build();
        ClientFixture client = cluster.clientFixture()) {
        chooseImpl(client, testLegacy);
        List<QueryDataBatch> results = client.queryBuilder().physicalResource("xsort/one_key_sort_descending.json").results();
        assertEquals(1_000_000, client.countResults(results));
        validateResults(client.allocator(), results);
    }
}