Examples with Drillbit org.apache.drill.exec.server.Drillbit used on opensource projects

Example 61 with Drillbit

use of org.apache.drill.exec.server.Drillbit in project drill by apache.

the class TestAllocators method testAllocators.

@Test
public void testAllocators() throws Exception {
    // Setup a drillbit (initializes a root allocator)
    final DrillConfig config = DrillConfig.create(TEST_CONFIGURATIONS);
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(config, serviceSet)) {
        bit.run();
        final DrillbitContext bitContext = bit.getContext();
        FunctionImplementationRegistry functionRegistry = bitContext.getFunctionImplementationRegistry();
        StoragePluginRegistry storageRegistry = new StoragePluginRegistryImpl(bitContext);
        // Create a few Fragment Contexts
        BitControl.PlanFragment.Builder pfBuilder1 = BitControl.PlanFragment.newBuilder();
        pfBuilder1.setMemInitial(1500000);
        BitControl.PlanFragment pf1 = pfBuilder1.build();
        BitControl.PlanFragment.Builder pfBuilder2 = BitControl.PlanFragment.newBuilder();
        pfBuilder2.setMemInitial(500000);
        BitControl.PlanFragment pf2 = pfBuilder1.build();
        FragmentContextImpl fragmentContext1 = new FragmentContextImpl(bitContext, pf1, null, functionRegistry);
        FragmentContextImpl fragmentContext2 = new FragmentContextImpl(bitContext, pf2, null, functionRegistry);
        // Get a few physical operators. Easiest way is to read a physical plan.
        PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext, storageRegistry);
        PhysicalPlan plan = planReader.readPhysicalPlan(Files.asCharSource(DrillFileUtils.getResourceAsFile(planFile), Charsets.UTF_8).read());
        List<PhysicalOperator> physicalOperators = plan.getSortedOperators();
        Iterator<PhysicalOperator> physicalOperatorIterator = physicalOperators.iterator();
        PhysicalOperator physicalOperator1 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator2 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator3 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator4 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator5 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator6 = physicalOperatorIterator.next();
        // Create some bogus Operator profile defs and stats to create operator contexts
        OpProfileDef def;
        OperatorStats stats;
        // Use some bogus operator type to create a new operator context.
        def = new OpProfileDef(physicalOperator1.getOperatorId(), MockSubScanPOP.OPERATOR_TYPE, OperatorUtilities.getChildCount(physicalOperator1));
        stats = fragmentContext1.getStats().newOperatorStats(def, fragmentContext1.getAllocator());
        // Add a couple of Operator Contexts
        // Initial allocation = 1000000 bytes for all operators
        OperatorContext oContext11 = fragmentContext1.newOperatorContext(physicalOperator1);
        DrillBuf b11 = oContext11.getAllocator().buffer(1000000);
        OperatorContext oContext12 = fragmentContext1.newOperatorContext(physicalOperator2, stats);
        DrillBuf b12 = oContext12.getAllocator().buffer(500000);
        OperatorContext oContext21 = fragmentContext1.newOperatorContext(physicalOperator3);
        def = new OpProfileDef(physicalOperator4.getOperatorId(), TextFormatPlugin.WRITER_OPERATOR_TYPE, OperatorUtilities.getChildCount(physicalOperator4));
        stats = fragmentContext2.getStats().newOperatorStats(def, fragmentContext2.getAllocator());
        OperatorContext oContext22 = fragmentContext2.newOperatorContext(physicalOperator4, stats);
        DrillBuf b22 = oContext22.getAllocator().buffer(2000000);
        // New Fragment begins
        BitControl.PlanFragment.Builder pfBuilder3 = BitControl.PlanFragment.newBuilder();
        pfBuilder3.setMemInitial(1000000);
        BitControl.PlanFragment pf3 = pfBuilder3.build();
        FragmentContextImpl fragmentContext3 = new FragmentContextImpl(bitContext, pf3, null, functionRegistry);
        // New fragment starts an operator that allocates an amount within the limit
        def = new OpProfileDef(physicalOperator5.getOperatorId(), UnionAll.OPERATOR_TYPE, OperatorUtilities.getChildCount(physicalOperator5));
        stats = fragmentContext3.getStats().newOperatorStats(def, fragmentContext3.getAllocator());
        OperatorContext oContext31 = fragmentContext3.newOperatorContext(physicalOperator5, stats);
        DrillBuf b31a = oContext31.getAllocator().buffer(200000);
        // Previously running operator completes
        b22.release();
        ((AutoCloseable) oContext22).close();
        // Fragment 3 asks for more and fails
        boolean outOfMem = false;
        try {
            oContext31.getAllocator().buffer(44000000);
            fail("Fragment 3 should fail to allocate buffer");
        } catch (OutOfMemoryException e) {
            // Expected.
            outOfMem = true;
        }
        assertTrue(outOfMem);
        // Operator is Exempt from Fragment limits. Fragment 3 asks for more and succeeds
        OperatorContext oContext32 = fragmentContext3.newOperatorContext(physicalOperator6);
        try {
            DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
            b32.release();
        } catch (OutOfMemoryException e) {
            fail("Fragment 3 failed to allocate buffer");
        } finally {
            closeOp(oContext32);
        }
        b11.release();
        closeOp(oContext11);
        b12.release();
        closeOp(oContext12);
        closeOp(oContext21);
        b31a.release();
        closeOp(oContext31);
        fragmentContext1.close();
        fragmentContext2.close();
        fragmentContext3.close();
    }
}

Example 62 with Drillbit

use of org.apache.drill.exec.server.Drillbit in project drill by apache.

the class TestDistributedFragmentRun method oneBitOneExchangeOneEntryRun.

@Test
public void oneBitOneExchangeOneEntryRun() throws Exception {
    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
        DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        bit1.run();
        client.connect();
        List<QueryDataBatch> results = client.runQuery(QueryType.PHYSICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/physical_single_exchange.json"), Charsets.UTF_8).read());
        int count = 0;
        for (QueryDataBatch b : results) {
            count += b.getHeader().getRowCount();
            b.release();
        }
        assertEquals(100, count);
    }
}

Example 63 with Drillbit

use of org.apache.drill.exec.server.Drillbit in project drill by apache.

the class TestDistributedFragmentRun method oneBitOneExchangeTwoEntryRunLogical.

@Test
public void oneBitOneExchangeTwoEntryRunLogical() throws Exception {
    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
        DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        bit1.run();
        client.connect();
        List<QueryDataBatch> results = client.runQuery(QueryType.LOGICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/scan_screen_logical.json"), Charsets.UTF_8).read());
        int count = 0;
        for (QueryDataBatch b : results) {
            count += b.getHeader().getRowCount();
            b.release();
        }
        assertEquals(100, count);
    }
}

Example 64 with Drillbit

use of org.apache.drill.exec.server.Drillbit in project drill by apache.

the class TestHashToRandomExchange method twoBitTwoExchangeTwoEntryRun.

@Test
public void twoBitTwoExchangeTwoEntryRun() throws Exception {
    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
    try (Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
        Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
        DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        bit1.run();
        bit2.run();
        client.connect();
        List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/sender/hash_exchange.json"), Charsets.UTF_8).read());
        int count = 0;
        for (QueryDataBatch b : results) {
            if (b.getHeader().getRowCount() != 0) {
                count += b.getHeader().getRowCount();
            }
            b.release();
        }
        assertEquals(200, count);
    }
}

Example 65 with Drillbit

use of org.apache.drill.exec.server.Drillbit in project drill by apache.

the class TestDecimal method testSimpleDecimal.

@Test
public void testSimpleDecimal() throws Exception {
    /* Function checks casting from VarChar to Decimal9, Decimal18 and vice versa
         * Also tests instances where the scale might have to truncated when scale provided < input fraction
         */
    try (RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        Drillbit bit = new Drillbit(CONFIG, serviceSet);
        DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
        // run query.
        bit.run();
        client.connect();
        List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Files.asCharSource(DrillFileUtils.getResourceAsFile("/decimal/cast_simple_decimal.json"), Charsets.UTF_8).read().replace("#{TEST_FILE}", "/input_simple_decimal.json"));
        RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
        QueryDataBatch batch = results.get(0);
        assertTrue(batchLoader.load(batch.getHeader().getDef(), batch.getData()));
        String[] decimal9Output = { "99.0000", "11.1235", "0.1000", "-0.1200", "-123.1234", "-1.0001" };
        String[] decimal18Output = { "123456789.000000000", "11.123456789", "0.100000000", "-0.100400000", "-987654321.123456789", "-2.030100000" };
        Iterator<VectorWrapper<?>> itr = batchLoader.iterator();
        // Check the output of decimal9
        ValueVector.Accessor dec9Accessor = itr.next().getValueVector().getAccessor();
        ValueVector.Accessor dec18Accessor = itr.next().getValueVector().getAccessor();
        for (int i = 0; i < dec9Accessor.getValueCount(); i++) {
            assertEquals(dec9Accessor.getObject(i).toString(), decimal9Output[i]);
            assertEquals(dec18Accessor.getObject(i).toString(), decimal18Output[i]);
        }
        assertEquals(6, dec9Accessor.getValueCount());
        assertEquals(6, dec18Accessor.getValueCount());
        batchLoader.clear();
        for (QueryDataBatch result : results) {
            result.release();
        }
    }
}