Search in sources :

Example 51 with DrillbitContext

use of org.apache.drill.exec.server.DrillbitContext in project drill by apache.

the class TestSimpleFilter method testSV4Filter.

@Test
@Ignore("Filter does not support SV4")
public void testSV4Filter() throws Throwable {
    final DrillbitContext bitContext = mockDrillbitContext();
    final UserClientConnection connection = Mockito.mock(UserClientConnection.class);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.asCharSource(DrillFileUtils.getResourceAsFile("/filter/test_sv4.json"), Charsets.UTF_8).read());
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContextImpl context = new FragmentContextImpl(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    int recordCount = 0;
    while (exec.next()) {
        recordCount += exec.getSelectionVector4().getCount();
    }
    exec.close();
    assertEquals(50, recordCount);
    if (context.getExecutorState().getFailureCause() != null) {
        throw context.getExecutorState().getFailureCause();
    }
    assertTrue(!context.getExecutorState().isFailed());
}
Also used : DrillbitContext(org.apache.drill.exec.server.DrillbitContext) SimpleRootExec(org.apache.drill.exec.physical.impl.SimpleRootExec) PhysicalPlan(org.apache.drill.exec.physical.PhysicalPlan) PhysicalPlanReader(org.apache.drill.exec.planner.PhysicalPlanReader) UserClientConnection(org.apache.drill.exec.rpc.UserClientConnection) FragmentContextImpl(org.apache.drill.exec.ops.FragmentContextImpl) FragmentRoot(org.apache.drill.exec.physical.base.FragmentRoot) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) Ignore(org.junit.Ignore) OperatorTest(org.apache.drill.categories.OperatorTest) Test(org.junit.Test) ExecTest(org.apache.drill.exec.ExecTest)

Example 52 with DrillbitContext

use of org.apache.drill.exec.server.DrillbitContext in project drill by apache.

the class TestAllocators method testAllocators.

@Test
public void testAllocators() throws Exception {
    // Setup a drillbit (initializes a root allocator)
    final DrillConfig config = DrillConfig.create(TEST_CONFIGURATIONS);
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(config, serviceSet)) {
        bit.run();
        final DrillbitContext bitContext = bit.getContext();
        FunctionImplementationRegistry functionRegistry = bitContext.getFunctionImplementationRegistry();
        StoragePluginRegistry storageRegistry = new StoragePluginRegistryImpl(bitContext);
        // Create a few Fragment Contexts
        BitControl.PlanFragment.Builder pfBuilder1 = BitControl.PlanFragment.newBuilder();
        pfBuilder1.setMemInitial(1500000);
        BitControl.PlanFragment pf1 = pfBuilder1.build();
        BitControl.PlanFragment.Builder pfBuilder2 = BitControl.PlanFragment.newBuilder();
        pfBuilder2.setMemInitial(500000);
        BitControl.PlanFragment pf2 = pfBuilder1.build();
        FragmentContextImpl fragmentContext1 = new FragmentContextImpl(bitContext, pf1, null, functionRegistry);
        FragmentContextImpl fragmentContext2 = new FragmentContextImpl(bitContext, pf2, null, functionRegistry);
        // Get a few physical operators. Easiest way is to read a physical plan.
        PhysicalPlanReader planReader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(bitContext, storageRegistry);
        PhysicalPlan plan = planReader.readPhysicalPlan(Files.asCharSource(DrillFileUtils.getResourceAsFile(planFile), Charsets.UTF_8).read());
        List<PhysicalOperator> physicalOperators = plan.getSortedOperators();
        Iterator<PhysicalOperator> physicalOperatorIterator = physicalOperators.iterator();
        PhysicalOperator physicalOperator1 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator2 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator3 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator4 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator5 = physicalOperatorIterator.next();
        PhysicalOperator physicalOperator6 = physicalOperatorIterator.next();
        // Create some bogus Operator profile defs and stats to create operator contexts
        OpProfileDef def;
        OperatorStats stats;
        // Use some bogus operator type to create a new operator context.
        def = new OpProfileDef(physicalOperator1.getOperatorId(), MockSubScanPOP.OPERATOR_TYPE, OperatorUtilities.getChildCount(physicalOperator1));
        stats = fragmentContext1.getStats().newOperatorStats(def, fragmentContext1.getAllocator());
        // Add a couple of Operator Contexts
        // Initial allocation = 1000000 bytes for all operators
        OperatorContext oContext11 = fragmentContext1.newOperatorContext(physicalOperator1);
        DrillBuf b11 = oContext11.getAllocator().buffer(1000000);
        OperatorContext oContext12 = fragmentContext1.newOperatorContext(physicalOperator2, stats);
        DrillBuf b12 = oContext12.getAllocator().buffer(500000);
        OperatorContext oContext21 = fragmentContext1.newOperatorContext(physicalOperator3);
        def = new OpProfileDef(physicalOperator4.getOperatorId(), TextFormatPlugin.WRITER_OPERATOR_TYPE, OperatorUtilities.getChildCount(physicalOperator4));
        stats = fragmentContext2.getStats().newOperatorStats(def, fragmentContext2.getAllocator());
        OperatorContext oContext22 = fragmentContext2.newOperatorContext(physicalOperator4, stats);
        DrillBuf b22 = oContext22.getAllocator().buffer(2000000);
        // New Fragment begins
        BitControl.PlanFragment.Builder pfBuilder3 = BitControl.PlanFragment.newBuilder();
        pfBuilder3.setMemInitial(1000000);
        BitControl.PlanFragment pf3 = pfBuilder3.build();
        FragmentContextImpl fragmentContext3 = new FragmentContextImpl(bitContext, pf3, null, functionRegistry);
        // New fragment starts an operator that allocates an amount within the limit
        def = new OpProfileDef(physicalOperator5.getOperatorId(), UnionAll.OPERATOR_TYPE, OperatorUtilities.getChildCount(physicalOperator5));
        stats = fragmentContext3.getStats().newOperatorStats(def, fragmentContext3.getAllocator());
        OperatorContext oContext31 = fragmentContext3.newOperatorContext(physicalOperator5, stats);
        DrillBuf b31a = oContext31.getAllocator().buffer(200000);
        // Previously running operator completes
        b22.release();
        ((AutoCloseable) oContext22).close();
        // Fragment 3 asks for more and fails
        boolean outOfMem = false;
        try {
            oContext31.getAllocator().buffer(44000000);
            fail("Fragment 3 should fail to allocate buffer");
        } catch (OutOfMemoryException e) {
            // Expected.
            outOfMem = true;
        }
        assertTrue(outOfMem);
        // Operator is Exempt from Fragment limits. Fragment 3 asks for more and succeeds
        OperatorContext oContext32 = fragmentContext3.newOperatorContext(physicalOperator6);
        try {
            DrillBuf b32 = oContext32.getAllocator().buffer(4400000);
            b32.release();
        } catch (OutOfMemoryException e) {
            fail("Fragment 3 failed to allocate buffer");
        } finally {
            closeOp(oContext32);
        }
        b11.release();
        closeOp(oContext11);
        b12.release();
        closeOp(oContext12);
        closeOp(oContext21);
        b31a.release();
        closeOp(oContext31);
        fragmentContext1.close();
        fragmentContext2.close();
        fragmentContext3.close();
    }
}
Also used : DrillbitContext(org.apache.drill.exec.server.DrillbitContext) StoragePluginRegistry(org.apache.drill.exec.store.StoragePluginRegistry) PhysicalPlan(org.apache.drill.exec.physical.PhysicalPlan) OpProfileDef(org.apache.drill.exec.ops.OpProfileDef) PhysicalPlanReader(org.apache.drill.exec.planner.PhysicalPlanReader) BitControl(org.apache.drill.exec.proto.BitControl) FragmentContextImpl(org.apache.drill.exec.ops.FragmentContextImpl) OperatorStats(org.apache.drill.exec.ops.OperatorStats) DrillConfig(org.apache.drill.common.config.DrillConfig) Drillbit(org.apache.drill.exec.server.Drillbit) RemoteServiceSet(org.apache.drill.exec.server.RemoteServiceSet) PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator) StoragePluginRegistryImpl(org.apache.drill.exec.store.StoragePluginRegistryImpl) OperatorContext(org.apache.drill.exec.ops.OperatorContext) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) OutOfMemoryException(org.apache.drill.exec.exception.OutOfMemoryException) DrillBuf(io.netty.buffer.DrillBuf) MemoryTest(org.apache.drill.categories.MemoryTest) DrillTest(org.apache.drill.test.DrillTest) Test(org.junit.Test)

Example 53 with DrillbitContext

use of org.apache.drill.exec.server.DrillbitContext in project drill by apache.

the class TestSimpleFunctions method testByteSubstring.

@Test
public void testByteSubstring() throws Throwable {
    final DrillbitContext bitContext = mockDrillbitContext();
    final UserClientConnection connection = Mockito.mock(UserClientConnection.class);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.asCharSource(DrillFileUtils.getResourceAsFile("/functions/testByteSubstring.json"), Charsets.UTF_8).read());
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContextImpl context = new FragmentContextImpl(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    while (exec.next()) {
        final NullableVarBinaryVector c1 = exec.getValueVectorById(new SchemaPath("col3", ExpressionPosition.UNKNOWN), NullableVarBinaryVector.class);
        final NullableVarBinaryVector.Accessor a1 = c1.getAccessor();
        int count = 0;
        for (int i = 0; i < c1.getAccessor().getValueCount(); i++) {
            if (!a1.isNull(i)) {
                final NullableVarBinaryHolder holder = new NullableVarBinaryHolder();
                a1.get(i, holder);
                assertEquals("aa", StringFunctionHelpers.toStringFromUTF8(holder.start, holder.end, holder.buffer));
                ++count;
            }
        }
        assertEquals(50, count);
    }
    if (context.getExecutorState().getFailureCause() != null) {
        throw context.getExecutorState().getFailureCause();
    }
    assertTrue(!context.getExecutorState().isFailed());
}
Also used : DrillbitContext(org.apache.drill.exec.server.DrillbitContext) PhysicalPlan(org.apache.drill.exec.physical.PhysicalPlan) PhysicalPlanReader(org.apache.drill.exec.planner.PhysicalPlanReader) NullableVarBinaryVector(org.apache.drill.exec.vector.NullableVarBinaryVector) FragmentContextImpl(org.apache.drill.exec.ops.FragmentContextImpl) FragmentRoot(org.apache.drill.exec.physical.base.FragmentRoot) NullableVarBinaryHolder(org.apache.drill.exec.expr.holders.NullableVarBinaryHolder) SchemaPath(org.apache.drill.common.expression.SchemaPath) UserClientConnection(org.apache.drill.exec.rpc.UserClientConnection) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry) ExecTest(org.apache.drill.exec.ExecTest) Test(org.junit.Test)

Example 54 with DrillbitContext

use of org.apache.drill.exec.server.DrillbitContext in project drill by apache.

the class TestImplicitCastFunctions method runTest.

public void runTest(Object[] expectedResults, String planPath) throws Throwable {
    final DrillbitContext bitContext = mockDrillbitContext();
    final UserClientConnection connection = Mockito.mock(UserClientConnection.class);
    final String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8);
    if (reader == null) {
        reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    }
    if (registry == null) {
        registry = new FunctionImplementationRegistry(c);
    }
    if (context == null) {
        context = new FragmentContextImpl(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
    }
    final PhysicalPlan plan = reader.readPhysicalPlan(planString);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    // skip schema batch
    exec.next();
    while (exec.next()) {
        final Object[] res = getRunResult(exec);
        assertEquals("return count does not match", res.length, expectedResults.length);
        for (int i = 0; i < res.length; i++) {
            assertEquals(String.format("column %s does not match", i), res[i], expectedResults[i]);
        }
    }
    if (context.getExecutorState().getFailureCause() != null) {
        throw context.getExecutorState().getFailureCause();
    }
    assertTrue(!context.getExecutorState().isFailed());
}
Also used : DrillbitContext(org.apache.drill.exec.server.DrillbitContext) PhysicalPlan(org.apache.drill.exec.physical.PhysicalPlan) UserClientConnection(org.apache.drill.exec.rpc.UserClientConnection) FragmentContextImpl(org.apache.drill.exec.ops.FragmentContextImpl) FragmentRoot(org.apache.drill.exec.physical.base.FragmentRoot) FunctionImplementationRegistry(org.apache.drill.exec.expr.fn.FunctionImplementationRegistry)

Example 55 with DrillbitContext

use of org.apache.drill.exec.server.DrillbitContext in project drill by apache.

the class TestSpnegoAuthentication method testSPNEGOAndFORMEnabled.

/**
 * Both SPNEGO and FORM mechanism is enabled for WebServer in configuration. Test to see if the respective security
 * handlers are created successfully or not.
 */
@Test
public void testSPNEGOAndFORMEnabled() throws Exception {
    final DrillConfig newConfig = new DrillConfig(DrillConfig.create().withValue(ExecConstants.USER_AUTHENTICATION_ENABLED, ConfigValueFactory.fromAnyRef(true)).withValue(ExecConstants.HTTP_AUTHENTICATION_MECHANISMS, ConfigValueFactory.fromIterable(Lists.newArrayList("form", "spnego"))).withValue(ExecConstants.HTTP_SPNEGO_PRINCIPAL, ConfigValueFactory.fromAnyRef(spnegoHelper.SERVER_PRINCIPAL)).withValue(ExecConstants.HTTP_SPNEGO_KEYTAB, ConfigValueFactory.fromAnyRef(spnegoHelper.serverKeytab.toString())));
    final ScanResult scanResult = ClassPathScanner.fromPrescan(newConfig);
    final AuthenticatorProviderImpl authenticatorProvider = Mockito.mock(AuthenticatorProviderImpl.class);
    Mockito.when(authenticatorProvider.containsFactory(PlainFactory.SIMPLE_NAME)).thenReturn(true);
    final DrillbitContext context = Mockito.mock(DrillbitContext.class);
    Mockito.when(context.getClasspathScan()).thenReturn(scanResult);
    Mockito.when(context.getConfig()).thenReturn(newConfig);
    Mockito.when(context.getAuthProvider()).thenReturn(authenticatorProvider);
    final DrillHttpSecurityHandlerProvider securityProvider = new DrillHttpSecurityHandlerProvider(newConfig, context);
    assertTrue(securityProvider.isFormEnabled());
    assertTrue(securityProvider.isSpnegoEnabled());
}
Also used : DrillbitContext(org.apache.drill.exec.server.DrillbitContext) AuthenticatorProviderImpl(org.apache.drill.exec.rpc.security.AuthenticatorProviderImpl) ScanResult(org.apache.drill.common.scanner.persistence.ScanResult) DrillHttpSecurityHandlerProvider(org.apache.drill.exec.server.rest.auth.DrillHttpSecurityHandlerProvider) DrillConfig(org.apache.drill.common.config.DrillConfig) BaseTest(org.apache.drill.test.BaseTest) SecurityTest(org.apache.drill.categories.SecurityTest) Test(org.junit.Test)

Aggregations

DrillbitContext (org.apache.drill.exec.server.DrillbitContext)137 Test (org.junit.Test)96 FunctionImplementationRegistry (org.apache.drill.exec.expr.fn.FunctionImplementationRegistry)86 PhysicalPlan (org.apache.drill.exec.physical.PhysicalPlan)80 UserClientConnection (org.apache.drill.exec.rpc.UserClientConnection)78 FragmentContextImpl (org.apache.drill.exec.ops.FragmentContextImpl)77 FragmentRoot (org.apache.drill.exec.physical.base.FragmentRoot)72 PhysicalPlanReader (org.apache.drill.exec.planner.PhysicalPlanReader)72 SimpleRootExec (org.apache.drill.exec.physical.impl.SimpleRootExec)44 OperatorTest (org.apache.drill.categories.OperatorTest)39 ExecTest (org.apache.drill.exec.ExecTest)35 SchemaPath (org.apache.drill.common.expression.SchemaPath)32 DrillConfig (org.apache.drill.common.config.DrillConfig)28 ValueVector (org.apache.drill.exec.vector.ValueVector)17 Ignore (org.junit.Ignore)14 IntVector (org.apache.drill.exec.vector.IntVector)13 SecurityTest (org.apache.drill.categories.SecurityTest)12 ScanResult (org.apache.drill.common.scanner.persistence.ScanResult)12 DrillbitEndpoint (org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint)12 SystemOptionManager (org.apache.drill.exec.server.options.SystemOptionManager)12