Search in sources :

Example 6 with Pointer

use of org.apache.drill.exec.util.Pointer in project drill by apache.

the class PlanSplitter method getFragments.

private List<PlanFragment> getFragments(final DrillbitContext dContext, final GetQueryPlanFragments req, final QueryContext queryContext, final QueryId queryId) throws Exception {
    final PhysicalPlan plan;
    final String query = req.getQuery();
    switch(req.getType()) {
        case SQL:
            final Pointer<String> textPlan = new Pointer<>();
            plan = DrillSqlWorker.getPlan(queryContext, query, textPlan);
            break;
        case PHYSICAL:
            plan = dContext.getPlanReader().readPhysicalPlan(query);
            break;
        default:
            throw new IllegalStateException("Planning fragments supports only SQL or PHYSICAL QueryType");
    }
    QueryResourceAllocator planner = dContext.getResourceManager().newResourceAllocator(queryContext);
    planner.visitAbstractPlan(plan);
    final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
    final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null);
    final SimpleParallelizer parallelizer = new SplittingParallelizer(plan.getProperties().hasResourcePlan, queryContext);
    List<PlanFragment> fragments = Lists.newArrayList();
    if (req.getSplitPlan()) {
        final List<QueryWorkUnit> queryWorkUnits = parallelizer.getSplitFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), dContext.getPlanReader(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        for (QueryWorkUnit queryWorkUnit : queryWorkUnits) {
            planner.visitPhysicalPlan(queryWorkUnit);
            queryWorkUnit.applyPlan(dContext.getPlanReader());
            fragments.add(queryWorkUnit.getRootFragment());
            List<PlanFragment> childFragments = queryWorkUnit.getFragments();
            if (!childFragments.isEmpty()) {
                throw new IllegalStateException("Split plans can not have more then one fragment");
            }
        }
    } else {
        final QueryWorkUnit queryWorkUnit = parallelizer.generateWorkUnit(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        planner.visitPhysicalPlan(queryWorkUnit);
        queryWorkUnit.applyPlan(dContext.getPlanReader());
        fragments.add(queryWorkUnit.getRootFragment());
        fragments.addAll(queryWorkUnit.getFragments());
    }
    return fragments;
}
Also used : PhysicalPlan(org.apache.drill.exec.physical.PhysicalPlan) QueryWorkUnit(org.apache.drill.exec.work.QueryWorkUnit) SimpleParallelizer(org.apache.drill.exec.planner.fragment.SimpleParallelizer) Pointer(org.apache.drill.exec.util.Pointer) PlanFragment(org.apache.drill.exec.proto.BitControl.PlanFragment) Fragment(org.apache.drill.exec.planner.fragment.Fragment) PlanFragment(org.apache.drill.exec.proto.BitControl.PlanFragment) PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator) SplittingParallelizer(org.apache.drill.exec.planner.fragment.contrib.SplittingParallelizer) QueryResourceAllocator(org.apache.drill.exec.work.foreman.rm.QueryResourceAllocator)

Example 7 with Pointer

use of org.apache.drill.exec.util.Pointer in project drill by axbaretto.

the class TestPauseInjection method timedPauseOnSpecificBit.

@Test
public void timedPauseOnSpecificBit() {
    final RemoteServiceSet remoteServiceSet = RemoteServiceSet.getLocalServiceSet();
    final ZookeeperHelper zkHelper = new ZookeeperHelper();
    zkHelper.startZookeeper(1);
    final long pauseDuration = 2000L;
    final long expectedDuration = pauseDuration;
    try {
        // Creating two drillbits
        final Drillbit drillbit1, drillbit2;
        final DrillConfig drillConfig = zkHelper.getConfig();
        try {
            drillbit1 = Drillbit.start(drillConfig, remoteServiceSet);
            drillbit2 = Drillbit.start(drillConfig, remoteServiceSet);
        } catch (final DrillbitStartupException e) {
            throw new RuntimeException("Failed to start two drillbits.", e);
        }
        final DrillbitContext drillbitContext1 = drillbit1.getContext();
        final DrillbitContext drillbitContext2 = drillbit2.getContext();
        final UserSession session = UserSession.Builder.newBuilder().withCredentials(UserCredentials.newBuilder().setUserName("foo").build()).withUserProperties(UserProperties.getDefaultInstance()).withOptionManager(drillbitContext1.getOptionManager()).build();
        final DrillbitEndpoint drillbitEndpoint1 = drillbitContext1.getEndpoint();
        final String controls = Controls.newBuilder().addTimedPauseOnBit(DummyClass.class, DummyClass.PAUSES, drillbitEndpoint1, 0, pauseDuration).build();
        ControlsInjectionUtil.setControls(session, controls);
        {
            final ExtendedLatch trigger = new ExtendedLatch(1);
            final Pointer<Exception> ex = new Pointer<>();
            final QueryContext queryContext = new QueryContext(session, drillbitContext1, QueryId.getDefaultInstance());
            // test that the pause happens
            final DummyClass dummyClass = new DummyClass(queryContext, trigger);
            final long actualDuration = dummyClass.pauses();
            assertTrue(String.format("Test should stop for at least %d milliseconds.", expectedDuration), expectedDuration <= actualDuration);
            assertTrue("No exception should be thrown.", ex.value == null);
            try {
                queryContext.close();
            } catch (final Exception e) {
                fail("Failed to close query context: " + e);
            }
        }
        {
            final ExtendedLatch trigger = new ExtendedLatch(1);
            final QueryContext queryContext = new QueryContext(session, drillbitContext2, QueryId.getDefaultInstance());
            // if the resume did not happen, the test would hang
            final DummyClass dummyClass = new DummyClass(queryContext, trigger);
            dummyClass.pauses();
            try {
                queryContext.close();
            } catch (final Exception e) {
                fail("Failed to close query context: " + e);
            }
        }
    } finally {
        zkHelper.stopZookeeper();
    }
}
Also used : DrillbitContext(org.apache.drill.exec.server.DrillbitContext) ZookeeperHelper(org.apache.drill.exec.ZookeeperHelper) Pointer(org.apache.drill.exec.util.Pointer) QueryContext(org.apache.drill.exec.ops.QueryContext) DrillbitStartupException(org.apache.drill.exec.exception.DrillbitStartupException) ExtendedLatch(org.apache.drill.common.concurrent.ExtendedLatch) DrillbitEndpoint(org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint) DrillConfig(org.apache.drill.common.config.DrillConfig) DrillbitStartupException(org.apache.drill.exec.exception.DrillbitStartupException) Drillbit(org.apache.drill.exec.server.Drillbit) RemoteServiceSet(org.apache.drill.exec.server.RemoteServiceSet) UserSession(org.apache.drill.exec.rpc.user.UserSession) Test(org.junit.Test)

Example 8 with Pointer

use of org.apache.drill.exec.util.Pointer in project drill by axbaretto.

the class TestPauseInjection method pauseInjected.

@Test
public void pauseInjected() {
    final long expectedDuration = 1000L;
    final ExtendedLatch trigger = new ExtendedLatch(1);
    final Pointer<Exception> ex = new Pointer<>();
    final String controls = Controls.newBuilder().addPause(DummyClass.class, DummyClass.PAUSES).build();
    ControlsInjectionUtil.setControls(session, controls);
    final QueryContext queryContext = new QueryContext(session, bits[0].getContext(), QueryId.getDefaultInstance());
    (new ResumingThread(queryContext, trigger, ex, expectedDuration)).start();
    // test that the pause happens
    final DummyClass dummyClass = new DummyClass(queryContext, trigger);
    final long actualDuration = dummyClass.pauses();
    assertTrue(String.format("Test should stop for at least %d milliseconds.", expectedDuration), expectedDuration <= actualDuration);
    assertTrue("No exception should be thrown.", ex.value == null);
    try {
        queryContext.close();
    } catch (final Exception e) {
        fail("Failed to close query context: " + e);
    }
}
Also used : Pointer(org.apache.drill.exec.util.Pointer) QueryContext(org.apache.drill.exec.ops.QueryContext) DrillbitStartupException(org.apache.drill.exec.exception.DrillbitStartupException) ExtendedLatch(org.apache.drill.common.concurrent.ExtendedLatch) Test(org.junit.Test)

Example 9 with Pointer

use of org.apache.drill.exec.util.Pointer in project drill by axbaretto.

the class TestPauseInjection method timedPauseInjected.

@Test
public void timedPauseInjected() {
    final long pauseDuration = 2000L;
    final long expectedDuration = pauseDuration;
    final ExtendedLatch trigger = new ExtendedLatch(1);
    final Pointer<Exception> ex = new Pointer<>();
    final String controls = Controls.newBuilder().addTimedPause(DummyClass.class, DummyClass.PAUSES, 0, pauseDuration).build();
    ControlsInjectionUtil.setControls(session, controls);
    final QueryContext queryContext = new QueryContext(session, bits[0].getContext(), QueryId.getDefaultInstance());
    // We don't need a ResumingThread because this should automatically resume
    // test that the pause happens
    final DummyClass dummyClass = new DummyClass(queryContext, trigger);
    final long actualDuration = dummyClass.pauses();
    assertTrue(String.format("Test should stop for at least %d milliseconds.", expectedDuration), expectedDuration <= actualDuration);
    assertTrue("No exception should be thrown.", ex.value == null);
    try {
        queryContext.close();
    } catch (final Exception e) {
        fail("Failed to close query context: " + e);
    }
}
Also used : Pointer(org.apache.drill.exec.util.Pointer) QueryContext(org.apache.drill.exec.ops.QueryContext) DrillbitStartupException(org.apache.drill.exec.exception.DrillbitStartupException) ExtendedLatch(org.apache.drill.common.concurrent.ExtendedLatch) Test(org.junit.Test)

Example 10 with Pointer

use of org.apache.drill.exec.util.Pointer in project drill by axbaretto.

the class PlanSplitter method getFragments.

private List<PlanFragment> getFragments(final DrillbitContext dContext, final GetQueryPlanFragments req, final QueryContext queryContext, final QueryId queryId) throws Exception {
    final PhysicalPlan plan;
    final String query = req.getQuery();
    switch(req.getType()) {
        case SQL:
            final Pointer<String> textPlan = new Pointer<>();
            plan = DrillSqlWorker.getPlan(queryContext, query, textPlan);
            break;
        case PHYSICAL:
            plan = dContext.getPlanReader().readPhysicalPlan(query);
            break;
        default:
            throw new IllegalStateException("Planning fragments supports only SQL or PHYSICAL QueryType");
    }
    QueryResourceAllocator planner = dContext.getResourceManager().newResourceAllocator(queryContext);
    planner.visitAbstractPlan(plan);
    final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
    final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null);
    final SimpleParallelizer parallelizer = new SplittingParallelizer(queryContext);
    List<PlanFragment> fragments = Lists.newArrayList();
    if (req.getSplitPlan()) {
        final List<QueryWorkUnit> queryWorkUnits = parallelizer.getSplitFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), dContext.getPlanReader(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        for (QueryWorkUnit queryWorkUnit : queryWorkUnits) {
            planner.visitPhysicalPlan(queryWorkUnit);
            queryWorkUnit.applyPlan(dContext.getPlanReader());
            fragments.add(queryWorkUnit.getRootFragment());
            List<PlanFragment> childFragments = queryWorkUnit.getFragments();
            if (!childFragments.isEmpty()) {
                throw new IllegalStateException("Split plans can not have more then one fragment");
            }
        }
    } else {
        final QueryWorkUnit queryWorkUnit = parallelizer.getFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        planner.visitPhysicalPlan(queryWorkUnit);
        queryWorkUnit.applyPlan(dContext.getPlanReader());
        fragments.add(queryWorkUnit.getRootFragment());
        fragments.addAll(queryWorkUnit.getFragments());
    }
    return fragments;
}
Also used : PhysicalPlan(org.apache.drill.exec.physical.PhysicalPlan) QueryWorkUnit(org.apache.drill.exec.work.QueryWorkUnit) SimpleParallelizer(org.apache.drill.exec.planner.fragment.SimpleParallelizer) Pointer(org.apache.drill.exec.util.Pointer) PlanFragment(org.apache.drill.exec.proto.BitControl.PlanFragment) Fragment(org.apache.drill.exec.planner.fragment.Fragment) PlanFragment(org.apache.drill.exec.proto.BitControl.PlanFragment) PhysicalOperator(org.apache.drill.exec.physical.base.PhysicalOperator) SplittingParallelizer(org.apache.drill.exec.planner.fragment.contrib.SplittingParallelizer) QueryResourceAllocator(org.apache.drill.exec.work.foreman.rm.QueryResourceAllocator)

Aggregations

Pointer (org.apache.drill.exec.util.Pointer)15 ExtendedLatch (org.apache.drill.common.concurrent.ExtendedLatch)10 QueryContext (org.apache.drill.exec.ops.QueryContext)10 Test (org.junit.Test)10 DrillbitStartupException (org.apache.drill.exec.exception.DrillbitStartupException)8 DrillConfig (org.apache.drill.common.config.DrillConfig)4 ZookeeperHelper (org.apache.drill.exec.ZookeeperHelper)4 PhysicalPlan (org.apache.drill.exec.physical.PhysicalPlan)4 DrillbitEndpoint (org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint)4 UserSession (org.apache.drill.exec.rpc.user.UserSession)4 Drillbit (org.apache.drill.exec.server.Drillbit)4 DrillbitContext (org.apache.drill.exec.server.DrillbitContext)4 RemoteServiceSet (org.apache.drill.exec.server.RemoteServiceSet)4 PhysicalOperator (org.apache.drill.exec.physical.base.PhysicalOperator)2 Fragment (org.apache.drill.exec.planner.fragment.Fragment)2 SimpleParallelizer (org.apache.drill.exec.planner.fragment.SimpleParallelizer)2 SplittingParallelizer (org.apache.drill.exec.planner.fragment.contrib.SplittingParallelizer)2 PlanFragment (org.apache.drill.exec.proto.BitControl.PlanFragment)2 QueryWorkUnit (org.apache.drill.exec.work.QueryWorkUnit)2 QueryResourceAllocator (org.apache.drill.exec.work.foreman.rm.QueryResourceAllocator)2