Examples with PlanFragment org.apache.drill.exec.proto.BitControl.PlanFragment used on opensource projects

Example 26 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by axbaretto.

the class TestLocalExchange method testHelperVerifyPartitionSenderParallelization.

// Verify the number of partition senders in a major fragments is not more than the cluster size and each endpoint
// in the cluster has at most one fragment from a given major fragment that has the partition sender.
private static void testHelperVerifyPartitionSenderParallelization(String plan, boolean isMuxOn, boolean isDeMuxOn) throws Exception {
    final DrillbitContext drillbitContext = getDrillbitContext();
    final PhysicalPlanReader planReader = drillbitContext.getPlanReader();
    final Fragment rootFragment = PopUnitTestBase.getRootFragmentFromPlanString(planReader, plan);
    final List<Integer> deMuxFragments = Lists.newLinkedList();
    final List<Integer> htrFragments = Lists.newLinkedList();
    final PlanningSet planningSet = new PlanningSet();
    // Create a planningSet to get the assignment of major fragment ids to fragments.
    PARALLELIZER.initFragmentWrappers(rootFragment, planningSet);
    findFragmentsWithPartitionSender(rootFragment, planningSet, deMuxFragments, htrFragments);
    final QueryContextInformation queryContextInfo = Utilities.createQueryContextInfo("dummySchemaName", "938ea2d9-7cb9-4baf-9414-a5a0b7777e8e");
    QueryWorkUnit qwu = PARALLELIZER.getFragments(new OptionList(), drillbitContext.getEndpoint(), QueryId.getDefaultInstance(), drillbitContext.getBits(), rootFragment, USER_SESSION, queryContextInfo);
    qwu.applyPlan(planReader);
    // Make sure the number of minor fragments with HashPartitioner within a major fragment is not more than the
    // number of Drillbits in cluster
    ArrayListMultimap<Integer, DrillbitEndpoint> partitionSenderMap = ArrayListMultimap.create();
    for (PlanFragment planFragment : qwu.getFragments()) {
        if (planFragment.getFragmentJson().contains("hash-partition-sender")) {
            int majorFragmentId = planFragment.getHandle().getMajorFragmentId();
            DrillbitEndpoint assignedEndpoint = planFragment.getAssignment();
            partitionSenderMap.get(majorFragmentId).add(assignedEndpoint);
        }
    }
    if (isMuxOn) {
        verifyAssignment(htrFragments, partitionSenderMap);
    }
    if (isDeMuxOn) {
        verifyAssignment(deMuxFragments, partitionSenderMap);
    }
}

Example 27 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by axbaretto.

the class PlanSplitter method getFragments.

private List<PlanFragment> getFragments(final DrillbitContext dContext, final GetQueryPlanFragments req, final QueryContext queryContext, final QueryId queryId) throws Exception {
    final PhysicalPlan plan;
    final String query = req.getQuery();
    switch(req.getType()) {
        case SQL:
            final Pointer<String> textPlan = new Pointer<>();
            plan = DrillSqlWorker.getPlan(queryContext, query, textPlan);
            break;
        case PHYSICAL:
            plan = dContext.getPlanReader().readPhysicalPlan(query);
            break;
        default:
            throw new IllegalStateException("Planning fragments supports only SQL or PHYSICAL QueryType");
    }
    QueryResourceAllocator planner = dContext.getResourceManager().newResourceAllocator(queryContext);
    planner.visitAbstractPlan(plan);
    final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
    final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null);
    final SimpleParallelizer parallelizer = new SplittingParallelizer(queryContext);
    List<PlanFragment> fragments = Lists.newArrayList();
    if (req.getSplitPlan()) {
        final List<QueryWorkUnit> queryWorkUnits = parallelizer.getSplitFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), dContext.getPlanReader(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        for (QueryWorkUnit queryWorkUnit : queryWorkUnits) {
            planner.visitPhysicalPlan(queryWorkUnit);
            queryWorkUnit.applyPlan(dContext.getPlanReader());
            fragments.add(queryWorkUnit.getRootFragment());
            List<PlanFragment> childFragments = queryWorkUnit.getFragments();
            if (!childFragments.isEmpty()) {
                throw new IllegalStateException("Split plans can not have more then one fragment");
            }
        }
    } else {
        final QueryWorkUnit queryWorkUnit = parallelizer.getFragments(queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), rootFragment, queryContext.getSession(), queryContext.getQueryContextInfo());
        planner.visitPhysicalPlan(queryWorkUnit);
        queryWorkUnit.applyPlan(dContext.getPlanReader());
        fragments.add(queryWorkUnit.getRootFragment());
        fragments.addAll(queryWorkUnit.getFragments());
    }
    return fragments;
}

Example 28 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by axbaretto.

the class DrillSeparatePlanningTest method testPlanningNoSplit.

@Test(timeout = 60_000)
public void testPlanningNoSplit() throws Exception {
    final String query = "SELECT dir0, sum(o_totalprice) FROM dfs.`multilevel/json` group by dir0 order by dir0";
    client.alterSession("planner.slice_target", 1);
    try {
        final QueryPlanFragments planFragments = client.planQuery(query);
        assertNotNull(planFragments);
        assertTrue((planFragments.getFragmentsCount() > 1));
        PlanFragment rootFragment = planFragments.getFragments(0);
        assertFalse(rootFragment.getLeafFragment());
        QuerySummary summary = client.queryBuilder().plan(planFragments.getFragmentsList()).run();
        assertEquals(3, summary.recordCount());
    } finally {
        client.resetSession("planner.slice_target");
    }
}

Example 29 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by axbaretto.

the class DrillSeparatePlanningTest method getResultsHelper.

private int getResultsHelper(final QueryPlanFragments planFragments) throws Exception {
    int totalRows = 0;
    for (PlanFragment fragment : planFragments.getFragmentsList()) {
        DrillbitEndpoint assignedNode = fragment.getAssignment();
        ClientFixture fragmentClient = cluster.client(assignedNode.getAddress(), assignedNode.getUserPort());
        RowSet rowSet = fragmentClient.queryBuilder().sql("select hostname, user_port from sys.drillbits where `current`=true").rowSet();
        assertEquals(1, rowSet.rowCount());
        RowSetReader reader = rowSet.reader();
        assertTrue(reader.next());
        String host = reader.scalar("hostname").getString();
        int port = reader.scalar("user_port").getInt();
        rowSet.clear();
        assertEquals(assignedNode.getAddress(), host);
        assertEquals(assignedNode.getUserPort(), port);
        List<PlanFragment> fragmentList = Lists.newArrayList();
        fragmentList.add(fragment);
        QuerySummary summary = fragmentClient.queryBuilder().plan(fragmentList).run();
        totalRows += summary.recordCount();
        fragmentClient.close();
    }
    return totalRows;
}

Example 30 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by axbaretto.

the class DrillSeparatePlanningTest method testMultiMinorFragmentSimpleQuery.

@Test(timeout = 60_000)
public void testMultiMinorFragmentSimpleQuery() throws Exception {
    final String query = "SELECT o_orderkey FROM dfs.`multilevel/json`";
    QueryPlanFragments planFragments = getFragmentsHelper(query);
    assertNotNull(planFragments);
    assertTrue((planFragments.getFragmentsCount() > 1));
    for (PlanFragment planFragment : planFragments.getFragmentsList()) {
        assertTrue(planFragment.getLeafFragment());
    }
    int rowCount = getResultsHelper(planFragments);
    assertEquals(120, rowCount);
}