Examples with PlanFragment org.apache.drill.exec.proto.BitControl.PlanFragment used on opensource projects

Example 31 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class Foreman method runFragment.

/**
 * This is a helper method to run query based on the list of PlanFragment that were planned
 * at some point of time
 * @param fragmentsList fragment list
 * @throws ExecutionSetupException
 */
private void runFragment(List<PlanFragment> fragmentsList) throws ExecutionSetupException {
    // need to set QueryId, MinorFragment for incoming Fragments
    PlanFragment rootFragment = null;
    boolean isFirst = true;
    final List<PlanFragment> planFragments = Lists.newArrayList();
    for (PlanFragment myFragment : fragmentsList) {
        final FragmentHandle handle = myFragment.getHandle();
        // though we have new field in the FragmentHandle - parentQueryId
        // it can not be used until every piece of code that creates handle is using it, as otherwise
        // comparisons on that handle fail that causes fragment runtime failure
        final FragmentHandle newFragmentHandle = FragmentHandle.newBuilder().setMajorFragmentId(handle.getMajorFragmentId()).setMinorFragmentId(handle.getMinorFragmentId()).setQueryId(queryId).build();
        final PlanFragment newFragment = PlanFragment.newBuilder(myFragment).setHandle(newFragmentHandle).build();
        if (isFirst) {
            rootFragment = newFragment;
            isFirst = false;
        } else {
            planFragments.add(newFragment);
        }
    }
    assert rootFragment != null;
    final FragmentRoot rootOperator;
    try {
        rootOperator = drillbitContext.getPlanReader().readFragmentRoot(rootFragment.getFragmentJson());
    } catch (IOException e) {
        throw new ExecutionSetupException(String.format("Unable to parse FragmentRoot from fragment: %s", rootFragment.getFragmentJson()));
    }
    queryRM.setCost(rootOperator.getCost().getOutputRowCount());
    fragmentsRunner.setFragmentsInfo(planFragments, rootFragment, rootOperator);
    startQueryProcessing();
}

Example 32 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class FragmentsRunner method sendRemoteFragments.

/**
 * Send all the remote fragments belonging to a single target drillbit in one request. If the assignment
 * DrillbitEndpoint is local Drillbit then {@link Controller#getTunnel(DrillbitEndpoint)} takes care of submitting it
 * locally without actually creating a Control Connection to itself.
 *
 * @param assignment the drillbit assigned to these fragments
 * @param fragments the set of fragments
 * @param latch the countdown latch used to track the requests to all endpoints
 * @param fragmentSubmitFailures the submission failure counter used to track the requests to all endpoints
 */
private void sendRemoteFragments(final DrillbitEndpoint assignment, final Collection<PlanFragment> fragments, final CountDownLatch latch, final FragmentSubmitFailures fragmentSubmitFailures) {
    final Controller controller = drillbitContext.getController();
    final InitializeFragments.Builder fb = InitializeFragments.newBuilder();
    for (final PlanFragment planFragment : fragments) {
        fb.addFragment(planFragment);
    }
    final InitializeFragments initFrags = fb.build();
    logger.debug("Sending remote fragments to node: {}\nData: {}", assignment, initFrags);
    final FragmentSubmitListener listener = new FragmentSubmitListener(assignment, initFrags, latch, fragmentSubmitFailures);
    controller.getTunnel(assignment).sendFragments(listener, initFrags);
}

Example 33 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class DrillSeparatePlanningTest method testMultiMinorFragmentComplexQuery.

@Test(timeout = 60_000)
public void testMultiMinorFragmentComplexQuery() throws Exception {
    final String query = "SELECT dir0, sum(o_totalprice) FROM dfs.`multilevel/json` group by dir0 order by dir0";
    QueryPlanFragments planFragments = getFragmentsHelper(query);
    assertNotNull(planFragments);
    assertTrue((planFragments.getFragmentsCount() > 1));
    for (PlanFragment planFragment : planFragments.getFragmentsList()) {
        assertTrue(planFragment.getLeafFragment());
    }
    int rowCount = getResultsHelper(planFragments);
    assertEquals(8, rowCount);
}

Example 34 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class TestLocalExchange method testHelperVerifyPartitionSenderParallelization.

// Verify the number of partition senders in a major fragments is not more than the cluster size and each endpoint
// in the cluster has at most one fragment from a given major fragment that has the partition sender.
private static void testHelperVerifyPartitionSenderParallelization(String plan, boolean isMuxOn, boolean isDeMuxOn) throws Exception {
    final DrillbitContext drillbitContext = getDrillbitContext();
    final PhysicalPlanReader planReader = drillbitContext.getPlanReader();
    final Fragment rootFragment = PopUnitTestBase.getRootFragmentFromPlanString(planReader, plan);
    final List<Integer> deMuxFragments = Lists.newLinkedList();
    final List<Integer> htrFragments = Lists.newLinkedList();
    final PlanningSet planningSet = new PlanningSet();
    // Create a planningSet to get the assignment of major fragment ids to fragments.
    PARALLELIZER.initFragmentWrappers(rootFragment, planningSet);
    findFragmentsWithPartitionSender(rootFragment, planningSet, deMuxFragments, htrFragments);
    final QueryContextInformation queryContextInfo = Utilities.createQueryContextInfo("dummySchemaName", "938ea2d9-7cb9-4baf-9414-a5a0b7777e8e");
    QueryWorkUnit qwu = PARALLELIZER.generateWorkUnit(new OptionList(), drillbitContext.getEndpoint(), QueryId.getDefaultInstance(), drillbitContext.getBits(), rootFragment, USER_SESSION, queryContextInfo);
    qwu.applyPlan(planReader);
    // Make sure the number of minor fragments with HashPartitioner within a major fragment is not more than the
    // number of Drillbits in cluster
    ArrayListMultimap<Integer, DrillbitEndpoint> partitionSenderMap = ArrayListMultimap.create();
    for (PlanFragment planFragment : qwu.getFragments()) {
        if (planFragment.getFragmentJson().contains("hash-partition-sender")) {
            int majorFragmentId = planFragment.getHandle().getMajorFragmentId();
            DrillbitEndpoint assignedEndpoint = planFragment.getAssignment();
            partitionSenderMap.get(majorFragmentId).add(assignedEndpoint);
        }
    }
    if (isMuxOn) {
        verifyAssignment(htrFragments, partitionSenderMap);
    }
    if (isDeMuxOn) {
        verifyAssignment(deMuxFragments, partitionSenderMap);
    }
}

Example 35 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class DrillClient method runQuery.

/**
 * Run query based on list of fragments that were supposedly produced during query planning phase. Supported
 * query type is {@link QueryType#EXECUTION}
 * @param type
 * @param planFragments
 * @param resultsListener
 * @throws RpcException
 */
public void runQuery(QueryType type, List<PlanFragment> planFragments, UserResultsListener resultsListener) throws RpcException {
    // QueryType can be only executional
    checkArgument((QueryType.EXECUTION == type), "Only EXECUTION type query is supported with PlanFragments");
    // setting Plan on RunQuery will be used for logging purposes and therefore can not be null
    // since there is no Plan string provided we will create a JsonArray out of individual fragment Plans
    ArrayNode jsonArray = objectMapper.createArrayNode();
    for (PlanFragment fragment : planFragments) {
        try {
            jsonArray.add(objectMapper.readTree(fragment.getFragmentJson()));
        } catch (IOException e) {
            logger.error("Exception while trying to read PlanFragment JSON for {}", fragment.getHandle().getQueryId(), e);
            throw new RpcException(e);
        }
    }
    final String fragmentsToJsonString;
    try {
        fragmentsToJsonString = objectMapper.writeValueAsString(jsonArray);
    } catch (JsonProcessingException e) {
        logger.error("Exception while trying to get JSONString from Array of individual Fragments Json for %s", e);
        throw new RpcException(e);
    }
    final UserProtos.RunQuery query = newBuilder().setType(type).addAllFragments(planFragments).setPlan(fragmentsToJsonString).setResultsMode(STREAM_FULL).build();
    client.submitQuery(resultsListener, query);
}