Examples with PlanFragment org.apache.drill.exec.proto.BitControl.PlanFragment used on opensource projects

Example 11 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class Foreman method runFragment.

/**
   * This is a helper method to run query based on the list of PlanFragment that were planned
   * at some point of time
   * @param fragmentsList
   * @throws ExecutionSetupException
   */
private void runFragment(List<PlanFragment> fragmentsList) throws ExecutionSetupException {
    // need to set QueryId, MinorFragment for incoming Fragments
    PlanFragment rootFragment = null;
    boolean isFirst = true;
    final List<PlanFragment> planFragments = Lists.newArrayList();
    for (PlanFragment myFragment : fragmentsList) {
        final FragmentHandle handle = myFragment.getHandle();
        // though we have new field in the FragmentHandle - parentQueryId
        // it can not be used until every piece of code that creates handle is using it, as otherwise
        // comparisons on that handle fail that causes fragment runtime failure
        final FragmentHandle newFragmentHandle = FragmentHandle.newBuilder().setMajorFragmentId(handle.getMajorFragmentId()).setMinorFragmentId(handle.getMinorFragmentId()).setQueryId(queryId).build();
        final PlanFragment newFragment = PlanFragment.newBuilder(myFragment).setHandle(newFragmentHandle).build();
        if (isFirst) {
            rootFragment = newFragment;
            isFirst = false;
        } else {
            planFragments.add(newFragment);
        }
    }
    final FragmentRoot rootOperator;
    try {
        rootOperator = drillbitContext.getPlanReader().readFragmentOperator(rootFragment.getFragmentJson());
    } catch (IOException e) {
        throw new ExecutionSetupException(String.format("Unable to parse FragmentRoot from fragment: %s", rootFragment.getFragmentJson()));
    }
    if (queuingEnabled) {
        acquireQuerySemaphore(rootOperator.getCost());
        moveToState(QueryState.STARTING, null);
    }
    drillbitContext.getWorkBus().addFragmentStatusListener(queryId, queryManager.getFragmentStatusListener());
    drillbitContext.getClusterCoordinator().addDrillbitStatusListener(queryManager.getDrillbitStatusListener());
    logger.debug("Submitting fragments to run.");
    // set up the root fragment first so we'll have incoming buffers available.
    setupRootFragment(rootFragment, rootOperator);
    setupNonRootFragments(planFragments);
    moveToState(QueryState.RUNNING, null);
    logger.debug("Fragments running.");
}

Example 12 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class DrillClient method runQuery.

/**
   * Run query based on list of fragments that were supposedly produced during query planning phase. Supported
   * query type is {@link QueryType#EXECUTION}
   * @param type
   * @param planFragments
   * @param resultsListener
   * @throws RpcException
   */
public void runQuery(QueryType type, List<PlanFragment> planFragments, UserResultsListener resultsListener) throws RpcException {
    // QueryType can be only executional
    checkArgument((QueryType.EXECUTION == type), "Only EXECUTION type query is supported with PlanFragments");
    // setting Plan on RunQuery will be used for logging purposes and therefore can not be null
    // since there is no Plan string provided we will create a JsonArray out of individual fragment Plans
    ArrayNode jsonArray = objectMapper.createArrayNode();
    for (PlanFragment fragment : planFragments) {
        try {
            jsonArray.add(objectMapper.readTree(fragment.getFragmentJson()));
        } catch (IOException e) {
            logger.error("Exception while trying to read PlanFragment JSON for %s", fragment.getHandle().getQueryId(), e);
            throw new RpcException(e);
        }
    }
    final String fragmentsToJsonString;
    try {
        fragmentsToJsonString = objectMapper.writeValueAsString(jsonArray);
    } catch (JsonProcessingException e) {
        logger.error("Exception while trying to get JSONString from Array of individual Fragments Json for %s", e);
        throw new RpcException(e);
    }
    final UserProtos.RunQuery query = newBuilder().setType(type).addAllFragments(planFragments).setPlan(fragmentsToJsonString).setResultsMode(STREAM_FULL).build();
    client.submitQuery(resultsListener, query);
}

Example 13 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class SimpleParallelizer method generateWorkUnit.

protected QueryWorkUnit generateWorkUnit(OptionList options, DrillbitEndpoint foremanNode, QueryId queryId, PhysicalPlanReader reader, Fragment rootNode, PlanningSet planningSet, UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
    List<PlanFragment> fragments = Lists.newArrayList();
    PlanFragment rootFragment = null;
    FragmentRoot rootOperator = null;
    // assigned before we can materialize, so we start a new loop here rather than utilizing the previous one.
    for (Wrapper wrapper : planningSet) {
        Fragment node = wrapper.getNode();
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        boolean isRootNode = rootNode == node;
        if (isRootNode && wrapper.getWidth() != 1) {
            throw new ForemanSetupException(String.format("Failure while trying to setup fragment. " + "The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth()));
        }
        // a fragment is self driven if it doesn't rely on any other exchanges.
        boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
        // Create a minorFragment for each major fragment.
        for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
            IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
            wrapper.resetAllocation();
            PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
            Preconditions.checkArgument(op instanceof FragmentRoot);
            FragmentRoot root = (FragmentRoot) op;
            // get plan as JSON
            String plan;
            String optionsData;
            try {
                plan = reader.writeJson(root);
                optionsData = reader.writeJson(options);
            } catch (JsonProcessingException e) {
                throw new ForemanSetupException("Failure while trying to convert fragment into json.", e);
            }
            FragmentHandle handle = //
            FragmentHandle.newBuilder().setMajorFragmentId(//
            wrapper.getMajorFragmentId()).setMinorFragmentId(//
            minorFragmentId).setQueryId(//
            queryId).build();
            PlanFragment fragment = //
            PlanFragment.newBuilder().setForeman(//
            foremanNode).setFragmentJson(//
            plan).setHandle(//
            handle).setAssignment(//
            wrapper.getAssignedEndpoint(minorFragmentId)).setLeafFragment(//
            isLeafFragment).setContext(queryContextInfo).setMemInitial(//
            wrapper.getInitialAllocation()).setMemMax(wrapper.getMaxAllocation()).setOptionsJson(optionsData).setCredentials(session.getCredentials()).addAllCollector(CountRequiredFragments.getCollectors(root)).build();
            if (isRootNode) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                rootFragment = fragment;
                rootOperator = root;
            } else {
                if (logger.isDebugEnabled()) {
                    logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                fragments.add(fragment);
            }
        }
    }
    return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}

Example 14 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class TestFragmentChecker method print.

private void print(String fragmentFile, int bitCount, int expectedFragmentCount) throws Exception {
    System.out.println(String.format("=================Building plan fragments for [%s].  Allowing %d total Drillbits.==================", fragmentFile, bitCount));
    PhysicalPlanReader ppr = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(CONFIG);
    Fragment fragmentRoot = getRootFragment(ppr, fragmentFile);
    SimpleParallelizer par = new SimpleParallelizer(1000 * 1000, 5, 10, 1.2);
    List<DrillbitEndpoint> endpoints = Lists.newArrayList();
    DrillbitEndpoint localBit = null;
    for (int i = 0; i < bitCount; i++) {
        DrillbitEndpoint b1 = DrillbitEndpoint.newBuilder().setAddress("localhost").setControlPort(1234 + i).build();
        if (i == 0) {
            localBit = b1;
        }
        endpoints.add(b1);
    }
    final QueryContextInformation queryContextInfo = Utilities.createQueryContextInfo("dummySchemaName", "938ea2d9-7cb9-4baf-9414-a5a0b7777e8e");
    QueryWorkUnit qwu = par.getFragments(new OptionList(), localBit, QueryId.getDefaultInstance(), endpoints, ppr, fragmentRoot, UserSession.Builder.newBuilder().withCredentials(UserBitShared.UserCredentials.newBuilder().setUserName("foo").build()).build(), queryContextInfo);
    System.out.println(String.format("=========ROOT FRAGMENT [%d:%d] =========", qwu.getRootFragment().getHandle().getMajorFragmentId(), qwu.getRootFragment().getHandle().getMinorFragmentId()));
    System.out.print(qwu.getRootFragment().getFragmentJson());
    for (PlanFragment f : qwu.getFragments()) {
        System.out.println(String.format("=========Fragment [%d:%d]=====", f.getHandle().getMajorFragmentId(), f.getHandle().getMinorFragmentId()));
        System.out.print(f.getFragmentJson());
    }
    assertEquals(expectedFragmentCount, qwu.getFragments().size() + 1);
}

Example 15 with PlanFragment

use of org.apache.drill.exec.proto.BitControl.PlanFragment in project drill by apache.

the class TestLocalExchange method testHelperVerifyPartitionSenderParallelization.

// Verify the number of partition senders in a major fragments is not more than the cluster size and each endpoint
// in the cluster has at most one fragment from a given major fragment that has the partition sender.
private static void testHelperVerifyPartitionSenderParallelization(String plan, boolean isMuxOn, boolean isDeMuxOn) throws Exception {
    final DrillbitContext drillbitContext = getDrillbitContext();
    final PhysicalPlanReader planReader = drillbitContext.getPlanReader();
    final Fragment rootFragment = PopUnitTestBase.getRootFragmentFromPlanString(planReader, plan);
    final List<Integer> deMuxFragments = Lists.newLinkedList();
    final List<Integer> htrFragments = Lists.newLinkedList();
    final PlanningSet planningSet = new PlanningSet();
    // Create a planningSet to get the assignment of major fragment ids to fragments.
    PARALLELIZER.initFragmentWrappers(rootFragment, planningSet);
    findFragmentsWithPartitionSender(rootFragment, planningSet, deMuxFragments, htrFragments);
    final QueryContextInformation queryContextInfo = Utilities.createQueryContextInfo("dummySchemaName", "938ea2d9-7cb9-4baf-9414-a5a0b7777e8e");
    QueryWorkUnit qwu = PARALLELIZER.getFragments(new OptionList(), drillbitContext.getEndpoint(), QueryId.getDefaultInstance(), drillbitContext.getBits(), planReader, rootFragment, USER_SESSION, queryContextInfo);
    // Make sure the number of minor fragments with HashPartitioner within a major fragment is not more than the
    // number of Drillbits in cluster
    ArrayListMultimap<Integer, DrillbitEndpoint> partitionSenderMap = ArrayListMultimap.create();
    for (PlanFragment planFragment : qwu.getFragments()) {
        if (planFragment.getFragmentJson().contains("hash-partition-sender")) {
            int majorFragmentId = planFragment.getHandle().getMajorFragmentId();
            DrillbitEndpoint assignedEndpoint = planFragment.getAssignment();
            partitionSenderMap.get(majorFragmentId).add(assignedEndpoint);
        }
    }
    if (isMuxOn) {
        verifyAssignment(htrFragments, partitionSenderMap);
    }
    if (isDeMuxOn) {
        verifyAssignment(deMuxFragments, partitionSenderMap);
    }
}