Examples with IndexedFragmentNode org.apache.drill.exec.planner.fragment.Materializer.IndexedFragmentNode used on opensource projects

Example 1 with IndexedFragmentNode

use of org.apache.drill.exec.planner.fragment.Materializer.IndexedFragmentNode in project drill by axbaretto.

the class SplittingParallelizer method generateWorkUnits.

/**
 * Split plan into multiple plans based on parallelization
 * Ideally it is applicable only to plans with two major fragments: Screen and UnionExchange
 * But there could be cases where we can remove even multiple exchanges like in case of "order by"
 * End goal is to get single major fragment: Screen with chain that ends up with a single minor fragment
 * from Leaf Exchange. This way each plan can run independently without any exchange involvement
 * @param options
 * @param foremanNode - not really applicable
 * @param queryId
 * @param reader
 * @param rootNode
 * @param planningSet
 * @param session
 * @param queryContextInfo
 * @return
 * @throws ExecutionSetupException
 */
private List<QueryWorkUnit> generateWorkUnits(OptionList options, DrillbitEndpoint foremanNode, QueryId queryId, PhysicalPlanReader reader, Fragment rootNode, PlanningSet planningSet, UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
    // now we generate all the individual plan fragments and associated assignments. Note, we need all endpoints
    // assigned before we can materialize, so we start a new loop here rather than utilizing the previous one.
    List<QueryWorkUnit> workUnits = Lists.newArrayList();
    int plansCount = 0;
    DrillbitEndpoint[] endPoints = null;
    long initialAllocation = 0;
    final Iterator<Wrapper> iter = planningSet.iterator();
    while (iter.hasNext()) {
        Wrapper wrapper = iter.next();
        Fragment node = wrapper.getNode();
        boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        // get all the needed info from leaf fragment
        if ((physicalOperatorRoot instanceof Exchange) && isLeafFragment) {
            // need to get info about
            // number of minor fragments
            // assignedEndPoints
            // allocation
            plansCount = wrapper.getWidth();
            initialAllocation = (wrapper.getInitialAllocation() != 0) ? wrapper.getInitialAllocation() / plansCount : 0;
            endPoints = new DrillbitEndpoint[plansCount];
            for (int mfId = 0; mfId < plansCount; mfId++) {
                endPoints[mfId] = wrapper.getAssignedEndpoint(mfId);
            }
        }
    }
    if (plansCount == 0) {
        // no exchange, return list of single QueryWorkUnit
        workUnits.add(generateWorkUnit(options, foremanNode, queryId, rootNode, planningSet, session, queryContextInfo));
        return workUnits;
    }
    for (Wrapper wrapper : planningSet) {
        Fragment node = wrapper.getNode();
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        if (physicalOperatorRoot instanceof Exchange) {
            // get to 0 MajorFragment
            continue;
        }
        boolean isRootNode = rootNode == node;
        if (isRootNode && wrapper.getWidth() != 1) {
            throw new ForemanSetupException(String.format("Failure while trying to setup fragment. " + "The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth()));
        }
        // this fragment is always leaf, as we are removing all the exchanges
        boolean isLeafFragment = true;
        FragmentHandle handle = // 
        FragmentHandle.newBuilder().setMajorFragmentId(// 
        wrapper.getMajorFragmentId()).setMinorFragmentId(// minor fragment ID is going to be always 0, as plan will be split
        0).setQueryId(// 
        queryId).build();
        // Create a minorFragment for each major fragment.
        for (int minorFragmentId = 0; minorFragmentId < plansCount; minorFragmentId++) {
            // those fragments should be empty
            List<MinorFragmentDefn> fragments = Lists.newArrayList();
            MinorFragmentDefn rootFragment = null;
            FragmentRoot rootOperator = null;
            IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
            wrapper.resetAllocation();
            // two visitors here
            // 1. To remove exchange
            // 2. To reset operator IDs as exchanges were removed
            PhysicalOperator op = physicalOperatorRoot.accept(ExchangeRemoverMaterializer.INSTANCE, iNode).accept(OperatorIdVisitor.INSTANCE, 0);
            Preconditions.checkArgument(op instanceof FragmentRoot);
            FragmentRoot root = (FragmentRoot) op;
            PlanFragment fragment = // 
            PlanFragment.newBuilder().setForeman(// 
            endPoints[minorFragmentId]).setHandle(// 
            handle).setAssignment(// 
            endPoints[minorFragmentId]).setLeafFragment(// 
            isLeafFragment).setContext(queryContextInfo).setMemInitial(// 
            initialAllocation).setMemMax(// TODO - for some reason OOM is using leaf fragment max allocation divided by width
            wrapper.getMaxAllocation()).setCredentials(session.getCredentials()).addAllCollector(CountRequiredFragments.getCollectors(root)).build();
            MinorFragmentDefn fragmentDefn = new MinorFragmentDefn(fragment, root, options);
            if (isRootNode) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                rootFragment = fragmentDefn;
                rootOperator = root;
            } else {
                if (logger.isDebugEnabled()) {
                    logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                throw new ForemanSetupException(String.format("There should not be non-root/remote fragment present in plan split, but there is:", DrillStringUtils.unescapeJava(fragment.toString())));
            }
            // fragments should be always empty here
            workUnits.add(new QueryWorkUnit(rootOperator, rootFragment, fragments));
        }
    }
    return workUnits;
}

Example 2 with IndexedFragmentNode

use of org.apache.drill.exec.planner.fragment.Materializer.IndexedFragmentNode in project drill by apache.

the class SplittingParallelizer method generateWorkUnits.

/**
 * Split plan into multiple plans based on parallelization
 * Ideally it is applicable only to plans with two major fragments: Screen and UnionExchange
 * But there could be cases where we can remove even multiple exchanges like in case of "order by"
 * End goal is to get single major fragment: Screen with chain that ends up with a single minor fragment
 * from Leaf Exchange. This way each plan can run independently without any exchange involvement
 * @param options
 * @param foremanNode - not really applicable
 * @param queryId
 * @param reader
 * @param rootNode
 * @param planningSet
 * @param session
 * @param queryContextInfo
 * @return
 * @throws ExecutionSetupException
 */
private List<QueryWorkUnit> generateWorkUnits(OptionList options, DrillbitEndpoint foremanNode, QueryId queryId, PhysicalPlanReader reader, Fragment rootNode, PlanningSet planningSet, UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
    // Generate all the individual plan fragments and associated assignments. Note, we need all endpoints
    // assigned before we can materialize.
    List<QueryWorkUnit> workUnits = Lists.newArrayList();
    int plansCount = 0;
    DrillbitEndpoint[] leafFragEndpoints = null;
    long initialAllocation = 0;
    final Iterator<Wrapper> iter = planningSet.iterator();
    while (iter.hasNext()) {
        Wrapper wrapper = iter.next();
        Fragment node = wrapper.getNode();
        boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        // get all the needed info from leaf fragment
        if ((physicalOperatorRoot instanceof Exchange) && isLeafFragment) {
            // need to get info about
            // number of minor fragments
            // assignedEndPoints
            // allocation
            plansCount = wrapper.getWidth();
            initialAllocation = (wrapper.getInitialAllocation() != 0) ? wrapper.getInitialAllocation() / plansCount : 0;
            leafFragEndpoints = new DrillbitEndpoint[plansCount];
            for (int mfId = 0; mfId < plansCount; mfId++) {
                leafFragEndpoints[mfId] = wrapper.getAssignedEndpoint(mfId);
            }
        }
    }
    DrillbitEndpoint[] endPoints = leafFragEndpoints;
    if (plansCount == 0) {
        // no exchange, return list of single QueryWorkUnit
        workUnits.add(generateWorkUnit(options, foremanNode, queryId, rootNode, planningSet, session, queryContextInfo));
        return workUnits;
    }
    for (Wrapper wrapper : planningSet) {
        Fragment node = wrapper.getNode();
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        if (physicalOperatorRoot instanceof Exchange) {
            // get to 0 MajorFragment
            continue;
        }
        boolean isRootNode = rootNode == node;
        if (isRootNode && wrapper.getWidth() != 1) {
            throw new ForemanSetupException(String.format("Failure while trying to setup fragment. " + "The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth()));
        }
        // this fragment is always leaf, as we are removing all the exchanges
        boolean isLeafFragment = true;
        FragmentHandle handle = FragmentHandle.newBuilder().setMajorFragmentId(wrapper.getMajorFragmentId()).setMinorFragmentId(// minor fragment ID is going to be always 0, as plan will be split
        0).setQueryId(queryId).build();
        // Create a minorFragment for each major fragment.
        for (int minorFragmentId = 0; minorFragmentId < plansCount; minorFragmentId++) {
            // those fragments should be empty
            List<MinorFragmentDefn> fragments = Lists.newArrayList();
            MinorFragmentDefn rootFragment = null;
            FragmentRoot rootOperator = null;
            IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper, (fragmentWrapper, minorFragment) -> endPoints[minorFragment], getMemory());
            wrapper.resetAllocation();
            // two visitors here
            // 1. To remove exchange
            // 2. To reset operator IDs as exchanges were removed
            PhysicalOperator op = physicalOperatorRoot.accept(ExchangeRemoverMaterializer.INSTANCE, iNode).accept(OperatorIdVisitor.INSTANCE, 0);
            Preconditions.checkArgument(op instanceof FragmentRoot);
            FragmentRoot root = (FragmentRoot) op;
            PlanFragment fragment = PlanFragment.newBuilder().setForeman(endPoints[minorFragmentId]).setHandle(handle).setAssignment(endPoints[minorFragmentId]).setLeafFragment(isLeafFragment).setContext(queryContextInfo).setMemInitial(initialAllocation).setMemMax(// TODO - for some reason OOM is using leaf fragment max allocation divided by width
            wrapper.getMaxAllocation()).setCredentials(session.getCredentials()).addAllCollector(CountRequiredFragments.getCollectors(root, enableDynamicFC)).build();
            MinorFragmentDefn fragmentDefn = new MinorFragmentDefn(fragment, root, options);
            if (isRootNode) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                rootFragment = fragmentDefn;
                rootOperator = root;
            } else {
                if (logger.isDebugEnabled()) {
                    logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                throw new ForemanSetupException(String.format("There should not be non-root/remote fragment present in plan split, but there is: %s", DrillStringUtils.unescapeJava(fragment.toString())));
            }
            // fragments should be always empty here
            workUnits.add(new QueryWorkUnit(rootOperator, rootFragment, fragments, planningSet.getRootWrapper()));
        }
    }
    return workUnits;
}

Example 3 with IndexedFragmentNode

use of org.apache.drill.exec.planner.fragment.Materializer.IndexedFragmentNode in project drill by apache.

the class SimpleParallelizer method generateWorkUnit.

protected QueryWorkUnit generateWorkUnit(OptionList options, DrillbitEndpoint foremanNode, QueryId queryId, Fragment rootNode, PlanningSet planningSet, UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
    List<MinorFragmentDefn> fragmentDefns = new ArrayList<>();
    MinorFragmentDefn rootFragmentDefn = null;
    FragmentRoot rootOperator = null;
    // assigned before we can materialize.
    for (Wrapper wrapper : planningSet) {
        Fragment node = wrapper.getNode();
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        boolean isRootNode = rootNode == node;
        if (isRootNode && wrapper.getWidth() != 1) {
            throw new ForemanSetupException(String.format("Failure while trying to setup fragment. " + "The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth()));
        }
        // A fragment is self-driven if it doesn't rely on any other exchanges.
        boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
        // Create a minorFragment for each major fragment.
        for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
            IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper, (fragmentWrapper, minorFragment) -> fragmentWrapper.getAssignedEndpoint(minorFragment), getMemory());
            wrapper.resetAllocation();
            PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
            Preconditions.checkArgument(op instanceof FragmentRoot);
            FragmentRoot root = (FragmentRoot) op;
            FragmentHandle handle = FragmentHandle.newBuilder().setMajorFragmentId(wrapper.getMajorFragmentId()).setMinorFragmentId(minorFragmentId).setQueryId(queryId).build();
            PlanFragment fragment = PlanFragment.newBuilder().setForeman(foremanNode).setHandle(handle).setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)).setLeafFragment(isLeafFragment).setContext(queryContextInfo).setMemInitial(wrapper.getInitialAllocation()).setMemMax(wrapper.getMaxAllocation()).setCredentials(session.getCredentials()).addAllCollector(CountRequiredFragments.getCollectors(root, enableDynamicFC)).build();
            MinorFragmentDefn fragmentDefn = new MinorFragmentDefn(fragment, root, options);
            if (isRootNode) {
                logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                rootFragmentDefn = fragmentDefn;
                rootOperator = root;
            } else {
                logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                fragmentDefns.add(fragmentDefn);
            }
        }
    }
    Wrapper rootWrapper = planningSet.getRootWrapper();
    return new QueryWorkUnit(rootOperator, rootFragmentDefn, fragmentDefns, rootWrapper);
}

Example 4 with IndexedFragmentNode

use of org.apache.drill.exec.planner.fragment.Materializer.IndexedFragmentNode in project drill by apache.

the class SimpleParallelizer method generateWorkUnit.

protected QueryWorkUnit generateWorkUnit(OptionList options, DrillbitEndpoint foremanNode, QueryId queryId, PhysicalPlanReader reader, Fragment rootNode, PlanningSet planningSet, UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
    List<PlanFragment> fragments = Lists.newArrayList();
    PlanFragment rootFragment = null;
    FragmentRoot rootOperator = null;
    // assigned before we can materialize, so we start a new loop here rather than utilizing the previous one.
    for (Wrapper wrapper : planningSet) {
        Fragment node = wrapper.getNode();
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        boolean isRootNode = rootNode == node;
        if (isRootNode && wrapper.getWidth() != 1) {
            throw new ForemanSetupException(String.format("Failure while trying to setup fragment. " + "The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth()));
        }
        // a fragment is self driven if it doesn't rely on any other exchanges.
        boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
        // Create a minorFragment for each major fragment.
        for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
            IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
            wrapper.resetAllocation();
            PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
            Preconditions.checkArgument(op instanceof FragmentRoot);
            FragmentRoot root = (FragmentRoot) op;
            // get plan as JSON
            String plan;
            String optionsData;
            try {
                plan = reader.writeJson(root);
                optionsData = reader.writeJson(options);
            } catch (JsonProcessingException e) {
                throw new ForemanSetupException("Failure while trying to convert fragment into json.", e);
            }
            FragmentHandle handle = //
            FragmentHandle.newBuilder().setMajorFragmentId(//
            wrapper.getMajorFragmentId()).setMinorFragmentId(//
            minorFragmentId).setQueryId(//
            queryId).build();
            PlanFragment fragment = //
            PlanFragment.newBuilder().setForeman(//
            foremanNode).setFragmentJson(//
            plan).setHandle(//
            handle).setAssignment(//
            wrapper.getAssignedEndpoint(minorFragmentId)).setLeafFragment(//
            isLeafFragment).setContext(queryContextInfo).setMemInitial(//
            wrapper.getInitialAllocation()).setMemMax(wrapper.getMaxAllocation()).setOptionsJson(optionsData).setCredentials(session.getCredentials()).addAllCollector(CountRequiredFragments.getCollectors(root)).build();
            if (isRootNode) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                rootFragment = fragment;
                rootOperator = root;
            } else {
                if (logger.isDebugEnabled()) {
                    logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                }
                fragments.add(fragment);
            }
        }
    }
    return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}

Example 5 with IndexedFragmentNode

use of org.apache.drill.exec.planner.fragment.Materializer.IndexedFragmentNode in project drill by axbaretto.

the class SimpleParallelizer method generateWorkUnit.

protected QueryWorkUnit generateWorkUnit(OptionList options, DrillbitEndpoint foremanNode, QueryId queryId, Fragment rootNode, PlanningSet planningSet, UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
    List<MinorFragmentDefn> fragmentDefns = new ArrayList<>();
    MinorFragmentDefn rootFragmentDefn = null;
    FragmentRoot rootOperator = null;
    // assigned before we can materialize, so we start a new loop here rather than utilizing the previous one.
    for (Wrapper wrapper : planningSet) {
        Fragment node = wrapper.getNode();
        final PhysicalOperator physicalOperatorRoot = node.getRoot();
        boolean isRootNode = rootNode == node;
        if (isRootNode && wrapper.getWidth() != 1) {
            throw new ForemanSetupException(String.format("Failure while trying to setup fragment. " + "The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth()));
        }
        // a fragment is self driven if it doesn't rely on any other exchanges.
        boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
        // Create a minorFragment for each major fragment.
        for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
            IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
            wrapper.resetAllocation();
            PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
            Preconditions.checkArgument(op instanceof FragmentRoot);
            FragmentRoot root = (FragmentRoot) op;
            FragmentHandle handle = // 
            FragmentHandle.newBuilder().setMajorFragmentId(// 
            wrapper.getMajorFragmentId()).setMinorFragmentId(// 
            minorFragmentId).setQueryId(// 
            queryId).build();
            PlanFragment fragment = // 
            PlanFragment.newBuilder().setForeman(// 
            foremanNode).setHandle(// 
            handle).setAssignment(// 
            wrapper.getAssignedEndpoint(minorFragmentId)).setLeafFragment(// 
            isLeafFragment).setContext(queryContextInfo).setMemInitial(// 
            wrapper.getInitialAllocation()).setMemMax(wrapper.getMaxAllocation()).setCredentials(session.getCredentials()).addAllCollector(CountRequiredFragments.getCollectors(root)).build();
            MinorFragmentDefn fragmentDefn = new MinorFragmentDefn(fragment, root, options);
            if (isRootNode) {
                logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                rootFragmentDefn = fragmentDefn;
                rootOperator = root;
            } else {
                logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
                fragmentDefns.add(fragmentDefn);
            }
        }
    }
    return new QueryWorkUnit(rootOperator, rootFragmentDefn, fragmentDefns);
}