Examples with PlanNode org.teiid.query.optimizer.relational.plantree.PlanNode used on opensource projects

Example 21 with PlanNode

use of org.teiid.query.optimizer.relational.plantree.PlanNode in project teiid by teiid.

the class TestCalculateCostUtil method helpTestEstimateCost.

void helpTestEstimateCost(String critString, float childCost, float expectedResult, QueryMetadataInterface metadata) throws Exception {
    Criteria crit = helpGetCriteria(critString, metadata);
    PlanNode select = RelationalPlanner.createSelectNode(crit, false);
    float resultCost = NewCalculateCostUtil.recursiveEstimateCostOfCriteria(childCost, select, crit, metadata);
    assertEquals((int) expectedResult, (int) resultCost);
}

Example 22 with PlanNode

use of org.teiid.query.optimizer.relational.plantree.PlanNode in project teiid by teiid.

the class RuleImplementJoinStrategy method execute.

/**
 * @see org.teiid.query.optimizer.relational.OptimizerRule#execute(org.teiid.query.optimizer.relational.plantree.PlanNode, org.teiid.query.metadata.QueryMetadataInterface, org.teiid.query.optimizer.capabilities.CapabilitiesFinder, org.teiid.query.optimizer.relational.RuleStack, org.teiid.query.analysis.AnalysisRecord, org.teiid.query.util.CommandContext)
 */
public PlanNode execute(PlanNode plan, QueryMetadataInterface metadata, CapabilitiesFinder capabilitiesFinder, RuleStack rules, AnalysisRecord analysisRecord, CommandContext context) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
    for (PlanNode sourceNode : NodeEditor.findAllNodes(plan, NodeConstants.Types.SOURCE, NodeConstants.Types.ACCESS)) {
        SymbolMap references = (SymbolMap) sourceNode.getProperty(NodeConstants.Info.CORRELATED_REFERENCES);
        if (references != null) {
            Set<GroupSymbol> groups = GroupsUsedByElementsVisitor.getGroups(references.getValues());
            PlanNode joinNode = NodeEditor.findParent(sourceNode, NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE);
            while (joinNode != null) {
                if (joinNode.getGroups().containsAll(groups)) {
                    joinNode.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.NESTED_TABLE);
                    Info info = Info.RIGHT_NESTED_REFERENCES;
                    if (!FrameUtil.findJoinSourceNode(joinNode.getFirstChild()).getGroups().containsAll(groups)) {
                        // $NON-NLS-1$
                        throw new AssertionError("Should not have reordered the join tree to reverse the lateral join");
                    }
                    SymbolMap map = (SymbolMap) joinNode.getProperty(info);
                    if (map == null) {
                        map = new SymbolMap();
                    }
                    joinNode.setProperty(info, map);
                    map.asUpdatableMap().putAll(references.asMap());
                    if (joinNode.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE) != null) {
                        // $NON-NLS-1$
                        throw new AssertionError("Cannot use a depenedent join when the join involves a correlated nested table.");
                    }
                    break;
                }
                joinNode = NodeEditor.findParent(joinNode, NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE);
            }
        }
    }
    for (PlanNode joinNode : NodeEditor.findAllNodes(plan, NodeConstants.Types.JOIN, NodeConstants.Types.ACCESS)) {
        JoinStrategyType stype = (JoinStrategyType) joinNode.getProperty(NodeConstants.Info.JOIN_STRATEGY);
        if (!JoinStrategyType.MERGE.equals(stype)) {
            continue;
        }
        List<Expression> leftExpressions = (List<Expression>) joinNode.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS);
        List<Expression> rightExpressions = (List<Expression>) joinNode.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS);
        int origExpressionCount = leftExpressions.size();
        // check index information on each side
        // TODO: don't do null order compensation - in fact we should check what the order actually is, but we don't have that metadata
        Object key = null;
        boolean right = true;
        // we check the right first, since it should be larger
        if (joinNode.getLastChild().getType() == NodeConstants.Types.ACCESS && NewCalculateCostUtil.isSingleTable(joinNode.getLastChild())) {
            key = NewCalculateCostUtil.getKeyUsed(rightExpressions, null, metadata, null);
        }
        if (key == null && joinNode.getFirstChild().getType() == NodeConstants.Types.ACCESS && NewCalculateCostUtil.isSingleTable(joinNode.getFirstChild())) {
            key = NewCalculateCostUtil.getKeyUsed(leftExpressions, null, metadata, null);
            right = false;
        }
        JoinType joinType = (JoinType) joinNode.getProperty(NodeConstants.Info.JOIN_TYPE);
        /**
         * Don't push sorts for unbalanced inner joins, we prefer to use a processing time cost based decision
         */
        boolean pushLeft = true;
        boolean pushRight = true;
        if ((joinType == JoinType.JOIN_INNER || joinType == JoinType.JOIN_LEFT_OUTER) && context != null) {
            float leftCost = NewCalculateCostUtil.computeCostForTree(joinNode.getFirstChild(), metadata);
            float rightCost = NewCalculateCostUtil.computeCostForTree(joinNode.getLastChild(), metadata);
            if (leftCost != NewCalculateCostUtil.UNKNOWN_VALUE && rightCost != NewCalculateCostUtil.UNKNOWN_VALUE && (leftCost > context.getProcessorBatchSize() || rightCost > context.getProcessorBatchSize())) {
                // we use a larger constant here to ensure that we don't unwisely prevent pushdown
                pushLeft = leftCost < context.getProcessorBatchSize() || leftCost / rightCost < 8 || (key != null && !right);
                pushRight = rightCost < context.getProcessorBatchSize() || rightCost / leftCost < 8 || joinType == JoinType.JOIN_LEFT_OUTER || (key != null && right);
            }
        }
        if (key != null && joinNode.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE) == null) {
            // redo the join predicates based upon the key alone
            List<Object> keyCols = metadata.getElementIDsInKey(key);
            int[] reorder = new int[keyCols.size()];
            LinkedHashSet<Integer> toCriteria = new LinkedHashSet<Integer>();
            List<Expression> keyExpressions = right ? rightExpressions : leftExpressions;
            Map<Object, Integer> indexMap = new LinkedHashMap<Object, Integer>();
            for (int i = 0; i < keyExpressions.size(); i++) {
                Expression ses = keyExpressions.get(i);
                if (!(ses instanceof ElementSymbol)) {
                    toCriteria.add(i);
                    continue;
                }
                Integer existing = indexMap.put(((ElementSymbol) ses).getMetadataID(), i);
                if (existing != null) {
                    toCriteria.add(existing);
                }
            }
            boolean found = true;
            for (int i = 0; i < keyCols.size(); i++) {
                Object id = keyCols.get(i);
                Integer index = indexMap.remove(id);
                if (index == null) {
                    found = false;
                    break;
                }
                reorder[i] = index;
            }
            if (found) {
                toCriteria.addAll(indexMap.values());
                List<Criteria> joinCriteria = (List<Criteria>) joinNode.getProperty(Info.NON_EQUI_JOIN_CRITERIA);
                for (int index : toCriteria) {
                    Expression lses = leftExpressions.get(index);
                    Expression rses = rightExpressions.get(index);
                    CompareCriteria cc = new CompareCriteria(lses, CompareCriteria.EQ, rses);
                    if (joinCriteria == null || joinCriteria.isEmpty()) {
                        joinCriteria = new ArrayList<Criteria>();
                    }
                    joinCriteria.add(cc);
                }
                joinNode.setProperty(Info.NON_EQUI_JOIN_CRITERIA, joinCriteria);
                leftExpressions = RelationalNode.projectTuple(reorder, leftExpressions);
                rightExpressions = RelationalNode.projectTuple(reorder, rightExpressions);
                joinNode.setProperty(NodeConstants.Info.LEFT_EXPRESSIONS, leftExpressions);
                joinNode.setProperty(NodeConstants.Info.RIGHT_EXPRESSIONS, rightExpressions);
            }
        }
        boolean pushedLeft = insertSort(joinNode.getFirstChild(), leftExpressions, joinNode, metadata, capabilitiesFinder, pushLeft, context);
        if (origExpressionCount == 1 && joinType == JoinType.JOIN_INNER && joinNode.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE) != null && !joinNode.hasCollectionProperty(Info.NON_EQUI_JOIN_CRITERIA)) {
            Collection<Expression> output = (Collection<Expression>) joinNode.getProperty(NodeConstants.Info.OUTPUT_COLS);
            Collection<GroupSymbol> groups = GroupsUsedByElementsVisitor.getGroups(output);
            if (Collections.disjoint(groups, FrameUtil.findJoinSourceNode(joinNode.getFirstChild()).getGroups())) {
                pushRight = false;
                joinNode.setProperty(Info.IS_SEMI_DEP, Boolean.TRUE);
            }
        }
        boolean pushedRight = insertSort(joinNode.getLastChild(), rightExpressions, joinNode, metadata, capabilitiesFinder, pushRight, context);
        if ((!pushedRight || !pushedLeft) && (joinType == JoinType.JOIN_INNER || (joinType == JoinType.JOIN_LEFT_OUTER && !pushedLeft))) {
            joinNode.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.ENHANCED_SORT);
        }
    }
    return plan;
}

Example 23 with PlanNode

use of org.teiid.query.optimizer.relational.plantree.PlanNode in project teiid by teiid.

the class RuleMergeCriteria method planMergeJoin.

/**
 * Look for:
 * [NOT] EXISTS ( )
 * IN ( ) / SOME ( )
 *
 * and replace with a semi join
 */
private PlanNode planMergeJoin(PlanNode current, PlanNode root) throws QueryMetadataException, TeiidComponentException {
    float sourceCost = NewCalculateCostUtil.computeCostForTree(current.getFirstChild(), metadata);
    Criteria crit = (Criteria) current.getProperty(NodeConstants.Info.SELECT_CRITERIA);
    PlannedResult plannedResult = findSubquery(crit, true);
    if (plannedResult.query == null) {
        return current;
    }
    if (sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && sourceCost < RuleChooseDependent.DEFAULT_INDEPENDENT_CARDINALITY && !plannedResult.mergeJoin) {
        // TODO: see if a dependent join applies the other direction
        return current;
    }
    RelationalPlan originalPlan = (RelationalPlan) plannedResult.query.getProcessorPlan();
    Number originalCardinality = originalPlan.getRootNode().getEstimateNodeCardinality();
    if (!plannedResult.mergeJoin && originalCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE) {
        // if it's currently unknown, removing criteria won't make it any better
        return current;
    }
    Collection<GroupSymbol> leftGroups = FrameUtil.findJoinSourceNode(current).getGroups();
    if (!planQuery(leftGroups, false, plannedResult)) {
        if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) {
            // $NON-NLS-1$ //$NON-NLS-2$
            this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join: " + crit, "ignoring MJ hint", Priority.HIGH));
        }
        return current;
    }
    // check if the child is already ordered.  TODO: see if the ordering is compatible.
    PlanNode childSort = NodeEditor.findNodePreOrder(root, NodeConstants.Types.SORT, NodeConstants.Types.SOURCE | NodeConstants.Types.JOIN);
    if (childSort != null) {
        if (plannedResult.mergeJoin && analysisRecord != null && analysisRecord.recordAnnotations()) {
            // $NON-NLS-1$ //$NON-NLS-2$
            this.analysisRecord.addAnnotation(new Annotation(Annotation.HINTS, "Could not plan as a merge join since the parent join requires a sort: " + crit, "ignoring MJ hint", Priority.HIGH));
        }
        return current;
    }
    // add an order by, which hopefully will get pushed down
    plannedResult.query.setOrderBy(new OrderBy(plannedResult.rightExpressions).clone());
    for (OrderByItem item : plannedResult.query.getOrderBy().getOrderByItems()) {
        int index = plannedResult.query.getProjectedSymbols().indexOf(item.getSymbol());
        if (index >= 0 && !(item.getSymbol() instanceof ElementSymbol)) {
            item.setSymbol((Expression) plannedResult.query.getProjectedSymbols().get(index).clone());
        }
        item.setExpressionPosition(index);
    }
    try {
        // clone the symbols as they may change during planning
        List<Expression> projectedSymbols = LanguageObject.Util.deepClone(plannedResult.query.getProjectedSymbols(), Expression.class);
        // NOTE: we could tap into the relationalplanner at a lower level to get this in a plan node form,
        // the major benefit would be to reuse the dependent join planning logic if possible.
        RelationalPlan subPlan = (RelationalPlan) QueryOptimizer.optimizePlan(plannedResult.query, metadata, idGenerator, capFinder, analysisRecord, context);
        Number planCardinality = subPlan.getRootNode().getEstimateNodeCardinality();
        if (!plannedResult.mergeJoin) {
            // if we don't have a specific hint, then use costing
            if (planCardinality.floatValue() == NewCalculateCostUtil.UNKNOWN_VALUE || planCardinality.floatValue() > 10000000 || (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() > 1000) || (sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && sourceCost * originalCardinality.floatValue() < planCardinality.floatValue() / (100 * Math.log(Math.max(4, sourceCost))))) {
                // bail-out if both are unknown or the new plan is too large
                if (analysisRecord != null && analysisRecord.recordDebug()) {
                    // $NON-NLS-1$ //$NON-NLS-2$
                    current.recordDebugAnnotation("cost of merge join plan was not favorable", null, "semi merge join will not be used", analysisRecord, metadata);
                }
                return current;
            }
        }
        // assume dependent
        if ((sourceCost != NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() != NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() < sourceCost / 8) || (sourceCost == NewCalculateCostUtil.UNKNOWN_VALUE && planCardinality.floatValue() <= 1000)) {
            plannedResult.makeInd = true;
        }
        /*if (plannedResult.makeInd 
					&& plannedResult.query.getCorrelatedReferences() == null
					&& !plannedResult.not
					&& plannedResult.leftExpressions.size() == 1) {
            	//TODO: this should just be a dependent criteria node to avoid sorts
            }*/
        // $NON-NLS-1$ //$NON-NLS-2$
        current.recordDebugAnnotation("Conditions met (hint or cost)", null, "Converting to a semi merge join", analysisRecord, metadata);
        PlanNode semiJoin = NodeFactory.getNewNode(NodeConstants.Types.JOIN);
        semiJoin.addGroups(current.getGroups());
        Set<GroupSymbol> groups = GroupsUsedByElementsVisitor.getGroups(plannedResult.rightExpressions);
        semiJoin.addGroups(groups);
        semiJoin.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE);
        semiJoin.setProperty(NodeConstants.Info.JOIN_TYPE, plannedResult.not ? JoinType.JOIN_ANTI_SEMI : JoinType.JOIN_SEMI);
        semiJoin.setProperty(NodeConstants.Info.NON_EQUI_JOIN_CRITERIA, plannedResult.nonEquiJoinCriteria);
        List<Criteria> joinCriteria = new ArrayList<Criteria>();
        joinCriteria.addAll(plannedResult.nonEquiJoinCriteria);
        for (int i = 0; i < plannedResult.leftExpressions.size(); i++) {
            joinCriteria.add(new CompareCriteria((Expression) plannedResult.rightExpressions.get(i), CompareCriteria.EQ, (Expression) plannedResult.leftExpressions.get(i)));
        }
        semiJoin.setProperty(NodeConstants.Info.JOIN_CRITERIA, joinCriteria);
        // nested subqueries are possibly being promoted, so they need their references updated
        List<SymbolMap> refMaps = semiJoin.getAllReferences();
        SymbolMap parentRefs = plannedResult.query.getCorrelatedReferences();
        for (SymbolMap refs : refMaps) {
            for (Map.Entry<ElementSymbol, Expression> ref : refs.asUpdatableMap().entrySet()) {
                Expression expr = ref.getValue();
                if (expr instanceof ElementSymbol) {
                    Expression convertedExpr = parentRefs.getMappedExpression((ElementSymbol) expr);
                    if (convertedExpr != null) {
                        ref.setValue(convertedExpr);
                    }
                }
                semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(ref.getValue()));
            }
        }
        semiJoin.setProperty(NodeConstants.Info.LEFT_EXPRESSIONS, plannedResult.leftExpressions);
        semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(plannedResult.leftExpressions));
        semiJoin.setProperty(NodeConstants.Info.RIGHT_EXPRESSIONS, plannedResult.rightExpressions);
        semiJoin.getGroups().addAll(GroupsUsedByElementsVisitor.getGroups(plannedResult.rightExpressions));
        semiJoin.setProperty(NodeConstants.Info.SORT_RIGHT, SortOption.ALREADY_SORTED);
        semiJoin.setProperty(NodeConstants.Info.OUTPUT_COLS, root.getProperty(NodeConstants.Info.OUTPUT_COLS));
        List childOutput = (List) current.getFirstChild().getProperty(NodeConstants.Info.OUTPUT_COLS);
        PlanNode toCorrect = root;
        while (toCorrect != current) {
            toCorrect.setProperty(NodeConstants.Info.OUTPUT_COLS, childOutput);
            toCorrect = toCorrect.getFirstChild();
        }
        PlanNode node = NodeFactory.getNewNode(NodeConstants.Types.ACCESS);
        node.setProperty(NodeConstants.Info.PROCESSOR_PLAN, subPlan);
        node.setProperty(NodeConstants.Info.OUTPUT_COLS, projectedSymbols);
        node.setProperty(NodeConstants.Info.EST_CARDINALITY, planCardinality);
        node.addGroups(groups);
        root.addAsParent(semiJoin);
        semiJoin.addLastChild(node);
        PlanNode result = current.getParent();
        NodeEditor.removeChildNode(result, current);
        RuleImplementJoinStrategy.insertSort(semiJoin.getFirstChild(), (List<Expression>) plannedResult.leftExpressions, semiJoin, metadata, capFinder, true, context);
        if (plannedResult.makeInd && !plannedResult.not) {
            // TODO: would like for an enhanced sort merge with the semi dep option to avoid the sorting
            // this is a little different than a typical dependent join in that the right is the independent side
            String id = RuleChooseDependent.nextId();
            PlanNode dep = RuleChooseDependent.getDependentCriteriaNode(id, plannedResult.rightExpressions, plannedResult.leftExpressions, node, metadata, null, false, null);
            semiJoin.getFirstChild().addAsParent(dep);
            semiJoin.setProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE, id);
            this.dependent = true;
        }
        return result;
    } catch (QueryPlannerException e) {
        // can't be done - probably access patterns - what about dependent
        return current;
    }
}

Example 24 with PlanNode

use of org.teiid.query.optimizer.relational.plantree.PlanNode in project teiid by teiid.

the class RuleMergeCriteria method findCriteriaChains.

/**
 * Walk the tree pre-order, looking for any chains of criteria
 * @param node Root node to search
 * @param foundNodes Roots of criteria chains
 */
void findCriteriaChains(PlanNode root, List<PlanNode> foundNodes, AnalysisRecord analysisRecord) throws QueryPlannerException, TeiidComponentException {
    PlanNode recurseRoot = root;
    if (root.getType() == NodeConstants.Types.SELECT) {
        // Walk to end of the chain and change recurse root
        while (recurseRoot.getType() == NodeConstants.Types.SELECT) {
            // Look for opportunities to replace with a semi-join
            recurseRoot = planMergeJoin(recurseRoot, root);
            if (root.getChildCount() == 0) {
                root = recurseRoot.getFirstChild();
                if (root.getType() != NodeConstants.Types.SELECT) {
                    root = root.getParent();
                }
            }
            recurseRoot = recurseRoot.getFirstChild();
        }
        // Ignore trivial 1-node case
        if (recurseRoot.getParent() != root) {
            // Found root for chain
            foundNodes.add(root);
        }
    }
    if (recurseRoot.getType() != NodeConstants.Types.ACCESS) {
        for (PlanNode child : recurseRoot.getChildren()) {
            findCriteriaChains(child, foundNodes, analysisRecord);
        }
    }
}

Example 25 with PlanNode

use of org.teiid.query.optimizer.relational.plantree.PlanNode in project teiid by teiid.

the class RuleMergeCriteria method mergeChain.

static void mergeChain(PlanNode chainRoot, QueryMetadataInterface metadata) {
    // Remove all of chain except root, collect crit from each
    CompoundCriteria critParts = new CompoundCriteria();
    LinkedList<Criteria> subqueryCriteria = new LinkedList<Criteria>();
    PlanNode current = chainRoot;
    boolean isDependentSet = false;
    while (current.getType() == NodeConstants.Types.SELECT) {
        if (!current.getCorrelatedReferenceElements().isEmpty()) {
            // add at the end for delayed evaluation
            subqueryCriteria.add(0, (Criteria) current.getProperty(NodeConstants.Info.SELECT_CRITERIA));
        } else {
            critParts.getCriteria().add(0, (Criteria) current.getProperty(NodeConstants.Info.SELECT_CRITERIA));
        }
        isDependentSet |= current.hasBooleanProperty(NodeConstants.Info.IS_DEPENDENT_SET);
        // Recurse
        PlanNode last = current;
        current = current.getLastChild();
        // Remove current
        if (last != chainRoot) {
            NodeEditor.removeChildNode(last.getParent(), last);
        }
    }
    critParts.getCriteria().addAll(subqueryCriteria);
    Criteria combinedCrit = QueryRewriter.optimizeCriteria(critParts, metadata);
    if (isDependentSet) {
        chainRoot.setProperty(NodeConstants.Info.IS_DEPENDENT_SET, Boolean.TRUE);
    }
    // Replace criteria at root with new combined criteria
    chainRoot.setProperty(NodeConstants.Info.SELECT_CRITERIA, combinedCrit);
    // Reset group for node based on combined criteria
    chainRoot.getGroups().clear();
    chainRoot.addGroups(GroupsUsedByElementsVisitor.getGroups(combinedCrit));
    chainRoot.addGroups(GroupsUsedByElementsVisitor.getGroups(chainRoot.getCorrelatedReferenceElements()));
}