Examples with SymbolMap org.teiid.query.sql.util.SymbolMap used on opensource projects

Example 11 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class RulePushAggregates method pushGroupNodeOverUnion.

/**
 * The plan tree looks like:
 * group [agg(x), {a, b}]
 *   source
 *     set op
 *       child 1
 *       ...
 *
 * we need to make it into
 *
 * group [agg(agg(x)), {a, b}]
 *   source
 *     set op
 *       project
 *         [select]
 *           group [agg(x), {a, b}]
 *             source
 *               child 1
 *       ...
 *
 * Or if the child does not support pushdown we add dummy aggregate projection
 * count(*) = 1, count(x) = case x is null then 0 else 1 end, avg(x) = x, etc.
 */
private void pushGroupNodeOverUnion(QueryMetadataInterface metadata, CapabilitiesFinder capFinder, PlanNode groupNode, PlanNode unionSourceParent, List<Expression> groupingExpressions, PlanNode setOp, AnalysisRecord record) throws TeiidComponentException, QueryMetadataException, QueryPlannerException, QueryResolverException {
    if (setOp == null || setOp.getProperty(NodeConstants.Info.SET_OPERATION) != Operation.UNION) {
        return;
    }
    LinkedHashSet<AggregateSymbol> aggregates = collectAggregates(groupNode);
    Map<ElementSymbol, List<Set<Constant>>> partitionInfo = (Map<ElementSymbol, List<Set<Constant>>>) unionSourceParent.getProperty(Info.PARTITION_INFO);
    // check to see if any aggregate is dependent upon cardinality
    boolean cardinalityDependent = AggregateSymbol.areAggregatesCardinalityDependent(aggregates);
    LinkedList<PlanNode> unionChildren = new LinkedList<PlanNode>();
    findUnionChildren(unionChildren, cardinalityDependent, setOp);
    SymbolMap parentMap = (SymbolMap) unionSourceParent.getProperty(NodeConstants.Info.SYMBOL_MAP);
    // partitioned union
    if (partitionInfo != null && !Collections.disjoint(partitionInfo.keySet(), groupingExpressions)) {
        decomposeGroupBy(groupNode, unionSourceParent, groupingExpressions, aggregates, unionChildren, parentMap, metadata, capFinder);
        return;
    }
    /*
		 * if there are no aggregates, this is just duplicate removal
		 * mark the union as not all, which should be removed later but
		 * serves as a hint to distribute a distinct to the union queries
		 */
    if (aggregates.isEmpty()) {
        if (!groupingExpressions.isEmpty()) {
            Set<Expression> expressions = new HashSet<Expression>();
            boolean allCols = true;
            for (Expression ex : groupingExpressions) {
                if (!(ex instanceof ElementSymbol)) {
                    allCols = false;
                    break;
                }
                Expression mapped = parentMap.getMappedExpression((ElementSymbol) ex);
                expressions.add(mapped);
            }
            if (allCols) {
                PlanNode project = NodeEditor.findNodePreOrder(unionSourceParent, NodeConstants.Types.PROJECT);
                boolean projectsGrouping = true;
                for (Expression ex : (List<Expression>) project.getProperty(Info.PROJECT_COLS)) {
                    if (!expressions.contains(SymbolMap.getExpression(ex))) {
                        projectsGrouping = false;
                        break;
                    }
                }
                if (projectsGrouping) {
                    // since there are no expressions in the grouping cols, we know the grouping node is now not needed.
                    RuleAssignOutputElements.removeGroupBy(groupNode, metadata);
                    setOp.setProperty(NodeConstants.Info.USE_ALL, Boolean.FALSE);
                }
            }
        }
        return;
    }
    for (AggregateSymbol agg : aggregates) {
        if (!agg.canStage()) {
            return;
        }
    }
    if (unionChildren.size() < 2) {
        return;
    }
    List<AggregateSymbol> copy = new ArrayList<AggregateSymbol>(aggregates);
    aggregates.clear();
    Map<AggregateSymbol, Expression> aggMap = buildAggregateMap(copy, metadata, aggregates, false);
    boolean shouldPushdown = false;
    List<Boolean> pushdownList = new ArrayList<Boolean>(unionChildren.size());
    for (PlanNode planNode : unionChildren) {
        boolean pushdown = canPushGroupByToUnionChild(metadata, capFinder, groupingExpressions, aggregates, planNode, record, groupNode);
        pushdownList.add(pushdown);
        shouldPushdown |= pushdown;
    }
    if (!shouldPushdown) {
        return;
    }
    GroupSymbol group = unionSourceParent.getGroups().iterator().next().clone();
    Iterator<Boolean> pushdownIterator = pushdownList.iterator();
    boolean first = true;
    for (PlanNode planNode : unionChildren) {
        addUnionGroupBy(groupingExpressions, aggregates, parentMap, metadata, capFinder, group, first, planNode, !pushdownIterator.next(), false);
        first = false;
    }
    updateParentAggs(groupNode, aggMap, metadata);
    List<Expression> symbols = (List<Expression>) NodeEditor.findNodePreOrder(unionSourceParent, NodeConstants.Types.PROJECT).getProperty(Info.PROJECT_COLS);
    GroupSymbol modifiedGroup = group.clone();
    SymbolMap symbolMap = createSymbolMap(modifiedGroup, symbols, unionSourceParent, metadata);
    unionSourceParent.setProperty(Info.SYMBOL_MAP, symbolMap);
    // correct the parent frame
    Map<Expression, ElementSymbol> mapping = new HashMap<Expression, ElementSymbol>();
    Iterator<ElementSymbol> elemIter = symbolMap.getKeys().iterator();
    for (Expression expr : groupingExpressions) {
        mapping.put(expr, elemIter.next());
    }
    for (AggregateSymbol agg : aggregates) {
        mapping.put(agg, elemIter.next());
    }
    PlanNode node = unionSourceParent;
    while (node != groupNode.getParent()) {
        FrameUtil.convertNode(node, null, null, mapping, metadata, false);
        node = node.getParent();
    }
    removeUnnecessaryViews(unionSourceParent, metadata, capFinder);
}

Example 12 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class RulePushAggregates method decomposeGroupBy.

/* if partitioned, then we don't need decomposition or the top level group by
     * 
	 *   source
	 *     set op
	 *       project
	 *           group [agg(x), {a, b}]
	 *             source
	 *               child 1
	 *       ...
     * 
     */
private void decomposeGroupBy(PlanNode groupNode, PlanNode sourceNode, List<Expression> groupingExpressions, LinkedHashSet<AggregateSymbol> aggregates, LinkedList<PlanNode> unionChildren, SymbolMap parentMap, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) throws QueryPlannerException, QueryMetadataException, TeiidComponentException, QueryResolverException {
    // remove the group node
    groupNode.getParent().replaceChild(groupNode, groupNode.getFirstChild());
    GroupSymbol group = sourceNode.getGroups().iterator().next().clone();
    boolean first = true;
    for (PlanNode planNode : unionChildren) {
        addUnionGroupBy(groupingExpressions, aggregates, parentMap, metadata, capFinder, group, first, planNode, false, true);
        first = false;
    }
    List<Expression> symbols = (List<Expression>) NodeEditor.findNodePreOrder(sourceNode, NodeConstants.Types.PROJECT).getProperty(Info.PROJECT_COLS);
    GroupSymbol modifiedGroup = group.clone();
    SymbolMap symbolMap = createSymbolMap(modifiedGroup, symbols, sourceNode, metadata);
    sourceNode.setProperty(Info.SYMBOL_MAP, symbolMap);
    // map from the anon group to the updated inline view group
    SymbolMap map = (SymbolMap) groupNode.getProperty(Info.SYMBOL_MAP);
    Map<Expression, ElementSymbol> inverse = map.inserseMapping();
    SymbolMap newMapping = (SymbolMap) NodeEditor.findNodePreOrder(sourceNode, NodeConstants.Types.GROUP).getProperty(Info.SYMBOL_MAP);
    GroupSymbol oldGroup = null;
    Map<ElementSymbol, ElementSymbol> updatedMapping = new HashMap<ElementSymbol, ElementSymbol>();
    for (Map.Entry<ElementSymbol, Expression> entry : symbolMap.asMap().entrySet()) {
        Expression ex = newMapping.getMappedExpression((ElementSymbol) entry.getValue());
        ElementSymbol orig = inverse.get(ex);
        oldGroup = orig.getGroupSymbol();
        updatedMapping.put(orig, entry.getKey());
    }
    FrameUtil.convertFrame(sourceNode, oldGroup, Collections.singleton(modifiedGroup), updatedMapping, metadata);
    removeUnnecessaryViews(sourceNode, metadata, capFinder);
}

Example 13 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class NewCalculateCostUtil method estimateSourceNodeCost.

/**
 * For a source node, the cost is basically the cardinality of the source
 * (if it is known).
 * @param node
 * @param metadata
 * @throws QueryMetadataException
 * @throws TeiidComponentException
 */
private static void estimateSourceNodeCost(PlanNode node, QueryMetadataInterface metadata) throws QueryMetadataException, TeiidComponentException {
    float cost = UNKNOWN_VALUE;
    if (node.getChildCount() > 0) {
        SymbolMap references = (SymbolMap) node.getProperty(NodeConstants.Info.CORRELATED_REFERENCES);
        // only cost non-correlated TODO: a better estimate for correlated
        if (references == null) {
            PlanNode child = node.getFirstChild();
            cost = child.getCardinality();
            SymbolMap symbolMap = (SymbolMap) node.getProperty(NodeConstants.Info.SYMBOL_MAP);
            if (symbolMap != null) {
                ColStats colStats = (ColStats) child.getProperty(Info.EST_COL_STATS);
                if (colStats != null) {
                    List<? extends Expression> outputCols = getOutputCols(node, metadata);
                    ColStats newColStats = new ColStats();
                    for (Expression expr : outputCols) {
                        if (!(expr instanceof ElementSymbol)) {
                            continue;
                        }
                        ElementSymbol es = (ElementSymbol) expr;
                        Expression ex = symbolMap.getMappedExpression(es);
                        float[] value = colStats.get(ex);
                        if (value == null) {
                            Collection<ElementSymbol> elems = ElementCollectorVisitor.getElements(ex, true);
                            value = new float[3];
                            value[Stat.NDV.ordinal()] = getStat(Stat.NDV, elems, node, cost, metadata);
                            value[Stat.NDV_HIGH.ordinal()] = getStat(Stat.NDV_HIGH, elems, node, cost, metadata);
                            value[Stat.NNV.ordinal()] = getStat(Stat.NNV, elems, node, cost, metadata);
                        }
                        newColStats.put(es, value);
                    }
                    node.setProperty(Info.EST_COL_STATS, newColStats);
                } else {
                    colStats = createColStats(node, metadata, cost);
                    node.setProperty(Info.EST_COL_STATS, colStats);
                }
            }
        }
    } else {
        GroupSymbol group = node.getGroups().iterator().next();
        float cardinality = metadata.getCardinality(group.getMetadataID());
        if (cardinality <= QueryMetadataInterface.UNKNOWN_CARDINALITY) {
            if (group.isTempTable() && metadata.getModelID(group.getMetadataID()) == TempMetadataAdapter.TEMP_MODEL) {
                // this should be with-in the scope of a procedure or an undefined size common table
                // 
                // the typical assumption is that this should drive other joins, thus assume
                // a relatively small number of rows.  This is a relatively safe assumption
                // as we do not need parallel processing with the temp fetch and the
                // dependent join backoff should prevent unacceptable performance
                // 
                // another strategy (that is generally applicable) is to delay the full affect of dependent join planning
                // until the size is known - however that is somewhat complicated with the current WITH logic
                // as the table is loaded on demand
                cardinality = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE;
            } else {
                cardinality = UNKNOWN_VALUE;
            }
        }
        cost = cardinality;
        if (!node.hasProperty(Info.ATOMIC_REQUEST)) {
            ColStats colStats = createColStats(node, metadata, cost);
            node.setProperty(Info.EST_COL_STATS, colStats);
        }
    }
    setCardinalityEstimate(node, new Float(cost), false, metadata);
}

Example 14 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class NewCalculateCostUtil method getOutputCols.

private static List<? extends Expression> getOutputCols(PlanNode node, QueryMetadataInterface metadata) throws QueryMetadataException, TeiidComponentException {
    List<Expression> outputCols = (List<Expression>) node.getProperty(Info.OUTPUT_COLS);
    if (outputCols != null) {
        return outputCols;
    }
    PlanNode projectNode = NodeEditor.findNodePreOrder(node, NodeConstants.Types.PROJECT | NodeConstants.Types.GROUP | NodeConstants.Types.SOURCE | NodeConstants.Types.JOIN | NodeConstants.Types.NULL);
    if (projectNode != null) {
        node = projectNode;
    }
    if (node.getType() == NodeConstants.Types.PROJECT) {
        return (List<? extends Expression>) node.getProperty(NodeConstants.Info.PROJECT_COLS);
    } else if (node.getType() == NodeConstants.Types.GROUP) {
        SymbolMap map = (SymbolMap) node.getProperty(Info.SYMBOL_MAP);
        return map.getKeys();
    }
    LinkedList<ElementSymbol> elements = new LinkedList<ElementSymbol>();
    for (GroupSymbol group : node.getGroups()) {
        elements.addAll(ResolverUtil.resolveElementsInGroup(group, metadata));
    }
    return elements;
}

Example 15 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class RuleAssignOutputElements method collectRequiredInputSymbols.

/**
 * Collect all required input symbols for a given node.  Input symbols
 * are any symbols that are required in the processing of this node,
 * for instance to create a new element symbol or sort on it, etc.
 * @param node Node to collect for
 * @param metadata
 * @param capFinder
 * @throws TeiidComponentException
 * @throws QueryMetadataException
 */
private List<Expression> collectRequiredInputSymbols(PlanNode node, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) throws QueryMetadataException, TeiidComponentException {
    Set<Expression> requiredSymbols = new LinkedHashSet<Expression>();
    Set<Expression> createdSymbols = new HashSet<Expression>();
    List<Expression> outputCols = (List<Expression>) node.getProperty(NodeConstants.Info.OUTPUT_COLS);
    switch(node.getType()) {
        case NodeConstants.Types.PROJECT:
            {
                List<Expression> projectCols = (List<Expression>) node.getProperty(NodeConstants.Info.PROJECT_COLS);
                PlanNode accessParent = NodeEditor.findParent(node, NodeConstants.Types.ACCESS);
                PlanNode accessNode = null;
                if (accessParent == null) {
                    // find the direct access node
                    accessNode = NodeEditor.findNodePreOrder(node, NodeConstants.Types.ACCESS, NodeConstants.Types.SOURCE | NodeConstants.Types.JOIN | NodeConstants.Types.SET_OP | NodeConstants.Types.GROUP);
                }
                for (Expression ss : projectCols) {
                    if (ss instanceof AliasSymbol) {
                        createdSymbols.add(ss);
                        ss = ((AliasSymbol) ss).getSymbol();
                    }
                    if (ss instanceof WindowFunction || ss instanceof ExpressionSymbol) {
                        createdSymbols.add(ss);
                    }
                    if (!pushProjection(node, metadata, capFinder, requiredSymbols, accessParent, accessNode, ss)) {
                        ElementCollectorVisitor.getElements(ss, requiredSymbols);
                    }
                }
                break;
            }
        case NodeConstants.Types.SELECT:
            Criteria selectCriteria = (Criteria) node.getProperty(NodeConstants.Info.SELECT_CRITERIA);
            ElementCollectorVisitor.getElements(selectCriteria, requiredSymbols);
            break;
        case NodeConstants.Types.JOIN:
            List<Criteria> crits = (List) node.getProperty(NodeConstants.Info.JOIN_CRITERIA);
            if (crits != null) {
                for (Criteria joinCriteria : crits) {
                    ElementCollectorVisitor.getElements(joinCriteria, requiredSymbols);
                }
            }
            break;
        case NodeConstants.Types.GROUP:
            List<Expression> groupCols = (List<Expression>) node.getProperty(NodeConstants.Info.GROUP_COLS);
            PlanNode accessParent = NodeEditor.findParent(node, NodeConstants.Types.ACCESS);
            PlanNode accessNode = null;
            if (accessParent == null) {
                // find the direct access node
                accessNode = NodeEditor.findNodePreOrder(node.getFirstChild(), NodeConstants.Types.ACCESS, NodeConstants.Types.SOURCE | NodeConstants.Types.JOIN | NodeConstants.Types.SET_OP | NodeConstants.Types.GROUP);
            }
            if (groupCols != null) {
                for (Expression expression : groupCols) {
                    if (!pushProjection(node, metadata, capFinder, requiredSymbols, accessParent, accessNode, expression)) {
                        ElementCollectorVisitor.getElements(expression, requiredSymbols);
                    }
                }
            }
            SymbolMap symbolMap = (SymbolMap) node.getProperty(NodeConstants.Info.SYMBOL_MAP);
            Set<ElementSymbol> usedAggregates = new HashSet<ElementSymbol>();
            // Take credit for creating any aggregates that are needed above
            for (Expression outputSymbol : outputCols) {
                if (!(outputSymbol instanceof ElementSymbol)) {
                    continue;
                }
                createdSymbols.add(outputSymbol);
                Expression ex = symbolMap.getMappedExpression((ElementSymbol) outputSymbol);
                if (ex instanceof AggregateSymbol) {
                    AggregateSymbol agg = (AggregateSymbol) ex;
                    Expression[] aggExprs = agg.getArgs();
                    for (Expression expression : aggExprs) {
                        if (!pushProjection(node, metadata, capFinder, requiredSymbols, accessParent, accessNode, expression)) {
                            ElementCollectorVisitor.getElements(expression, requiredSymbols);
                        }
                    }
                    OrderBy orderBy = agg.getOrderBy();
                    if (orderBy != null) {
                        ElementCollectorVisitor.getElements(orderBy, requiredSymbols);
                    }
                    Expression condition = agg.getCondition();
                    if (condition != null) {
                        ElementCollectorVisitor.getElements(condition, requiredSymbols);
                    }
                    usedAggregates.add((ElementSymbol) outputSymbol);
                }
            }
            // update the aggs in the symbolmap
            for (Map.Entry<ElementSymbol, Expression> entry : new ArrayList<Map.Entry<ElementSymbol, Expression>>(symbolMap.asMap().entrySet())) {
                if (entry.getValue() instanceof AggregateSymbol && !usedAggregates.contains(entry.getKey())) {
                    symbolMap.asUpdatableMap().remove(entry.getKey());
                }
            }
            if (requiredSymbols.isEmpty() && usedAggregates.isEmpty()) {
                node.setProperty(Info.IS_OPTIONAL, true);
            }
            break;
    }
    // Gather elements from correlated subquery references;
    for (SymbolMap refs : node.getAllReferences()) {
        for (Expression expr : refs.asMap().values()) {
            ElementCollectorVisitor.getElements(expr, requiredSymbols);
        }
    }
    // Add any columns to required that are in this node's output but were not created here
    for (Expression currentOutputSymbol : outputCols) {
        if (!createdSymbols.contains(currentOutputSymbol) && (finalRun || node.getType() != NodeConstants.Types.PROJECT || currentOutputSymbol instanceof ElementSymbol)) {
            requiredSymbols.add(currentOutputSymbol);
        }
    }
    // TODO: this should depend upon whether the expressions are deterministic
    if (node.getType() == NodeConstants.Types.PROJECT) {
        Set<Expression> expressions = new HashSet<Expression>();
        for (Iterator<Expression> iterator = requiredSymbols.iterator(); iterator.hasNext(); ) {
            Expression ses = iterator.next();
            if (!expressions.add(SymbolMap.getExpression(ses))) {
                iterator.remove();
            }
        }
    }
    return new ArrayList<Expression>(requiredSymbols);
}