Examples with SymbolMap org.teiid.query.sql.util.SymbolMap used on opensource projects

Example 1 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class SubqueryAwareEvaluator method evaluateSubquery.

@Override
protected ValueIterator evaluateSubquery(SubqueryContainer<?> container, List<?> tuple) throws TeiidProcessingException, BlockedException, TeiidComponentException {
    ContextReference ref = (ContextReference) container;
    String key = ref.getContextSymbol();
    SubqueryState state = this.subqueries.get(key);
    if (state == null) {
        String otherKey = commands.get(container.getCommand());
        if (otherKey != null) {
            state = this.subqueries.get(otherKey);
            if (state != null) {
                key = otherKey;
            }
        }
    }
    if (state == null) {
        state = new SubqueryState();
        state.plan = container.getCommand().getProcessorPlan().clone();
        if (container.getCommand().getCorrelatedReferences() != null) {
            for (ElementSymbol es : container.getCommand().getCorrelatedReferences().getKeys()) {
                if (DataTypeManager.isNonComparable(DataTypeManager.getDataTypeName(es.getType()))) {
                    state.comparable = false;
                    break;
                }
            }
        }
        this.subqueries.put(key, state);
        this.commands.put(container.getCommand(), key);
    }
    SymbolMap correlatedRefs = container.getCommand().getCorrelatedReferences();
    VariableContext currentContext = null;
    boolean shouldClose = false;
    boolean deterministic = true;
    if (state.processor != null && correlatedRefs != null) {
        Determinism determinism = state.processor.getContext().getDeterminismLevel();
        deterministic = Determinism.COMMAND_DETERMINISTIC.compareTo(determinism) <= 0;
    }
    boolean removeBuffer = true;
    if (correlatedRefs != null) {
        currentContext = new VariableContext();
        for (Map.Entry<ElementSymbol, Expression> entry : container.getCommand().getCorrelatedReferences().asMap().entrySet()) {
            currentContext.setValue(entry.getKey(), evaluate(entry.getValue(), tuple));
        }
        List<Object> refValues = currentContext.getLocalValues();
        if (!refValues.equals(state.refValues)) {
            if (state.comparable && deterministic) {
                if (state.processor != null) {
                    // cache the old value
                    TupleBuffer tb = state.collector.collectTuples();
                    // recheck determinism as the plan may not have been fully processed by the initial check
                    Determinism determinism = state.processor.getContext().getDeterminismLevel();
                    deterministic = Determinism.COMMAND_DETERMINISTIC.compareTo(determinism) <= 0;
                    if (deterministic) {
                        // allowed to track up to 4x the maximum results size
                        maxTuples = Math.max((int) Math.min(Integer.MAX_VALUE, tb.getRowCount() << 2), maxTuples);
                        ArrayList<Object> cacheKey = new ArrayList<Object>(state.refValues);
                        cacheKey.add(key);
                        // ensure that we aren't leaving large last batches in memory
                        tb.saveBatch();
                        this.cache.put(cacheKey, tb);
                        removeBuffer = false;
                        this.currentTuples += tb.getRowCount();
                        while (this.currentTuples > maxTuples && !cache.isEmpty()) {
                            Iterator<Map.Entry<List<?>, TupleBuffer>> i = this.cache.entrySet().iterator();
                            Map.Entry<List<?>, TupleBuffer> entry = i.next();
                            TupleBuffer buffer = entry.getValue();
                            if (buffer.getRowCount() <= 2) {
                                this.smallCache.put(entry.getKey(), buffer);
                            } else {
                                buffer.remove();
                            }
                            this.currentTuples -= buffer.getRowCount();
                            i.remove();
                        }
                    }
                }
                // find if we have cached values
                List<Object> cacheKey = new ArrayList<Object>(refValues);
                cacheKey.add(key);
                TupleBuffer cachedResult = cache.get(cacheKey);
                if (cachedResult == null) {
                    cachedResult = smallCache.get(cacheKey);
                }
                if (cachedResult != null) {
                    state.close(false);
                    return new TupleSourceValueIterator(cachedResult.createIndexedTupleSource(), 0);
                }
            }
            state.refValues = refValues;
            shouldClose = true;
        }
    }
    if (shouldClose || (!deterministic && !state.blocked)) {
        state.close(removeBuffer);
    }
    state.blocked = true;
    if (state.processor == null) {
        CommandContext subContext = context.clone();
        state.plan.reset();
        state.processor = new QueryProcessor(state.plan, subContext, manager, this.dataMgr);
        if (currentContext != null) {
            state.processor.getContext().pushVariableContext(currentContext);
        }
        state.collector = state.processor.createBatchCollector();
    }
    TupleSourceValueIterator iter = new TupleSourceValueIterator(state.collector.collectTuples().createIndexedTupleSource(), 0);
    state.blocked = false;
    return iter;
}

Example 2 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class SubqueryAwareEvaluator method evaluatePushdown.

/**
 * Implements must pushdown function handling if supported by the source.
 *
 * The basic strategy is to create a dummy subquery to represent the evaluation
 */
@Override
protected Object evaluatePushdown(Function function, List<?> tuple, Object[] values) throws TeiidComponentException, TeiidProcessingException {
    final FunctionDescriptor fd = function.getFunctionDescriptor();
    if (fd.getMethod() == null) {
        throw new FunctionExecutionException(QueryPlugin.Event.TEIID30341, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30341, fd.getFullName()));
    }
    String schema = null;
    if (fd.getMethod().getParent() == null || !fd.getMethod().getParent().isPhysical()) {
        // find a suitable target
        // TODO: do better than a linear search
        VDBMetaData vdb = this.context.getVdb();
        CapabilitiesFinder capabiltiesFinder = this.context.getQueryProcessorFactory().getCapabiltiesFinder();
        for (ModelMetaData mmd : vdb.getModelMetaDatas().values()) {
            if (!mmd.isSource()) {
                continue;
            }
            SourceCapabilities caps = capabiltiesFinder.findCapabilities(mmd.getName());
            if (caps.supportsCapability(Capability.SELECT_WITHOUT_FROM) && caps.supportsFunction(fd.getMethod().getFullName())) {
                schema = mmd.getName();
                break;
            }
        }
        if (schema == null) {
            throw new FunctionExecutionException(QueryPlugin.Event.TEIID30341, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30341, fd.getFullName()));
        }
    } else {
        if (!CapabilitiesUtil.supports(Capability.SELECT_WITHOUT_FROM, fd.getMethod().getParent(), context.getMetadata(), context.getQueryProcessorFactory().getCapabiltiesFinder())) {
            if (elements != null) {
                Integer index = (Integer) elements.get(function);
                if (index != null) {
                    return tuple.get(index.intValue());
                }
            }
            throw new FunctionExecutionException(QueryPlugin.Event.TEIID30341, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30341, fd.getFullName()));
        }
        schema = fd.getSchema();
    }
    ScalarSubquery ss = null;
    if (functionState != null) {
        ss = functionState.get(function);
    }
    Expression[] functionArgs = new Expression[values.length];
    for (int i = 0; i < values.length; i++) {
        functionArgs[i] = new Constant(values[i]);
    }
    if (ss == null) {
        final Query command = new Query();
        Select select = new Select();
        command.setSelect(select);
        Function f = new Function(function.getName(), functionArgs);
        f.setType(function.getType());
        f.setFunctionDescriptor(fd);
        select.addSymbol(f);
        ss = new ScalarSubquery(command);
        SymbolMap correlatedReferences = new SymbolMap();
        Collection<ElementSymbol> elements = ElementCollectorVisitor.getElements(function, true);
        if (!elements.isEmpty()) {
            for (ElementSymbol es : elements) {
                correlatedReferences.addMapping(es, es);
            }
            command.setCorrelatedReferences(correlatedReferences);
        }
        command.setProcessorPlan(new SimpleProcessorPlan(command, schema, fd, Arrays.asList(new Constant(null, fd.getReturnType()))));
    } else {
        ((Function) ((ExpressionSymbol) ss.getCommand().getProjectedSymbols().get(0)).getExpression()).setArgs(functionArgs);
    }
    if (functionState == null) {
        this.functionState = new HashMap<Function, ScalarSubquery>(2);
    }
    functionState.put(function, ss);
    return internalEvaluate(ss, tuple);
}

Example 3 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class TestGroupingNode method testDescriptionProperties.

@Test
public void testDescriptionProperties() {
    GroupingNode node = getExampleGroupingNode();
    SymbolMap outputMapping = new SymbolMap();
    outputMapping.addMapping(new ElementSymbol("agg0"), new AggregateSymbol("count", false, null));
    node.setOutputMapping(outputMapping);
    PlanNode pn = node.getDescriptionProperties();
    assertTrue(pn.toString().contains("agg0=count(*)"));
}

Example 4 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class GlobalTableStoreImpl method getGlobalTempTableMetadataId.

@Override
public TempMetadataID getGlobalTempTableMetadataId(Object viewId) throws TeiidProcessingException, TeiidComponentException {
    String matViewName = metadata.getFullName(viewId);
    String matTableName = RelationalPlanner.MAT_PREFIX + matViewName.toUpperCase();
    GroupSymbol group = new GroupSymbol(matViewName);
    group.setMetadataID(viewId);
    TempMetadataID id = tableStore.getMetadataStore().getTempGroupID(matTableName);
    // define the table preserving the key/index information and ensure that only a single instance exists
    if (id == null) {
        synchronized (viewId) {
            id = tableStore.getMetadataStore().getTempGroupID(matTableName);
            LinkedHashMap<Expression, Integer> newExprs = null;
            if (id == null) {
                List<ElementSymbol> allCols = ResolverUtil.resolveElementsInGroup(group, metadata);
                QueryNode qnode = metadata.getVirtualPlan(viewId);
                if (viewId instanceof Table) {
                    Table t = (Table) viewId;
                    List<KeyRecord> fbis = t.getFunctionBasedIndexes();
                    if (!fbis.isEmpty()) {
                        List<GroupSymbol> groups = Arrays.asList(group);
                        int i = 0;
                        newExprs = new LinkedHashMap<Expression, Integer>();
                        for (KeyRecord keyRecord : fbis) {
                            for (int j = 0; j < keyRecord.getColumns().size(); j++) {
                                Column c = keyRecord.getColumns().get(j);
                                if (c.getParent() != keyRecord) {
                                    continue;
                                }
                                String exprString = c.getNameInSource();
                                Expression ex = QueryParser.getQueryParser().parseExpression(exprString);
                                Integer index = newExprs.get(ex);
                                if (index == null) {
                                    ResolverVisitor.resolveLanguageObject(ex, groups, metadata);
                                    ex = QueryRewriter.rewriteExpression(ex, null, metadata);
                                    String colName = TEIID_FBI + i;
                                    while (t.getColumnByName(colName) != null) {
                                        colName = TEIID_FBI + (++i);
                                    }
                                    ElementSymbol es = new ElementSymbol(colName);
                                    es.setType(ex.getType());
                                    allCols.add(es);
                                    c.setPosition(allCols.size());
                                    newExprs.put(ex, allCols.size());
                                    ex = (Expression) ex.clone();
                                } else {
                                    c.setPosition(index);
                                }
                            }
                        }
                        ResolverUtil.clearGroupInfo(group, metadata);
                        // $NON-NLS-1$
                        StringBuilder query = new StringBuilder("SELECT ");
                        // $NON-NLS-1$
                        query.append(group).append(".*, ");
                        for (Iterator<Expression> iter = newExprs.keySet().iterator(); iter.hasNext(); ) {
                            query.append(iter.next());
                            if (iter.hasNext()) {
                                // $NON-NLS-1$
                                query.append(", ");
                            }
                        }
                        // $NON-NLS-1$ //$NON-NLS-2$
                        query.append(" FROM ").append(group).append(" option nocache ").append(group);
                        qnode = new QueryNode(query.toString());
                    }
                }
                id = tableStore.getMetadataStore().addTempGroup(matTableName, allCols, false, true);
                id.setQueryNode(qnode);
                id.setCardinality((int) metadata.getCardinality(viewId));
                id.setOriginalMetadataID(viewId);
                Object pk = metadata.getPrimaryKey(viewId);
                if (pk != null) {
                    ArrayList<TempMetadataID> primaryKey = resolveIndex(metadata, id, pk);
                    id.setPrimaryKey(primaryKey);
                }
                Collection keys = metadata.getUniqueKeysInGroup(viewId);
                for (Object key : keys) {
                    id.addUniqueKey(resolveIndex(metadata, id, key));
                }
                Collection indexes = metadata.getIndexesInGroup(viewId);
                for (Object index : indexes) {
                    id.addIndex(index, resolveIndex(metadata, id, index));
                }
                if (newExprs != null) {
                    Table table = (Table) viewId;
                    List<KeyRecord> fbis = table.getFunctionBasedIndexes();
                    for (KeyRecord keyRecord : fbis) {
                        id.addIndex(keyRecord, resolveIndex(metadata, id, keyRecord));
                    }
                    GroupSymbol gs = new GroupSymbol(matTableName);
                    gs.setMetadataID(id);
                    SymbolMap map = SymbolMap.createSymbolMap(group, ResolverUtil.resolveElementsInGroup(gs, metadata).subList(0, allCols.size() - newExprs.size()), metadata);
                    LinkedHashMap<Expression, Integer> mappedExprs = new LinkedHashMap<Expression, Integer>();
                    for (Map.Entry<Expression, Integer> entry : newExprs.entrySet()) {
                        Expression ex = (Expression) entry.getKey().clone();
                        ExpressionMappingVisitor.mapExpressions(ex, map.asMap());
                        mappedExprs.put(ex, entry.getValue());
                    }
                    id.getTableData().setFunctionBasedExpressions(mappedExprs);
                }
            }
        }
    }
    updateCacheHint(viewId, group, id);
    return id;
}

Example 5 with SymbolMap

use of org.teiid.query.sql.util.SymbolMap in project teiid by teiid.

the class RuleImplementJoinStrategy method execute.

/**
 * @see org.teiid.query.optimizer.relational.OptimizerRule#execute(org.teiid.query.optimizer.relational.plantree.PlanNode, org.teiid.query.metadata.QueryMetadataInterface, org.teiid.query.optimizer.capabilities.CapabilitiesFinder, org.teiid.query.optimizer.relational.RuleStack, org.teiid.query.analysis.AnalysisRecord, org.teiid.query.util.CommandContext)
 */
public PlanNode execute(PlanNode plan, QueryMetadataInterface metadata, CapabilitiesFinder capabilitiesFinder, RuleStack rules, AnalysisRecord analysisRecord, CommandContext context) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
    for (PlanNode sourceNode : NodeEditor.findAllNodes(plan, NodeConstants.Types.SOURCE, NodeConstants.Types.ACCESS)) {
        SymbolMap references = (SymbolMap) sourceNode.getProperty(NodeConstants.Info.CORRELATED_REFERENCES);
        if (references != null) {
            Set<GroupSymbol> groups = GroupsUsedByElementsVisitor.getGroups(references.getValues());
            PlanNode joinNode = NodeEditor.findParent(sourceNode, NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE);
            while (joinNode != null) {
                if (joinNode.getGroups().containsAll(groups)) {
                    joinNode.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.NESTED_TABLE);
                    Info info = Info.RIGHT_NESTED_REFERENCES;
                    if (!FrameUtil.findJoinSourceNode(joinNode.getFirstChild()).getGroups().containsAll(groups)) {
                        // $NON-NLS-1$
                        throw new AssertionError("Should not have reordered the join tree to reverse the lateral join");
                    }
                    SymbolMap map = (SymbolMap) joinNode.getProperty(info);
                    if (map == null) {
                        map = new SymbolMap();
                    }
                    joinNode.setProperty(info, map);
                    map.asUpdatableMap().putAll(references.asMap());
                    if (joinNode.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE) != null) {
                        // $NON-NLS-1$
                        throw new AssertionError("Cannot use a depenedent join when the join involves a correlated nested table.");
                    }
                    break;
                }
                joinNode = NodeEditor.findParent(joinNode, NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE);
            }
        }
    }
    for (PlanNode joinNode : NodeEditor.findAllNodes(plan, NodeConstants.Types.JOIN, NodeConstants.Types.ACCESS)) {
        JoinStrategyType stype = (JoinStrategyType) joinNode.getProperty(NodeConstants.Info.JOIN_STRATEGY);
        if (!JoinStrategyType.MERGE.equals(stype)) {
            continue;
        }
        List<Expression> leftExpressions = (List<Expression>) joinNode.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS);
        List<Expression> rightExpressions = (List<Expression>) joinNode.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS);
        int origExpressionCount = leftExpressions.size();
        // check index information on each side
        // TODO: don't do null order compensation - in fact we should check what the order actually is, but we don't have that metadata
        Object key = null;
        boolean right = true;
        // we check the right first, since it should be larger
        if (joinNode.getLastChild().getType() == NodeConstants.Types.ACCESS && NewCalculateCostUtil.isSingleTable(joinNode.getLastChild())) {
            key = NewCalculateCostUtil.getKeyUsed(rightExpressions, null, metadata, null);
        }
        if (key == null && joinNode.getFirstChild().getType() == NodeConstants.Types.ACCESS && NewCalculateCostUtil.isSingleTable(joinNode.getFirstChild())) {
            key = NewCalculateCostUtil.getKeyUsed(leftExpressions, null, metadata, null);
            right = false;
        }
        JoinType joinType = (JoinType) joinNode.getProperty(NodeConstants.Info.JOIN_TYPE);
        /**
         * Don't push sorts for unbalanced inner joins, we prefer to use a processing time cost based decision
         */
        boolean pushLeft = true;
        boolean pushRight = true;
        if ((joinType == JoinType.JOIN_INNER || joinType == JoinType.JOIN_LEFT_OUTER) && context != null) {
            float leftCost = NewCalculateCostUtil.computeCostForTree(joinNode.getFirstChild(), metadata);
            float rightCost = NewCalculateCostUtil.computeCostForTree(joinNode.getLastChild(), metadata);
            if (leftCost != NewCalculateCostUtil.UNKNOWN_VALUE && rightCost != NewCalculateCostUtil.UNKNOWN_VALUE && (leftCost > context.getProcessorBatchSize() || rightCost > context.getProcessorBatchSize())) {
                // we use a larger constant here to ensure that we don't unwisely prevent pushdown
                pushLeft = leftCost < context.getProcessorBatchSize() || leftCost / rightCost < 8 || (key != null && !right);
                pushRight = rightCost < context.getProcessorBatchSize() || rightCost / leftCost < 8 || joinType == JoinType.JOIN_LEFT_OUTER || (key != null && right);
            }
        }
        if (key != null && joinNode.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE) == null) {
            // redo the join predicates based upon the key alone
            List<Object> keyCols = metadata.getElementIDsInKey(key);
            int[] reorder = new int[keyCols.size()];
            LinkedHashSet<Integer> toCriteria = new LinkedHashSet<Integer>();
            List<Expression> keyExpressions = right ? rightExpressions : leftExpressions;
            Map<Object, Integer> indexMap = new LinkedHashMap<Object, Integer>();
            for (int i = 0; i < keyExpressions.size(); i++) {
                Expression ses = keyExpressions.get(i);
                if (!(ses instanceof ElementSymbol)) {
                    toCriteria.add(i);
                    continue;
                }
                Integer existing = indexMap.put(((ElementSymbol) ses).getMetadataID(), i);
                if (existing != null) {
                    toCriteria.add(existing);
                }
            }
            boolean found = true;
            for (int i = 0; i < keyCols.size(); i++) {
                Object id = keyCols.get(i);
                Integer index = indexMap.remove(id);
                if (index == null) {
                    found = false;
                    break;
                }
                reorder[i] = index;
            }
            if (found) {
                toCriteria.addAll(indexMap.values());
                List<Criteria> joinCriteria = (List<Criteria>) joinNode.getProperty(Info.NON_EQUI_JOIN_CRITERIA);
                for (int index : toCriteria) {
                    Expression lses = leftExpressions.get(index);
                    Expression rses = rightExpressions.get(index);
                    CompareCriteria cc = new CompareCriteria(lses, CompareCriteria.EQ, rses);
                    if (joinCriteria == null || joinCriteria.isEmpty()) {
                        joinCriteria = new ArrayList<Criteria>();
                    }
                    joinCriteria.add(cc);
                }
                joinNode.setProperty(Info.NON_EQUI_JOIN_CRITERIA, joinCriteria);
                leftExpressions = RelationalNode.projectTuple(reorder, leftExpressions);
                rightExpressions = RelationalNode.projectTuple(reorder, rightExpressions);
                joinNode.setProperty(NodeConstants.Info.LEFT_EXPRESSIONS, leftExpressions);
                joinNode.setProperty(NodeConstants.Info.RIGHT_EXPRESSIONS, rightExpressions);
            }
        }
        boolean pushedLeft = insertSort(joinNode.getFirstChild(), leftExpressions, joinNode, metadata, capabilitiesFinder, pushLeft, context);
        if (origExpressionCount == 1 && joinType == JoinType.JOIN_INNER && joinNode.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE) != null && !joinNode.hasCollectionProperty(Info.NON_EQUI_JOIN_CRITERIA)) {
            Collection<Expression> output = (Collection<Expression>) joinNode.getProperty(NodeConstants.Info.OUTPUT_COLS);
            Collection<GroupSymbol> groups = GroupsUsedByElementsVisitor.getGroups(output);
            if (Collections.disjoint(groups, FrameUtil.findJoinSourceNode(joinNode.getFirstChild()).getGroups())) {
                pushRight = false;
                joinNode.setProperty(Info.IS_SEMI_DEP, Boolean.TRUE);
            }
        }
        boolean pushedRight = insertSort(joinNode.getLastChild(), rightExpressions, joinNode, metadata, capabilitiesFinder, pushRight, context);
        if ((!pushedRight || !pushedLeft) && (joinType == JoinType.JOIN_INNER || (joinType == JoinType.JOIN_LEFT_OUTER && !pushedLeft))) {
            joinNode.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.ENHANCED_SORT);
        }
    }
    return plan;
}