Examples with Determinism org.teiid.metadata.FunctionMethod.Determinism used on opensource projects

Example 1 with Determinism

use of org.teiid.metadata.FunctionMethod.Determinism in project teiid by teiid.

the class SubqueryAwareEvaluator method evaluateSubquery.

@Override
protected ValueIterator evaluateSubquery(SubqueryContainer<?> container, List<?> tuple) throws TeiidProcessingException, BlockedException, TeiidComponentException {
    ContextReference ref = (ContextReference) container;
    String key = ref.getContextSymbol();
    SubqueryState state = this.subqueries.get(key);
    if (state == null) {
        String otherKey = commands.get(container.getCommand());
        if (otherKey != null) {
            state = this.subqueries.get(otherKey);
            if (state != null) {
                key = otherKey;
            }
        }
    }
    if (state == null) {
        state = new SubqueryState();
        state.plan = container.getCommand().getProcessorPlan().clone();
        if (container.getCommand().getCorrelatedReferences() != null) {
            for (ElementSymbol es : container.getCommand().getCorrelatedReferences().getKeys()) {
                if (DataTypeManager.isNonComparable(DataTypeManager.getDataTypeName(es.getType()))) {
                    state.comparable = false;
                    break;
                }
            }
        }
        this.subqueries.put(key, state);
        this.commands.put(container.getCommand(), key);
    }
    SymbolMap correlatedRefs = container.getCommand().getCorrelatedReferences();
    VariableContext currentContext = null;
    boolean shouldClose = false;
    boolean deterministic = true;
    if (state.processor != null && correlatedRefs != null) {
        Determinism determinism = state.processor.getContext().getDeterminismLevel();
        deterministic = Determinism.COMMAND_DETERMINISTIC.compareTo(determinism) <= 0;
    }
    boolean removeBuffer = true;
    if (correlatedRefs != null) {
        currentContext = new VariableContext();
        for (Map.Entry<ElementSymbol, Expression> entry : container.getCommand().getCorrelatedReferences().asMap().entrySet()) {
            currentContext.setValue(entry.getKey(), evaluate(entry.getValue(), tuple));
        }
        List<Object> refValues = currentContext.getLocalValues();
        if (!refValues.equals(state.refValues)) {
            if (state.comparable && deterministic) {
                if (state.processor != null) {
                    // cache the old value
                    TupleBuffer tb = state.collector.collectTuples();
                    // recheck determinism as the plan may not have been fully processed by the initial check
                    Determinism determinism = state.processor.getContext().getDeterminismLevel();
                    deterministic = Determinism.COMMAND_DETERMINISTIC.compareTo(determinism) <= 0;
                    if (deterministic) {
                        // allowed to track up to 4x the maximum results size
                        maxTuples = Math.max((int) Math.min(Integer.MAX_VALUE, tb.getRowCount() << 2), maxTuples);
                        ArrayList<Object> cacheKey = new ArrayList<Object>(state.refValues);
                        cacheKey.add(key);
                        // ensure that we aren't leaving large last batches in memory
                        tb.saveBatch();
                        this.cache.put(cacheKey, tb);
                        removeBuffer = false;
                        this.currentTuples += tb.getRowCount();
                        while (this.currentTuples > maxTuples && !cache.isEmpty()) {
                            Iterator<Map.Entry<List<?>, TupleBuffer>> i = this.cache.entrySet().iterator();
                            Map.Entry<List<?>, TupleBuffer> entry = i.next();
                            TupleBuffer buffer = entry.getValue();
                            if (buffer.getRowCount() <= 2) {
                                this.smallCache.put(entry.getKey(), buffer);
                            } else {
                                buffer.remove();
                            }
                            this.currentTuples -= buffer.getRowCount();
                            i.remove();
                        }
                    }
                }
                // find if we have cached values
                List<Object> cacheKey = new ArrayList<Object>(refValues);
                cacheKey.add(key);
                TupleBuffer cachedResult = cache.get(cacheKey);
                if (cachedResult == null) {
                    cachedResult = smallCache.get(cacheKey);
                }
                if (cachedResult != null) {
                    state.close(false);
                    return new TupleSourceValueIterator(cachedResult.createIndexedTupleSource(), 0);
                }
            }
            state.refValues = refValues;
            shouldClose = true;
        }
    }
    if (shouldClose || (!deterministic && !state.blocked)) {
        state.close(removeBuffer);
    }
    state.blocked = true;
    if (state.processor == null) {
        CommandContext subContext = context.clone();
        state.plan.reset();
        state.processor = new QueryProcessor(state.plan, subContext, manager, this.dataMgr);
        if (currentContext != null) {
            state.processor.getContext().pushVariableContext(currentContext);
        }
        state.collector = state.processor.createBatchCollector();
    }
    TupleSourceValueIterator iter = new TupleSourceValueIterator(state.collector.collectTuples().createIndexedTupleSource(), 0);
    state.blocked = false;
    return iter;
}

Example 2 with Determinism

use of org.teiid.metadata.FunctionMethod.Determinism in project teiid by teiid.

the class PreparedStatementRequest method generatePlan.

/**
 * @throws TeiidComponentException
 * @throws TeiidProcessingException
 * @see org.teiid.dqp.internal.process.Request#generatePlan()
 */
@Override
protected void generatePlan(boolean addLimit) throws TeiidComponentException, TeiidProcessingException {
    createCommandContext();
    String sqlQuery = requestMsg.getCommands()[0];
    if (this.preParser != null) {
        sqlQuery = this.preParser.preParse(sqlQuery, this.context);
    }
    CacheID id = new CacheID(this.workContext, Request.createParseInfo(this.requestMsg, this.workContext.getSession()), sqlQuery);
    prepPlan = prepPlanCache.get(id);
    if (prepPlan != null) {
        // already in cache. obtain the values from cache
        analysisRecord = prepPlan.getAnalysisRecord();
        ProcessorPlan cachedPlan = prepPlan.getPlan();
        this.userCommand = prepPlan.getCommand();
        if (validateAccess(requestMsg.getCommands(), userCommand, CommandType.PREPARED)) {
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, requestId, "AuthorizationValidator indicates that the prepared plan for command will not be used");
            prepPlan = null;
            analysisRecord = null;
        } else {
            // $NON-NLS-1$
            LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Query exist in cache: ", sqlQuery });
            processPlan = cachedPlan.clone();
        }
    }
    if (prepPlan == null) {
        // if prepared plan does not exist, create one
        prepPlan = new PreparedPlan();
        // $NON-NLS-1$
        LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Query does not exist in cache: ", sqlQuery });
        super.generatePlan(true);
        prepPlan.setCommand(this.userCommand);
        // there's no need to cache the plan if it's a stored procedure, since we already do that in the optimizer
        boolean cache = !(this.userCommand instanceof StoredProcedure);
        // Defect 13751: Clone the plan in its current state (i.e. before processing) so that it can be used for later queries
        prepPlan.setPlan(cache ? processPlan.clone() : processPlan, this.context);
        prepPlan.setAnalysisRecord(analysisRecord);
        if (cache) {
            Determinism determinismLevel = this.context.getDeterminismLevel();
            if (userCommand.getCacheHint() != null && userCommand.getCacheHint().getDeterminism() != null) {
                // $NON-NLS-1$ //$NON-NLS-2$
                LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Cache hint modified the query determinism from ", this.context.getDeterminismLevel(), " to ", determinismLevel });
                determinismLevel = userCommand.getCacheHint().getDeterminism();
            }
            this.prepPlanCache.put(id, determinismLevel, prepPlan, userCommand.getCacheHint() != null ? userCommand.getCacheHint().getTtl() : null);
        }
    }
    if (requestMsg.isBatchedUpdate()) {
        handlePreparedBatchUpdate();
    } else {
        List<Reference> params = prepPlan.getReferences();
        List<?> values = requestMsg.getParameterValues();
        PreparedStatementRequest.resolveParameterValues(params, values, this.context, this.metadata);
    }
}

Example 3 with Determinism

use of org.teiid.metadata.FunctionMethod.Determinism in project teiid by teiid.

the class CachingTupleSource method nextTuple.

@Override
public List<?> nextTuple() throws TeiidComponentException, TeiidProcessingException {
    if (dtts.scope == Scope.NONE || tb == null) {
        removeTupleBuffer();
        return ts.nextTuple();
    }
    // TODO: the cache directive object needs synchronized for consistency
    List<?> tuple = super.nextTuple();
    if (tuple == null && !dtts.errored) {
        synchronized (cd) {
            if (dtts.scope == Scope.NONE) {
                removeTupleBuffer();
                return tuple;
            }
            CachedResults cr = new CachedResults();
            cr.setResults(tb, null);
            if (!Boolean.FALSE.equals(cd.getUpdatable())) {
                if (accessedGroups != null) {
                    for (GroupSymbol gs : accessedGroups) {
                        cr.getAccessInfo().addAccessedObject(gs.getMetadataID());
                    }
                }
            } else {
                cr.getAccessInfo().setSensitiveToMetadataChanges(false);
            }
            if (parameterObject.limit > 0 && parameterObject.limit == rowNumber) {
                cr.setRowLimit(rowNumber);
            }
            tb.setPrefersMemory(Boolean.TRUE.equals(cd.getPrefersMemory()));
            Determinism determinismLevel = getDeterminismLevel(this.dtts.scope);
            this.dataTierManagerImpl.requestMgr.getRsCache().put(cid, determinismLevel, cr, cd.getTtl());
            tb = null;
        }
    }
    return tuple;
}

Example 4 with Determinism

use of org.teiid.metadata.FunctionMethod.Determinism in project teiid by teiid.

the class RequestWorkItem method addToCache.

private void addToCache() {
    if (!doneProducingBatches || cid == null) {
        return;
    }
    Determinism determinismLevel = processor.getContext().getDeterminismLevel();
    CachedResults cr = new CachedResults();
    cr.setCommand(originalCommand);
    cr.setResults(resultsBuffer, processor.getProcessorPlan());
    if (requestMsg.getRowLimit() > 0 && resultsBuffer.getRowCount() == requestMsg.getRowLimit() + (collector.isSaveLastRow() ? 1 : 0)) {
        cr.setRowLimit(requestMsg.getRowLimit());
    }
    if (originalCommand.getCacheHint() != null) {
        // $NON-NLS-1$
        LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Using cache hint", originalCommand.getCacheHint());
        if (originalCommand.getCacheHint().getMinRows() != null && resultsBuffer.getRowCount() <= originalCommand.getCacheHint().getMinRows()) {
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Not caching result as there are fewer rows than needed", resultsBuffer.getRowCount());
            return;
        }
        resultsBuffer.setPrefersMemory(originalCommand.getCacheHint().isPrefersMemory());
        if (originalCommand.getCacheHint().getDeterminism() != null) {
            determinismLevel = originalCommand.getCacheHint().getDeterminism();
            // $NON-NLS-1$ //$NON-NLS-2$
            LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Cache hint modified the query determinism from ", processor.getContext().getDeterminismLevel(), " to ", determinismLevel });
        }
        // if not updatable, then remove the access info
        if (!originalCommand.getCacheHint().isUpdatable(true)) {
            cr.getAccessInfo().setSensitiveToMetadataChanges(false);
            cr.getAccessInfo().getObjectsAccessed().clear();
        }
    }
    if (determinismLevel.compareTo(Determinism.SESSION_DETERMINISTIC) <= 0) {
        LogManager.logInfo(LogConstants.CTX_DQP, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30008, originalCommand));
    }
    try {
        this.resultsBuffer.persistLobs();
    } catch (TeiidException e) {
        // $NON-NLS-1$
        LogManager.logDetail(LogConstants.CTX_DQP, e, QueryPlugin.Util.getString("failed_to_cache"));
    }
    dqpCore.getRsCache().put(cid, determinismLevel, cr, originalCommand.getCacheHint() != null ? originalCommand.getCacheHint().getTtl() : null);
}

Example 5 with Determinism

use of org.teiid.metadata.FunctionMethod.Determinism in project teiid by teiid.

the class CriteriaCapabilityValidatorVisitor method canPushLanguageObject.

public static boolean canPushLanguageObject(LanguageObject obj, Object modelID, final QueryMetadataInterface metadata, CapabilitiesFinder capFinder, AnalysisRecord analysisRecord, boolean isJoin, boolean isSelectClause, final boolean multiValuedReferences) throws QueryMetadataException, TeiidComponentException {
    if (obj == null) {
        return true;
    }
    if (modelID == null || metadata.isVirtualModel(modelID)) {
        // Couldn't determine model ID, so give up
        return false;
    }
    String modelName = metadata.getFullName(modelID);
    SourceCapabilities caps = capFinder.findCapabilities(modelName);
    if (caps == null) {
        // this doesn't seem right, but tests were expecting it...
        return true;
    }
    CriteriaCapabilityValidatorVisitor visitor = new CriteriaCapabilityValidatorVisitor(modelID, metadata, capFinder, caps);
    visitor.setCheckEvaluation(!multiValuedReferences);
    visitor.analysisRecord = analysisRecord;
    visitor.isJoin = isJoin;
    visitor.isSelectClause = isSelectClause;
    // we use an array to represent multiple comparision attributes,
    // but we don't want that to inhibit pushdown as we'll account for that later
    // in criteria processing
    final EvaluatableVisitor ev = new EvaluatableVisitor(modelID, metadata, capFinder);
    PreOrPostOrderNavigator nav = new PreOrPostOrderNavigator(visitor, PreOrPostOrderNavigator.POST_ORDER, false) {

        @Override
        public void visit(DependentSetCriteria obj1) {
            if (obj1.hasMultipleAttributes()) {
                Array array = (Array) obj1.getExpression();
                visitNodes(array.getExpressions());
                super.postVisitVisitor(obj1);
            } else {
                super.visit(obj1);
            }
        }

        @Override
        protected void visitNode(LanguageObject obj) {
            if (obj == null) {
                return;
            }
            Determinism d = ev.getDeterminismLevel();
            boolean pushDown = ev.requiresEvaluation(EvaluationLevel.PUSH_DOWN);
            // decend with clean state, then restore
            ev.reset();
            super.visitNode(obj);
            ev.setDeterminismLevel(d);
            if (pushDown) {
                ev.evaluationNotPossible(EvaluationLevel.PUSH_DOWN);
            }
        }

        @Override
        protected void visitVisitor(LanguageObject obj) {
            if (obj == null) {
                return;
            }
            if (!ev.requiresEvaluation(EvaluationLevel.PUSH_DOWN) && ev.getDeterminismLevel() != Determinism.NONDETERMINISTIC) {
                if (obj instanceof ElementSymbol) {
                    ElementSymbol es = (ElementSymbol) obj;
                    if (es.getMetadataID() != null) {
                        try {
                            if (metadata.isMultiSourceElement(es.getMetadataID())) {
                                // no need to visit
                                return;
                            }
                        } catch (QueryMetadataException e) {
                        } catch (TeiidComponentException e) {
                        }
                    }
                }
                obj.acceptVisitor(ev);
                if (!multiValuedReferences && obj instanceof Expression) {
                    if (obj instanceof Function) {
                        if (!(obj instanceof AggregateSymbol)) {
                            Function f = (Function) obj;
                            if (f.getFunctionDescriptor().getPushdown() != PushDown.MUST_PUSHDOWN && f.getFunctionDescriptor().getDeterministic() != Determinism.NONDETERMINISTIC) {
                                // don't need to consider
                                return;
                            }
                        }
                    } else if (obj instanceof Criteria && !(obj instanceof SubqueryContainer) && !(obj instanceof DependentSetCriteria)) {
                        // don't need to consider
                        return;
                    }
                }
            }
            super.visitVisitor(obj);
        }
    };
    obj.acceptVisitor(nav);
    if (visitor.getException() != null) {
        throw visitor.getException();
    }
    return visitor.isValid();
}