Examples with TupleSource org.teiid.common.buffer.TupleSource used on opensource projects

Example 16 with TupleSource

use of org.teiid.common.buffer.TupleSource in project teiid by teiid.

the class DataTierManagerImpl method registerRequest.

public TupleSource registerRequest(CommandContext context, Command command, String modelName, final RegisterRequestParameter parameterObject) throws TeiidComponentException, TeiidProcessingException {
    RequestWorkItem workItem = context.getWorkItem();
    Assertion.isNotNull(workItem);
    if (CoreConstants.SYSTEM_MODEL.equals(modelName) || CoreConstants.SYSTEM_ADMIN_MODEL.equals(modelName)) {
        return processSystemQuery(context, command, workItem.getDqpWorkContext());
    }
    AtomicRequestMessage aqr = createRequest(workItem, command, modelName, parameterObject.connectorBindingId, parameterObject.nodeID);
    aqr.setCommandContext(context);
    if (parameterObject.fetchSize > 0) {
        aqr.setFetchSize(2 * parameterObject.fetchSize);
    }
    if (parameterObject.limit > 0) {
        aqr.setFetchSize(Math.min(parameterObject.limit, aqr.getFetchSize()));
    }
    aqr.setCopyStreamingLobs(parameterObject.copyStreamingLobs);
    Collection<GroupSymbol> accessedGroups = null;
    if (context.getDataObjects() != null) {
        QueryMetadataInterface metadata = context.getMetadata();
        accessedGroups = GroupCollectorVisitor.getGroupsIgnoreInlineViews(command, false);
        boolean usedModel = false;
        for (GroupSymbol gs : accessedGroups) {
            context.accessedDataObject(gs.getMetadataID());
            // check the source/tables/procs for determinism level
            Object mid = gs.getMetadataID();
            if (mid instanceof TempMetadataID) {
                TempMetadataID tid = (TempMetadataID) mid;
                if (tid.getOriginalMetadataID() != null) {
                    mid = tid.getOriginalMetadataID();
                }
            }
            String specificProp = metadata.getExtensionProperty(mid, AbstractMetadataRecord.RELATIONAL_URI + DDLConstants.DETERMINISM, false);
            if (specificProp == null) {
                if (!usedModel) {
                    Object modelId = metadata.getModelID(mid);
                    String prop = metadata.getExtensionProperty(modelId, AbstractMetadataRecord.RELATIONAL_URI + DDLConstants.DETERMINISM, false);
                    if (prop != null) {
                        usedModel = true;
                        // set model property
                        context.setDeterminismLevel(Determinism.valueOf(prop.toUpperCase()));
                    }
                }
                continue;
            }
            context.setDeterminismLevel(Determinism.valueOf(specificProp.toUpperCase()));
        }
    }
    ConnectorManagerRepository cmr = workItem.getDqpWorkContext().getVDB().getAttachment(ConnectorManagerRepository.class);
    ConnectorManager connectorManager = cmr.getConnectorManager(aqr.getConnectorName());
    if (connectorManager == null) {
        // can happen if sources are removed
        if (RelationalNodeUtil.hasOutputParams(command)) {
            // $NON-NLS-1$
            throw new AssertionError("A source is required to execute a procedure returning parameters");
        }
        // $NON-NLS-1$ //$NON-NLS-2$
        LogManager.logDetail(LogConstants.CTX_DQP, "source", aqr.getConnectorName(), "no longer exists, returning dummy results");
        return CollectionTupleSource.createNullTupleSource();
    }
    ConnectorWork work = connectorManager.registerRequest(aqr);
    if (!work.isForkable()) {
        aqr.setSerial(true);
    }
    CacheID cid = null;
    CacheDirective cd = null;
    if (workItem.getRsCache() != null && command.areResultsCachable()) {
        CachableVisitor cv = new CachableVisitor();
        PreOrPostOrderNavigator.doVisit(command, cv, PreOrPostOrderNavigator.PRE_ORDER, true);
        if (cv.cacheable) {
            try {
                cd = work.getCacheDirective();
            } catch (TranslatorException e) {
                // $NON-NLS-1$
                throw new TeiidProcessingException(QueryPlugin.Event.TEIID30504, e, aqr.getConnectorName() + ": " + e.getMessage());
            }
            if (cd != null) {
                if (cd.getScope() == Scope.NONE) {
                    parameterObject.doNotCache = true;
                } else {
                    String cmdString = command.toString();
                    if (cmdString.length() < 100000) {
                        // TODO: this check won't be needed if keys aren't exclusively held in memory
                        cid = new CacheID(workItem.getDqpWorkContext(), ParseInfo.DEFAULT_INSTANCE, cmdString);
                        cid.setParameters(cv.parameters);
                        if (cd.getInvalidation() == null || cd.getInvalidation() == Invalidation.NONE) {
                            CachedResults cr = workItem.getRsCache().get(cid);
                            if (cr != null && (cr.getRowLimit() == 0 || (parameterObject.limit > 0 && cr.getRowLimit() >= parameterObject.limit))) {
                                parameterObject.doNotCache = true;
                                // $NON-NLS-1$
                                LogManager.logDetail(LogConstants.CTX_DQP, "Using cache entry for", cid);
                                work.close();
                                return cr.getResults().createIndexedTupleSource();
                            }
                        } else if (cd.getInvalidation() == Invalidation.IMMEDIATE) {
                            workItem.getRsCache().remove(cid, CachingTupleSource.getDeterminismLevel(cd.getScope()));
                        }
                    }
                }
            } else {
                // $NON-NLS-1$
                LogManager.logTrace(LogConstants.CTX_DQP, aqr.getAtomicRequestID(), "no cache directive");
            }
        } else {
            // $NON-NLS-1$
            LogManager.logTrace(LogConstants.CTX_DQP, aqr.getAtomicRequestID(), "command not cachable");
        }
    }
    DataTierTupleSource dtts = new DataTierTupleSource(aqr, workItem, work, this, parameterObject.limit);
    TupleSource result = dtts;
    TupleBuffer tb = null;
    if (cid != null) {
        tb = getBufferManager().createTupleBuffer(aqr.getCommand().getProjectedSymbols(), aqr.getCommandContext().getConnectionId(), TupleSourceType.PROCESSOR);
        result = new CachingTupleSource(this, tb, (DataTierTupleSource) result, cid, parameterObject, cd, accessedGroups, workItem);
    }
    if (work.isThreadBound()) {
        result = handleThreadBound(workItem, aqr, work, cid, result, dtts, tb);
    } else if (!aqr.isSerial()) {
        dtts.addWork();
    }
    return result;
}

Example 17 with TupleSource

use of org.teiid.common.buffer.TupleSource in project teiid by teiid.

the class DataTierManagerImpl method handleThreadBound.

/**
 * thread bound work is tricky for our execution model
 *
 * the strategy here is that
 *
 * - if the result is not already a copying tuplesource (from caching)
 * then wrap in a copying tuple source
 *
 * - submit a workitem that will pull the results/fill the buffer,
 *
 * - return a tuplesource off of the buffer for use by the caller
 */
private TupleSource handleThreadBound(final RequestWorkItem workItem, AtomicRequestMessage aqr, ConnectorWork work, CacheID cid, TupleSource result, DataTierTupleSource dtts, TupleBuffer tb) throws AssertionError, TeiidComponentException, TeiidProcessingException {
    if (workItem.useCallingThread) {
        // in any case we want the underlying work done in the thread accessing the connectorworkitem
        aqr.setSerial(true);
        // simple case, just rely on the client using the same thread
        return result;
    }
    if (tb == null) {
        tb = getBufferManager().createTupleBuffer(aqr.getCommand().getProjectedSymbols(), aqr.getCommandContext().getConnectionId(), TupleSourceType.PROCESSOR);
    }
    final TupleSource ts = tb.createIndexedTupleSource(cid == null);
    if (cid == null) {
        result = new CopyOnReadTupleSource(tb, result) {

            @Override
            public void closeSource() {
                ts.closeSource();
            }
        };
    }
    final ThreadBoundTask callable = new ThreadBoundTask(workItem, result, dtts);
    // but we do so lazily just in case the results aren't needed
    if (aqr.isSerial()) {
        return new TupleSource() {

            boolean processed = false;

            @Override
            public List<?> nextTuple() throws TeiidComponentException, TeiidProcessingException {
                if (!processed) {
                    callable.call();
                    callable.onCompletion(null);
                    processed = true;
                }
                return ts.nextTuple();
            }

            @Override
            public void closeSource() {
                if (!processed) {
                    callable.onCompletion(null);
                    processed = true;
                }
                ts.closeSource();
            }
        };
    }
    aqr.setSerial(true);
    final FutureWork<Void> future = workItem.addWork(callable, callable, 100);
    final TupleBuffer buffer = tb;
    // return a thread-safe TupleSource
    return new TupleSource() {

        boolean checkedDone;

        @Override
        public List<?> nextTuple() throws TeiidComponentException, TeiidProcessingException {
            // TODO: could refactor as completion listener
            if (!checkedDone && future.isDone()) {
                checkedDone = true;
                try {
                    future.get();
                } catch (InterruptedException e) {
                    throw new TeiidComponentException(e);
                } catch (ExecutionException e) {
                    if (e.getCause() instanceof TeiidComponentException) {
                        throw (TeiidComponentException) e.getCause();
                    }
                    if (e.getCause() instanceof TeiidProcessingException) {
                        throw (TeiidProcessingException) e.getCause();
                    }
                    throw new TeiidComponentException(e);
                }
            }
            synchronized (buffer) {
                return ts.nextTuple();
            }
        }

        @Override
        public void closeSource() {
            synchronized (buffer) {
                ts.closeSource();
            }
            callable.done.set(true);
        }
    };
}

Example 18 with TupleSource

use of org.teiid.common.buffer.TupleSource in project teiid by teiid.

the class LanguageBridgeFactory method translate.

/* Insert */
org.teiid.language.Insert translate(Insert insert) {
    List<ElementSymbol> elements = insert.getVariables();
    List<ColumnReference> translatedElements = new ArrayList<ColumnReference>();
    for (ElementSymbol elementSymbol : elements) {
        translatedElements.add(translate(elementSymbol));
    }
    Iterator<List<?>> parameterValues = null;
    InsertValueSource valueSource = null;
    if (insert.getQueryExpression() != null) {
        valueSource = translate(insert.getQueryExpression());
    } else if (insert.getTupleSource() != null) {
        final TupleSource ts = insert.getTupleSource();
        parameterValues = new TupleSourceIterator(ts);
        List<org.teiid.language.Expression> translatedValues = new ArrayList<org.teiid.language.Expression>();
        for (int i = 0; i < insert.getVariables().size(); i++) {
            ElementSymbol es = insert.getVariables().get(i);
            Parameter param = new Parameter();
            param.setType(es.getType());
            param.setValueIndex(i);
            translatedValues.add(param);
        }
        valueSource = new ExpressionValueSource(translatedValues);
    } else {
        // This is for the simple one row insert.
        List values = insert.getValues();
        List<org.teiid.language.Expression> translatedValues = new ArrayList<org.teiid.language.Expression>();
        for (Iterator i = values.iterator(); i.hasNext(); ) {
            translatedValues.add(translate((Expression) i.next()));
        }
        valueSource = new ExpressionValueSource(translatedValues);
    }
    org.teiid.language.Insert result = new org.teiid.language.Insert(translate(insert.getGroup()), translatedElements, valueSource);
    result.setParameterValues(parameterValues);
    setBatchValues(result);
    result.setUpsert(insert.isUpsert());
    return result;
}

Example 19 with TupleSource

use of org.teiid.common.buffer.TupleSource in project teiid by teiid.

the class ExecDynamicSqlInstruction method process.

/**
 * <p>
 * Processing this instruction executes the ProcessorPlan for the command on
 * the CommandStatement of the update procedure language. Executing this
 * plan does not effect the values of any of the variables defined as part
 * of the update procedure and hence the results of the ProcessPlan
 * execution need not be stored for further processing. The results are
 * removed from the buffer manager immediately after execution. The program
 * counter is incremented after execution of the plan.
 * </p>
 *
 * @throws BlockedException
 *             if this processing the plan throws a currentVarContext
 */
public void process(ProcedurePlan procEnv) throws BlockedException, TeiidComponentException, TeiidProcessingException {
    VariableContext localContext = procEnv.getCurrentVariableContext();
    String query = null;
    try {
        Clob value = (Clob) procEnv.evaluateExpression(dynamicCommand.getSql());
        if (value == null) {
            throw new QueryProcessingException(QueryPlugin.Util.getString(// $NON-NLS-1$
            "ExecDynamicSqlInstruction.0"));
        }
        if (value.length() > MAX_SQL_LENGTH) {
            throw new QueryProcessingException(QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31204, MAX_SQL_LENGTH));
        }
        query = value.getSubString(1, MAX_SQL_LENGTH);
        LogManager.logTrace(org.teiid.logging.LogConstants.CTX_DQP, // $NON-NLS-1$
        new Object[] { "Executing dynamic sql ", query });
        Command command = QueryParser.getQueryParser().parseCommand(query);
        // special handling for dynamic anon blocks
        if (command instanceof CreateProcedureCommand) {
            if (dynamicCommand.getIntoGroup() != null || returnable) {
                // and the creation of an inline view
                throw new QueryProcessingException(QueryPlugin.Event.TEIID31250, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31250));
            }
            ((CreateProcedureCommand) command).setResultSetColumns(Collections.EMPTY_LIST);
        }
        command.setExternalGroupContexts(dynamicCommand.getExternalGroupContexts());
        command.setTemporaryMetadata(dynamicCommand.getTemporaryMetadata().clone());
        updateContextWithUsingValues(procEnv, localContext);
        TempMetadataStore metadataStore = command.getTemporaryMetadata();
        if (dynamicCommand.getUsing() != null && !dynamicCommand.getUsing().isEmpty()) {
            metadataStore.addTempGroup(Reserved.USING, new LinkedList<ElementSymbol>(dynamicCommand.getUsing().getClauseMap().keySet()));
            GroupSymbol using = new GroupSymbol(Reserved.USING);
            using.setMetadataID(metadataStore.getTempGroupID(Reserved.USING));
            command.addExternalGroupToContext(using);
            metadataStore.addTempGroup(ProcedureReservedWords.DVARS, new LinkedList<ElementSymbol>(dynamicCommand.getUsing().getClauseMap().keySet()));
            using = new GroupSymbol(ProcedureReservedWords.DVARS);
            using.setMetadataID(metadataStore.getTempGroupID(ProcedureReservedWords.DVARS));
            command.addExternalGroupToContext(using);
        }
        QueryResolver.resolveCommand(command, metadata.getDesignTimeMetadata());
        validateDynamicCommand(procEnv, command, value.toString());
        // create a new set of variables including vars
        Map<ElementSymbol, Expression> nameValueMap = createVariableValuesMap(localContext);
        ValidationVisitor visitor = new ValidationVisitor();
        Request.validateWithVisitor(visitor, metadata, command);
        boolean insertInto = false;
        boolean updateCommand = false;
        if (!command.returnsResultSet() && !(command instanceof StoredProcedure)) {
            if (dynamicCommand.isAsClauseSet()) {
                if (dynamicCommand.getProjectedSymbols().size() != 1 || ((Expression) dynamicCommand.getProjectedSymbols().get(0)).getType() != DataTypeManager.DefaultDataClasses.INTEGER) {
                    throw new QueryProcessingException(QueryPlugin.Event.TEIID31157, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31157));
                }
            }
            updateCommand = true;
        } else if (dynamicCommand.getAsColumns() != null && !dynamicCommand.getAsColumns().isEmpty()) {
            // $NON-NLS-1$
            command = QueryRewriter.createInlineViewQuery(new GroupSymbol("X"), command, metadata, dynamicCommand.getAsColumns());
            if (dynamicCommand.getIntoGroup() != null) {
                Insert insert = new Insert(dynamicCommand.getIntoGroup(), dynamicCommand.getAsColumns(), Collections.emptyList());
                insert.setQueryExpression((Query) command);
                command = insert;
                insertInto = true;
            }
        }
        // if this is an update procedure, it could reassign variables
        command = QueryRewriter.rewrite(command, metadata, procEnv.getContext(), command instanceof CreateProcedureCommand ? Collections.EMPTY_MAP : nameValueMap);
        ProcessorPlan commandPlan = QueryOptimizer.optimizePlan(command, metadata, idGenerator, capFinder, AnalysisRecord.createNonRecordingRecord(), procEnv.getContext());
        if (command instanceof CreateProcedureCommand && commandPlan instanceof ProcedurePlan) {
            ((ProcedurePlan) commandPlan).setValidateAccess(procEnv.isValidateAccess());
        }
        CreateCursorResultSetInstruction inst = new CreateCursorResultSetInstruction(null, commandPlan, (insertInto || updateCommand) ? Mode.UPDATE : returnable ? Mode.HOLD : Mode.NOHOLD) {

            @Override
            public void process(ProcedurePlan procEnv) throws BlockedException, TeiidComponentException, TeiidProcessingException {
                boolean done = true;
                try {
                    super.process(procEnv);
                } catch (BlockedException e) {
                    done = false;
                    throw e;
                } finally {
                    if (done) {
                        procEnv.getContext().popCall();
                    }
                }
            }
        };
        dynamicProgram = new Program(false);
        dynamicProgram.addInstruction(inst);
        if (dynamicCommand.getIntoGroup() != null) {
            String groupName = dynamicCommand.getIntoGroup().getName();
            if (!procEnv.getTempTableStore().hasTempTable(groupName, true)) {
                // create the temp table in the parent scope
                Create create = new Create();
                create.setTable(new GroupSymbol(groupName));
                for (ElementSymbol es : (List<ElementSymbol>) dynamicCommand.getAsColumns()) {
                    Column c = new Column();
                    c.setName(es.getShortName());
                    c.setRuntimeType(DataTypeManager.getDataTypeName(es.getType()));
                    create.getColumns().add(c);
                }
                procEnv.getDataManager().registerRequest(procEnv.getContext(), create, TempMetadataAdapter.TEMP_MODEL.getName(), new RegisterRequestParameter());
            }
            // backwards compatibility to support into with a rowcount
            if (updateCommand) {
                Insert insert = new Insert();
                insert.setGroup(new GroupSymbol(groupName));
                for (ElementSymbol es : (List<ElementSymbol>) dynamicCommand.getAsColumns()) {
                    ElementSymbol col = new ElementSymbol(es.getShortName(), insert.getGroup());
                    col.setType(es.getType());
                    insert.addVariable(col);
                }
                insert.addValue(new Constant(procEnv.getCurrentVariableContext().getValue(ProcedurePlan.ROWCOUNT)));
                QueryResolver.resolveCommand(insert, metadata.getDesignTimeMetadata());
                TupleSource ts = procEnv.getDataManager().registerRequest(procEnv.getContext(), insert, TempMetadataAdapter.TEMP_MODEL.getName(), new RegisterRequestParameter());
                ts.nextTuple();
                ts.closeSource();
            }
        }
        // Add group to recursion stack
        if (parentProcCommand.getUpdateType() != Command.TYPE_UNKNOWN) {
            // $NON-NLS-1$
            procEnv.getContext().pushCall(Command.getCommandToken(parentProcCommand.getUpdateType()) + " " + parentProcCommand.getVirtualGroup());
        } else {
            if (parentProcCommand.getVirtualGroup() != null) {
                procEnv.getContext().pushCall(parentProcCommand.getVirtualGroup().toString());
            }
        }
        procEnv.push(dynamicProgram);
    } catch (SQLException e) {
        Object[] params = { dynamicCommand, dynamicCommand.getSql(), e.getMessage() };
        throw new QueryProcessingException(QueryPlugin.Event.TEIID30168, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30168, params));
    } catch (TeiidProcessingException e) {
        Object[] params = { dynamicCommand, query == null ? dynamicCommand.getSql() : query, e.getMessage() };
        throw new QueryProcessingException(QueryPlugin.Event.TEIID30168, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30168, params));
    }
}

Example 20 with TupleSource

use of org.teiid.common.buffer.TupleSource in project teiid by teiid.

the class SourceState method sort.

public void sort(SortOption sortOption) throws TeiidComponentException, TeiidProcessingException {
    if (sortOption == SortOption.ALREADY_SORTED) {
        return;
    }
    if (this.sortUtility == null) {
        TupleSource ts = null;
        if (source.hasBuffer()) {
            this.buffer = getTupleBuffer();
        } else if (this.buffer == null && this.collector != null) {
            if (sortOption == SortOption.NOT_SORTED) {
                // pass the buffer and the source
                this.buffer = this.collector.getTupleBuffer();
                ts = new BatchCollector.BatchProducerTupleSource(this.source, this.buffer.getRowCount() + 1);
            } else {
                // fully read
                this.buffer = this.collector.collectTuples();
            }
        }
        if (this.buffer != null) {
            this.buffer.setForwardOnly(true);
        } else {
            ts = new BatchIterator(this.source);
        }
        this.sortUtility = new SortUtility(ts, expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), sortOption == SortOption.SORT_DISTINCT ? Mode.DUP_REMOVE_SORT : Mode.SORT, this.source.getBufferManager(), this.source.getConnectionID(), source.getElements());
        this.markExpressionsDistinct(sortOption == SortOption.SORT_DISTINCT && expressions.size() == this.getOuterVals().size());
        if (this.buffer != null) {
            this.sortUtility.setWorkingBuffer(this.buffer);
        }
    }
    TupleBuffer sorted = null;
    if (sortOption == SortOption.NOT_SORTED) {
        if (this.buffers != null || sortUtility.isDoneReading()) {
            return;
        }
        this.buffers = sortUtility.onePassSort(limited);
        if (this.buffers.size() != 1 || !sortUtility.isDoneReading()) {
            nextBuffer();
            return;
        }
        sorted = this.buffers.get(0);
        this.buffers = null;
    } else {
        sorted = sortUtility.sort();
    }
    // only remove the buffer if this is the first time through
    if (this.buffer != null && this.buffer != sorted) {
        this.buffer.remove();
    }
    this.buffer = sorted;
    this.markExpressionsDistinct(sortUtility.isDistinct());
}