Examples with BatchCollector org.teiid.query.processor.BatchCollector used on opensource projects

Example 1 with BatchCollector

use of org.teiid.query.processor.BatchCollector in project teiid by teiid.

the class RequestWorkItem method processNew.

protected void processNew() throws TeiidProcessingException, TeiidComponentException {
    planningStart = System.currentTimeMillis();
    SessionAwareCache<CachedResults> rsCache = dqpCore.getRsCache();
    boolean cachable = false;
    CacheID cacheId = null;
    if (rsCache != null) {
        boolean canUseCache = true;
        if (requestMsg.getRequestOptions().isContinuous()) {
            canUseCache = false;
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Command is continuous, result set caching will not be used");
        } else if (!requestMsg.useResultSetCache() && getCacheHint() == null) {
            canUseCache = false;
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Command has no cache hint and result set cache mode is not on.");
        }
        if (canUseCache) {
            ParseInfo pi = Request.createParseInfo(requestMsg, this.dqpWorkContext.getSession());
            cacheId = new CacheID(this.dqpWorkContext, pi, requestMsg.getCommandString());
            cachable = cacheId.setParameters(requestMsg.getParameterValues());
            if (cachable) {
                // allow cache to be transactionally aware
                if (rsCache.isTransactional()) {
                    TransactionContext tc = request.getTransactionContext(false);
                    if (tc != null && tc.getTransactionType() != Scope.NONE) {
                        initTransactionState(tc);
                        resume();
                    }
                }
                CachedResults cr = rsCache.get(cacheId);
                // TODO: possibly ignore max rows for caching
                if (cr != null && (cr.getRowLimit() == 0 || (requestMsg.getRowLimit() != 0 && requestMsg.getRowLimit() <= cr.getRowLimit()))) {
                    request.initMetadata();
                    this.originalCommand = cr.getCommand(requestMsg.getCommandString(), request.metadata, pi);
                    if (!request.validateAccess(requestMsg.getCommands(), this.originalCommand, CommandType.CACHED)) {
                        // $NON-NLS-1$
                        LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Using result set cached results", cacheId);
                        this.resultsBuffer = cr.getResults();
                        doneProducingBatches();
                        return;
                    }
                    // $NON-NLS-1$
                    LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Cached result command to be modified, will not use the cached results", cacheId);
                }
            } else {
                // $NON-NLS-1$
                LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Parameters are not serializable - cache cannot be used for", cacheId);
            }
        }
    } else {
        // $NON-NLS-1$
        LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Result set caching is disabled.");
    }
    try {
        request.processRequest();
    } finally {
        analysisRecord = request.analysisRecord;
    }
    originalCommand = request.userCommand;
    if (cachable && (requestMsg.useResultSetCache() || originalCommand.getCacheHint() != null) && rsCache != null && originalCommand.areResultsCachable()) {
        this.cid = cacheId;
        // turn on the collection of data objects used
        request.processor.getContext().setDataObjects(new HashSet<Object>(4));
    }
    request.processor.getContext().setWorkItem(this);
    processor = request.processor;
    planningEnd = System.currentTimeMillis();
    this.dqpCore.logMMCommand(this, Event.PLAN, null, null);
    collector = new BatchCollector(processor, processor.getBufferManager(), this.request.context, isForwardOnly()) {

        int maxRows = 0;

        @Override
        protected void flushBatchDirect(TupleBatch batch, boolean add) throws TeiidComponentException, TeiidProcessingException {
            resultsBuffer = getTupleBuffer();
            if (maxRows == 0) {
                maxRows = OUTPUT_BUFFER_MAX_BATCHES * resultsBuffer.getBatchSize();
            }
            if (cid != null) {
                super.flushBatchDirect(batch, add);
            }
            synchronized (lobStreams) {
                if (cid == null && resultsBuffer.isLobs()) {
                    super.flushBatchDirect(batch, false);
                }
                if (batch.getTerminationFlag()) {
                    done();
                }
                add = sendResultsIfNeeded(batch);
                if (cid != null) {
                    return;
                }
                super.flushBatchDirect(batch, add);
                if (!add && !processor.hasBuffer()) {
                    resultsBuffer.setRowCount(batch.getEndRow());
                }
                if (transactionState != TransactionState.ACTIVE && (requestMsg.getRequestOptions().isContinuous() || (useCallingThread && isForwardOnly()))) {
                    synchronized (this) {
                        if (resultsReceiver == null) {
                            // $NON-NLS-1$
                            throw BlockedException.block(requestID, "Blocking to allow asynch processing");
                        }
                    }
                    if (add && !returnsUpdateCount) {
                        // $NON-NLS-1$
                        throw new AssertionError("Should not add batch to buffer");
                    }
                }
                if (add) {
                    flowControl(batch);
                }
            }
        }

        private void flowControl(TupleBatch batch) throws BlockedException {
            if (processor.hasBuffer() || batch.getTerminationFlag() || transactionState == TransactionState.ACTIVE) {
                return;
            }
            synchronized (this) {
                if (!isForwardOnly() && resultsReceiver != null && begin > resultsBuffer.getRowCount()) {
                    // a valid request beyond the processed range
                    return;
                }
            }
            if (resultsBuffer.getManagedRowCount() < maxRows) {
                // continue to buffer
                return;
            }
            int timeOut = 500;
            if (!connectorInfo.isEmpty()) {
                if (explicitSourceClose) {
                    for (DataTierTupleSource ts : getConnectorRequests()) {
                        if (!ts.isExplicitClose()) {
                            timeOut = 100;
                            break;
                        }
                    }
                } else {
                    timeOut = 100;
                }
            }
            if (dqpCore.blockOnOutputBuffer(RequestWorkItem.this)) {
                if (moreWorkTask != null) {
                    moreWorkTask.cancel(false);
                    moreWorkTask = null;
                }
                if (getThreadState() != ThreadState.MORE_WORK) {
                    // we schedule the work to ensure that an idle client won't just indefinitely hold resources
                    moreWorkTask = scheduleWork(timeOut);
                }
                throw // $NON-NLS-1$
                BlockedException.block(// $NON-NLS-1$
                requestID, // $NON-NLS-1$
                "Blocking due to full results TupleBuffer", this.getTupleBuffer(), "rows", this.getTupleBuffer().getManagedRowCount(), "batch size", // $NON-NLS-1$ //$NON-NLS-2$
                this.getTupleBuffer().getBatchSize());
            }
            if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
                // $NON-NLS-1$
                LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Exceeding buffer limit since there are pending active plans or this is using the calling thread.");
            }
        }
    };
    if (!request.addedLimit && this.requestMsg.getRowLimit() > 0 && this.requestMsg.getRowLimit() < Integer.MAX_VALUE) {
        // covers maxrows for commands that already have a limit, are prepared, or are a stored procedure
        this.collector.setRowLimit(this.requestMsg.getRowLimit());
        this.collector.setSaveLastRow(request.isReturingParams());
    }
    this.resultsBuffer = collector.getTupleBuffer();
    if (this.resultsBuffer == null) {
        // This is just a dummy result it will get replaced by collector source
        resultsBuffer = this.processor.getBufferManager().createTupleBuffer(this.originalCommand.getProjectedSymbols(), this.request.context.getConnectionId(), TupleSourceType.FINAL);
    } else if (this.requestMsg.getRequestOptions().isContinuous()) {
        // TODO: this is based upon continuous being an embedded connection otherwise we have to do something like
        // forcing inlining, but truncating or erroring over a given size (similar to odbc handling)
        resultsBuffer.removeLobTracking();
    }
    initTransactionState(request.transactionContext);
    if (requestMsg.isNoExec()) {
        doneProducingBatches();
        resultsBuffer.close();
        this.cid = null;
    }
    this.returnsUpdateCount = request.returnsUpdateCount;
    if (this.returnsUpdateCount && this.requestMsg.getRequestOptions().isContinuous()) {
        // $NON-NLS-1$
        throw new IllegalStateException("Continuous requests are not allowed to be updates.");
    }
    request = null;
}

Example 2 with BatchCollector

use of org.teiid.query.processor.BatchCollector in project teiid by teiid.

the class SourceState method getTupleBuffer.

public TupleBuffer getTupleBuffer() throws TeiidComponentException, TeiidProcessingException {
    if (this.buffer == null) {
        if (this.iterator instanceof BatchIterator) {
            // $NON-NLS-1$
            throw new AssertionError("cannot buffer the source");
        }
        if (source.hasBuffer()) {
            this.buffer = source.getBuffer(-1);
            Assertion.assertTrue(this.buffer.isFinal());
            return this.buffer;
        }
        if (collector == null) {
            collector = new BatchCollector(source, source.getBufferManager(), source.getContext(), false);
        }
        this.buffer = collector.collectTuples();
    }
    return this.buffer;
}

Example 3 with BatchCollector

use of org.teiid.query.processor.BatchCollector in project teiid by teiid.

the class TempTableDataManager method handleCachedProcedure.

private TupleSource handleCachedProcedure(final CommandContext context, StoredProcedure proc) throws TeiidComponentException, QueryMetadataException, TeiidProcessingException {
    String fullName = context.getMetadata().getFullName(proc.getProcedureID());
    // $NON-NLS-1$
    LogManager.logDetail(LogConstants.CTX_DQP, "processing cached procedure request for", fullName);
    LinkedList<Object> vals = new LinkedList<Object>();
    for (SPParameter param : proc.getInputParameters()) {
        vals.add(((Constant) param.getExpression()).getValue());
    }
    // collapse the hash to single byte for the key to restrict the possible results to 256
    int hash = vals.hashCode();
    hash |= (hash >>> 16);
    hash |= (hash >>> 8);
    hash &= 0x000000ff;
    final CacheID cid = new CacheID(new ParseInfo(), fullName + hash, context.getVdbName(), context.getVdbVersion(), context.getConnectionId(), context.getUserName());
    cid.setParameters(vals);
    CachedResults results = cache.get(cid);
    if (results != null) {
        TupleBuffer buffer = results.getResults();
        return buffer.createIndexedTupleSource();
    }
    // construct a query with a no cache hint
    final CacheHint hint = proc.getCacheHint();
    proc.setCacheHint(null);
    Option option = new Option();
    option.setNoCache(true);
    option.addNoCacheGroup(fullName);
    proc.setOption(option);
    StoredProcedure cloneProc = (StoredProcedure) proc.clone();
    int i = 0;
    for (SPParameter param : cloneProc.getInputParameters()) {
        param.setExpression(new Reference(i++));
    }
    final QueryProcessor qp = context.getQueryProcessorFactory().createQueryProcessor(cloneProc.toString(), fullName.toUpperCase(), context, vals.toArray());
    final BatchCollector bc = qp.createBatchCollector();
    return new ProxyTupleSource() {

        boolean success = false;

        @Override
        protected TupleSource createTupleSource() throws TeiidComponentException, TeiidProcessingException {
            TupleBuffer tb = bc.collectTuples();
            CachedResults cr = new CachedResults();
            cr.setResults(tb, qp.getProcessorPlan());
            Determinism determinismLevel = qp.getContext().getDeterminismLevel();
            if (hint != null && hint.getDeterminism() != null) {
                // $NON-NLS-1$ //$NON-NLS-2$
                LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Cache hint modified the query determinism from ", determinismLevel, " to ", hint.getDeterminism() });
                determinismLevel = hint.getDeterminism();
            }
            cache.put(cid, determinismLevel, cr, hint != null ? hint.getTtl() : null);
            context.setDeterminismLevel(determinismLevel);
            success = true;
            return tb.createIndexedTupleSource();
        }

        @Override
        public void closeSource() {
            super.closeSource();
            qp.closeProcessing();
            if (!success && bc.getTupleBuffer() != null) {
                bc.getTupleBuffer().remove();
            }
        }
    };
}