Examples with TupleBatch org.teiid.common.buffer.TupleBatch used on opensource projects

Example 31 with TupleBatch

use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.

the class TestBatchedUpdatePlan method testMultipleBatches.

@Test
public void testMultipleBatches() throws Exception {
    FakeProcessorPlan[] plans = new FakeProcessorPlan[4];
    for (int i = 0; i < plans.length; i++) {
        TupleBatch last = new TupleBatch(2, Arrays.asList(Arrays.asList(1)));
        last.setTerminationFlag(true);
        plans[i] = new FakeProcessorPlan(Arrays.asList(Command.getUpdateCommandSymbol()), Arrays.asList(new TupleBatch(1, Arrays.asList(Arrays.asList(1))), BlockedException.INSTANCE, last));
    }
    BatchedUpdatePlan plan = new BatchedUpdatePlan(Arrays.asList(plans), plans.length * 2, null, false);
    plan.initialize(new CommandContext(), null, null);
    plan.open();
    // First plan may or may not be opened, but all subsequent plans should not be opened.
    for (int i = 1; i < plans.length; i++) {
        assertFalse(plans[i].isOpened());
    }
    for (int i = 0; i < 4; i++) {
        try {
            plan.nextBatch();
            fail();
        } catch (BlockedException e) {
        }
    }
    TupleBatch batch = plan.nextBatch();
    assertEquals(8, batch.getRowCount());
    assertTrue(batch.getTerminationFlag());
}

Example 32 with TupleBatch

use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.

the class TestEnginePerformance method process.

private void process(RelationalNode node, int expectedRows) throws TeiidComponentException, TeiidProcessingException {
    node.open();
    int currentRow = 1;
    while (true) {
        try {
            TupleBatch batch = node.nextBatch();
            currentRow += batch.getRowCount();
            if (batch.getTerminationFlag()) {
                break;
            }
        } catch (BlockedException e) {
        }
    }
    assertEquals(expectedRows, currentRow - 1);
    node.close();
}

Example 33 with TupleBatch

use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.

the class RequestWorkItem method sendResultsIfNeeded.

/**
 * Send results if they have been requested.  This should only be called from the processing thread.
 */
protected boolean sendResultsIfNeeded(TupleBatch batch) throws TeiidComponentException, TeiidProcessingException {
    ResultsMessage response = null;
    ResultsReceiver<ResultsMessage> receiver = null;
    boolean result = true;
    synchronized (this) {
        if (this.resultsReceiver == null) {
            if (cursorRequestExpected()) {
                if (batch != null) {
                    // $NON-NLS-1$
                    throw new AssertionError("batch has no handler");
                }
                // $NON-NLS-1$
                throw BlockedException.block(requestID, "Blocking until client is ready");
            }
            return result;
        }
        if (!this.requestMsg.getRequestOptions().isContinuous()) {
            if ((this.begin > (batch != null ? batch.getEndRow() : this.resultsBuffer.getRowCount()) && !doneProducingBatches) || (this.transactionState == TransactionState.ACTIVE) || (returnsUpdateCount && !doneProducingBatches)) {
                return result;
            }
            if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
                // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID:", requestID, "resultsID:", this.resultsBuffer, "done:", doneProducingBatches);
            }
            boolean fromBuffer = false;
            int count = this.end - this.begin + 1;
            if (returnsUpdateCount) {
                count = Integer.MAX_VALUE;
            }
            if (batch == null || !(batch.containsRow(this.begin) || (batch.getTerminationFlag() && batch.getEndRow() <= this.begin))) {
                if (savedBatch != null && savedBatch.containsRow(this.begin)) {
                    batch = savedBatch;
                } else {
                    batch = resultsBuffer.getBatch(begin);
                    // fetch more than 1 batch from the buffer
                    boolean first = true;
                    int rowSize = resultsBuffer.getRowSizeEstimate();
                    int batches = CLIENT_FETCH_MAX_BATCHES;
                    if (rowSize > 0) {
                        int totalSize = rowSize * resultsBuffer.getBatchSize();
                        if (schemaSize == 0) {
                            schemaSize = this.dqpCore.getBufferManager().getSchemaSize(this.originalCommand.getProjectedSymbols());
                        }
                        int multiplier = schemaSize / totalSize;
                        if (multiplier > 1) {
                            batches *= multiplier;
                        }
                    }
                    if (returnsUpdateCount) {
                        batches = Integer.MAX_VALUE;
                    }
                    for (int i = 1; i < batches && batch.getRowCount() + resultsBuffer.getBatchSize() <= count && !batch.getTerminationFlag(); i++) {
                        TupleBatch next = resultsBuffer.getBatch(batch.getEndRow() + 1);
                        if (next.getRowCount() == 0) {
                            break;
                        }
                        if (first) {
                            first = false;
                            TupleBatch old = batch;
                            batch = new TupleBatch(batch.getBeginRow(), new ResizingArrayList<List<?>>(batch.getTuples()));
                            batch.setTermination(old.getTermination());
                        }
                        batch.getTuples().addAll(next.getTuples());
                        batch.setTermination(next.getTermination());
                    }
                }
                savedBatch = null;
                fromBuffer = true;
            }
            if (batch.getRowCount() > count) {
                long beginRow = isForwardOnly() ? begin : Math.min(this.begin, batch.getEndRow() - count + 1);
                long endRow = Math.min(beginRow + count - 1, batch.getEndRow());
                boolean last = false;
                if (endRow == batch.getEndRow()) {
                    last = batch.getTerminationFlag();
                } else if (isForwardOnly()) {
                    savedBatch = batch;
                }
                List<List<?>> memoryRows = batch.getTuples();
                batch = new TupleBatch(beginRow, memoryRows.subList((int) (beginRow - batch.getBeginRow()), (int) (endRow - batch.getBeginRow() + 1)));
                batch.setTerminationFlag(last);
            } else if (!fromBuffer) {
                result = !isForwardOnly();
            }
        } else if (batch == null) {
            return result;
        } else {
            result = false;
        }
        long finalRowCount = (this.resultsBuffer.isFinal() && !this.requestMsg.getRequestOptions().isContinuous()) ? this.resultsBuffer.getRowCount() : (batch.getTerminationFlag() ? batch.getEndRow() : -1);
        if (batch.getBeginRow() > Integer.MAX_VALUE || batch.getEndRow() > Integer.MAX_VALUE) {
            throw new TeiidProcessingException(QueryPlugin.Event.TEIID31174, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31174));
        }
        response = createResultsMessage(batch.getTuples(), this.originalCommand.getProjectedSymbols());
        response.setFirstRow((int) batch.getBeginRow());
        if (batch.getTermination() == TupleBatch.ITERATION_TERMINATED) {
            response.setLastRow((int) batch.getEndRow() - 1);
        } else {
            response.setLastRow((int) batch.getEndRow());
        }
        response.setUpdateResult(this.returnsUpdateCount);
        if (this.returnsUpdateCount) {
            // batch updates can have special exceptions in addition to update count results
            Throwable t = this.processor.getContext().getBatchUpdateException();
            if (t != null) {
                t = logError(t);
                response.setException(t);
            }
            // swap the result for the generated keys
            if (this.processor.getContext().isReturnAutoGeneratedKeys() && finalRowCount == 1 && this.processor.getContext().getGeneratedKeys() != null && handleGeneratedKeys(response)) {
                finalRowCount = response.getLastRow();
            } else if (finalRowCount == 0 && response.getException() == null) {
                // anon block or other construct not setting an explicit update count
                response.setResults(Arrays.asList(Arrays.asList(0)));
                finalRowCount = 1;
            }
        }
        // set final row
        response.setFinalRow((int) Math.min(finalRowCount, Integer.MAX_VALUE));
        if (response.getLastRow() == finalRowCount) {
            response.setDelayDeserialization(false);
        }
        setWarnings(response);
        // If it is stored procedure, set parameters
        if (originalCommand instanceof StoredProcedure) {
            StoredProcedure proc = (StoredProcedure) originalCommand;
            if (proc.returnParameters()) {
                response.setParameters(getParameterInfo(proc));
            }
        }
        /*
	         * mark the results sent at this point.
	         * communication exceptions will be treated as non-recoverable 
	         */
        receiver = this.resultsReceiver;
        this.resultsReceiver = null;
    }
    cancelCancelTask();
    if ((!this.dqpWorkContext.getSession().isEmbedded() && requestMsg.isDelaySerialization() && this.requestMsg.getShowPlan() == ShowPlan.ON) || this.requestMsg.getShowPlan() == ShowPlan.DEBUG || LogManager.isMessageToBeRecorded(LogConstants.CTX_COMMANDLOGGING, MessageLevel.TRACE)) {
        int bytes;
        try {
            boolean keep = !this.dqpWorkContext.getSession().isEmbedded() && requestMsg.isDelaySerialization();
            bytes = response.serialize(keep);
            if (keep) {
                response.setDelayDeserialization(true);
            }
            dataBytes.addAndGet(bytes);
            // $NON-NLS-1$ //$NON-NLS-2$
            LogManager.logDetail(// $NON-NLS-1$ //$NON-NLS-2$
            LogConstants.CTX_DQP, // $NON-NLS-1$ //$NON-NLS-2$
            "Sending results for", // $NON-NLS-1$ //$NON-NLS-2$
            requestID, // $NON-NLS-1$ //$NON-NLS-2$
            "start row", response.getFirstRow(), "end row", response.getLastRow(), bytes, // $NON-NLS-1$ //$NON-NLS-2$
            "bytes");
        } catch (Exception e) {
        // do nothing.  there is a low level serialization error that we will let happen
        // later since it would be inconvenient here
        }
    }
    setAnalysisRecords(response);
    receiver.receiveResults(response);
    return result;
}

Example 34 with TupleBatch

use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.

the class RequestWorkItem method processNew.

protected void processNew() throws TeiidProcessingException, TeiidComponentException {
    planningStart = System.currentTimeMillis();
    SessionAwareCache<CachedResults> rsCache = dqpCore.getRsCache();
    boolean cachable = false;
    CacheID cacheId = null;
    if (rsCache != null) {
        boolean canUseCache = true;
        if (requestMsg.getRequestOptions().isContinuous()) {
            canUseCache = false;
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Command is continuous, result set caching will not be used");
        } else if (!requestMsg.useResultSetCache() && getCacheHint() == null) {
            canUseCache = false;
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Command has no cache hint and result set cache mode is not on.");
        }
        if (canUseCache) {
            ParseInfo pi = Request.createParseInfo(requestMsg, this.dqpWorkContext.getSession());
            cacheId = new CacheID(this.dqpWorkContext, pi, requestMsg.getCommandString());
            cachable = cacheId.setParameters(requestMsg.getParameterValues());
            if (cachable) {
                // allow cache to be transactionally aware
                if (rsCache.isTransactional()) {
                    TransactionContext tc = request.getTransactionContext(false);
                    if (tc != null && tc.getTransactionType() != Scope.NONE) {
                        initTransactionState(tc);
                        resume();
                    }
                }
                CachedResults cr = rsCache.get(cacheId);
                // TODO: possibly ignore max rows for caching
                if (cr != null && (cr.getRowLimit() == 0 || (requestMsg.getRowLimit() != 0 && requestMsg.getRowLimit() <= cr.getRowLimit()))) {
                    request.initMetadata();
                    this.originalCommand = cr.getCommand(requestMsg.getCommandString(), request.metadata, pi);
                    if (!request.validateAccess(requestMsg.getCommands(), this.originalCommand, CommandType.CACHED)) {
                        // $NON-NLS-1$
                        LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Using result set cached results", cacheId);
                        this.resultsBuffer = cr.getResults();
                        doneProducingBatches();
                        return;
                    }
                    // $NON-NLS-1$
                    LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Cached result command to be modified, will not use the cached results", cacheId);
                }
            } else {
                // $NON-NLS-1$
                LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Parameters are not serializable - cache cannot be used for", cacheId);
            }
        }
    } else {
        // $NON-NLS-1$
        LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Result set caching is disabled.");
    }
    try {
        request.processRequest();
    } finally {
        analysisRecord = request.analysisRecord;
    }
    originalCommand = request.userCommand;
    if (cachable && (requestMsg.useResultSetCache() || originalCommand.getCacheHint() != null) && rsCache != null && originalCommand.areResultsCachable()) {
        this.cid = cacheId;
        // turn on the collection of data objects used
        request.processor.getContext().setDataObjects(new HashSet<Object>(4));
    }
    request.processor.getContext().setWorkItem(this);
    processor = request.processor;
    planningEnd = System.currentTimeMillis();
    this.dqpCore.logMMCommand(this, Event.PLAN, null, null);
    collector = new BatchCollector(processor, processor.getBufferManager(), this.request.context, isForwardOnly()) {

        int maxRows = 0;

        @Override
        protected void flushBatchDirect(TupleBatch batch, boolean add) throws TeiidComponentException, TeiidProcessingException {
            resultsBuffer = getTupleBuffer();
            if (maxRows == 0) {
                maxRows = OUTPUT_BUFFER_MAX_BATCHES * resultsBuffer.getBatchSize();
            }
            if (cid != null) {
                super.flushBatchDirect(batch, add);
            }
            synchronized (lobStreams) {
                if (cid == null && resultsBuffer.isLobs()) {
                    super.flushBatchDirect(batch, false);
                }
                if (batch.getTerminationFlag()) {
                    done();
                }
                add = sendResultsIfNeeded(batch);
                if (cid != null) {
                    return;
                }
                super.flushBatchDirect(batch, add);
                if (!add && !processor.hasBuffer()) {
                    resultsBuffer.setRowCount(batch.getEndRow());
                }
                if (transactionState != TransactionState.ACTIVE && (requestMsg.getRequestOptions().isContinuous() || (useCallingThread && isForwardOnly()))) {
                    synchronized (this) {
                        if (resultsReceiver == null) {
                            // $NON-NLS-1$
                            throw BlockedException.block(requestID, "Blocking to allow asynch processing");
                        }
                    }
                    if (add && !returnsUpdateCount) {
                        // $NON-NLS-1$
                        throw new AssertionError("Should not add batch to buffer");
                    }
                }
                if (add) {
                    flowControl(batch);
                }
            }
        }

        private void flowControl(TupleBatch batch) throws BlockedException {
            if (processor.hasBuffer() || batch.getTerminationFlag() || transactionState == TransactionState.ACTIVE) {
                return;
            }
            synchronized (this) {
                if (!isForwardOnly() && resultsReceiver != null && begin > resultsBuffer.getRowCount()) {
                    // a valid request beyond the processed range
                    return;
                }
            }
            if (resultsBuffer.getManagedRowCount() < maxRows) {
                // continue to buffer
                return;
            }
            int timeOut = 500;
            if (!connectorInfo.isEmpty()) {
                if (explicitSourceClose) {
                    for (DataTierTupleSource ts : getConnectorRequests()) {
                        if (!ts.isExplicitClose()) {
                            timeOut = 100;
                            break;
                        }
                    }
                } else {
                    timeOut = 100;
                }
            }
            if (dqpCore.blockOnOutputBuffer(RequestWorkItem.this)) {
                if (moreWorkTask != null) {
                    moreWorkTask.cancel(false);
                    moreWorkTask = null;
                }
                if (getThreadState() != ThreadState.MORE_WORK) {
                    // we schedule the work to ensure that an idle client won't just indefinitely hold resources
                    moreWorkTask = scheduleWork(timeOut);
                }
                throw // $NON-NLS-1$
                BlockedException.block(// $NON-NLS-1$
                requestID, // $NON-NLS-1$
                "Blocking due to full results TupleBuffer", this.getTupleBuffer(), "rows", this.getTupleBuffer().getManagedRowCount(), "batch size", // $NON-NLS-1$ //$NON-NLS-2$
                this.getTupleBuffer().getBatchSize());
            }
            if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
                // $NON-NLS-1$
                LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Exceeding buffer limit since there are pending active plans or this is using the calling thread.");
            }
        }
    };
    if (!request.addedLimit && this.requestMsg.getRowLimit() > 0 && this.requestMsg.getRowLimit() < Integer.MAX_VALUE) {
        // covers maxrows for commands that already have a limit, are prepared, or are a stored procedure
        this.collector.setRowLimit(this.requestMsg.getRowLimit());
        this.collector.setSaveLastRow(request.isReturingParams());
    }
    this.resultsBuffer = collector.getTupleBuffer();
    if (this.resultsBuffer == null) {
        // This is just a dummy result it will get replaced by collector source
        resultsBuffer = this.processor.getBufferManager().createTupleBuffer(this.originalCommand.getProjectedSymbols(), this.request.context.getConnectionId(), TupleSourceType.FINAL);
    } else if (this.requestMsg.getRequestOptions().isContinuous()) {
        // TODO: this is based upon continuous being an embedded connection otherwise we have to do something like
        // forcing inlining, but truncating or erroring over a given size (similar to odbc handling)
        resultsBuffer.removeLobTracking();
    }
    initTransactionState(request.transactionContext);
    if (requestMsg.isNoExec()) {
        doneProducingBatches();
        resultsBuffer.close();
        this.cid = null;
    }
    this.returnsUpdateCount = request.returnsUpdateCount;
    if (this.returnsUpdateCount && this.requestMsg.getRequestOptions().isContinuous()) {
        // $NON-NLS-1$
        throw new IllegalStateException("Continuous requests are not allowed to be updates.");
    }
    request = null;
}

Example 35 with TupleBatch

use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.

the class XMLTableNode method nextBatchDirect.

@Override
protected synchronized TupleBatch nextBatchDirect() throws BlockedException, TeiidComponentException, TeiidProcessingException {
    evaluate(false);
    if (streaming) {
        while (state == State.BUILDING) {
            try {
                this.wait();
            } catch (InterruptedException e) {
                throw new TeiidRuntimeException(QueryPlugin.Event.TEIID30169, e);
            }
        }
        unwrapException(asynchException);
        TupleBatch batch = this.buffer.getBatch(outputRow);
        outputRow = batch.getEndRow() + 1;
        if (state != State.DONE && !batch.getTerminationFlag()) {
            state = hasNextBatch() ? State.AVAILABLE : State.BUILDING;
        }
        return batch;
    }
    while (!isBatchFull() && !isLastBatch()) {
        if (item == null) {
            try {
                item = result.iter.next();
            } catch (XPathException e) {
                throw new TeiidProcessingException(QueryPlugin.Event.TEIID30170, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30170, e.getMessage()));
            }
            rowCount++;
            if (item == null) {
                terminateBatches();
                break;
            }
        }
        addBatchRow(processRow());
        if (rowCount == rowLimit) {
            terminateBatches();
            break;
        }
    }
    return pullBatch();
}