use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.
the class TestBatchedUpdatePlan method testMultipleBatches.
@Test
public void testMultipleBatches() throws Exception {
FakeProcessorPlan[] plans = new FakeProcessorPlan[4];
for (int i = 0; i < plans.length; i++) {
TupleBatch last = new TupleBatch(2, Arrays.asList(Arrays.asList(1)));
last.setTerminationFlag(true);
plans[i] = new FakeProcessorPlan(Arrays.asList(Command.getUpdateCommandSymbol()), Arrays.asList(new TupleBatch(1, Arrays.asList(Arrays.asList(1))), BlockedException.INSTANCE, last));
}
BatchedUpdatePlan plan = new BatchedUpdatePlan(Arrays.asList(plans), plans.length * 2, null, false);
plan.initialize(new CommandContext(), null, null);
plan.open();
// First plan may or may not be opened, but all subsequent plans should not be opened.
for (int i = 1; i < plans.length; i++) {
assertFalse(plans[i].isOpened());
}
for (int i = 0; i < 4; i++) {
try {
plan.nextBatch();
fail();
} catch (BlockedException e) {
}
}
TupleBatch batch = plan.nextBatch();
assertEquals(8, batch.getRowCount());
assertTrue(batch.getTerminationFlag());
}
use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.
the class TestEnginePerformance method process.
private void process(RelationalNode node, int expectedRows) throws TeiidComponentException, TeiidProcessingException {
node.open();
int currentRow = 1;
while (true) {
try {
TupleBatch batch = node.nextBatch();
currentRow += batch.getRowCount();
if (batch.getTerminationFlag()) {
break;
}
} catch (BlockedException e) {
}
}
assertEquals(expectedRows, currentRow - 1);
node.close();
}
use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.
the class RequestWorkItem method sendResultsIfNeeded.
/**
* Send results if they have been requested. This should only be called from the processing thread.
*/
protected boolean sendResultsIfNeeded(TupleBatch batch) throws TeiidComponentException, TeiidProcessingException {
ResultsMessage response = null;
ResultsReceiver<ResultsMessage> receiver = null;
boolean result = true;
synchronized (this) {
if (this.resultsReceiver == null) {
if (cursorRequestExpected()) {
if (batch != null) {
// $NON-NLS-1$
throw new AssertionError("batch has no handler");
}
// $NON-NLS-1$
throw BlockedException.block(requestID, "Blocking until client is ready");
}
return result;
}
if (!this.requestMsg.getRequestOptions().isContinuous()) {
if ((this.begin > (batch != null ? batch.getEndRow() : this.resultsBuffer.getRowCount()) && !doneProducingBatches) || (this.transactionState == TransactionState.ACTIVE) || (returnsUpdateCount && !doneProducingBatches)) {
return result;
}
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
// $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID:", requestID, "resultsID:", this.resultsBuffer, "done:", doneProducingBatches);
}
boolean fromBuffer = false;
int count = this.end - this.begin + 1;
if (returnsUpdateCount) {
count = Integer.MAX_VALUE;
}
if (batch == null || !(batch.containsRow(this.begin) || (batch.getTerminationFlag() && batch.getEndRow() <= this.begin))) {
if (savedBatch != null && savedBatch.containsRow(this.begin)) {
batch = savedBatch;
} else {
batch = resultsBuffer.getBatch(begin);
// fetch more than 1 batch from the buffer
boolean first = true;
int rowSize = resultsBuffer.getRowSizeEstimate();
int batches = CLIENT_FETCH_MAX_BATCHES;
if (rowSize > 0) {
int totalSize = rowSize * resultsBuffer.getBatchSize();
if (schemaSize == 0) {
schemaSize = this.dqpCore.getBufferManager().getSchemaSize(this.originalCommand.getProjectedSymbols());
}
int multiplier = schemaSize / totalSize;
if (multiplier > 1) {
batches *= multiplier;
}
}
if (returnsUpdateCount) {
batches = Integer.MAX_VALUE;
}
for (int i = 1; i < batches && batch.getRowCount() + resultsBuffer.getBatchSize() <= count && !batch.getTerminationFlag(); i++) {
TupleBatch next = resultsBuffer.getBatch(batch.getEndRow() + 1);
if (next.getRowCount() == 0) {
break;
}
if (first) {
first = false;
TupleBatch old = batch;
batch = new TupleBatch(batch.getBeginRow(), new ResizingArrayList<List<?>>(batch.getTuples()));
batch.setTermination(old.getTermination());
}
batch.getTuples().addAll(next.getTuples());
batch.setTermination(next.getTermination());
}
}
savedBatch = null;
fromBuffer = true;
}
if (batch.getRowCount() > count) {
long beginRow = isForwardOnly() ? begin : Math.min(this.begin, batch.getEndRow() - count + 1);
long endRow = Math.min(beginRow + count - 1, batch.getEndRow());
boolean last = false;
if (endRow == batch.getEndRow()) {
last = batch.getTerminationFlag();
} else if (isForwardOnly()) {
savedBatch = batch;
}
List<List<?>> memoryRows = batch.getTuples();
batch = new TupleBatch(beginRow, memoryRows.subList((int) (beginRow - batch.getBeginRow()), (int) (endRow - batch.getBeginRow() + 1)));
batch.setTerminationFlag(last);
} else if (!fromBuffer) {
result = !isForwardOnly();
}
} else if (batch == null) {
return result;
} else {
result = false;
}
long finalRowCount = (this.resultsBuffer.isFinal() && !this.requestMsg.getRequestOptions().isContinuous()) ? this.resultsBuffer.getRowCount() : (batch.getTerminationFlag() ? batch.getEndRow() : -1);
if (batch.getBeginRow() > Integer.MAX_VALUE || batch.getEndRow() > Integer.MAX_VALUE) {
throw new TeiidProcessingException(QueryPlugin.Event.TEIID31174, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31174));
}
response = createResultsMessage(batch.getTuples(), this.originalCommand.getProjectedSymbols());
response.setFirstRow((int) batch.getBeginRow());
if (batch.getTermination() == TupleBatch.ITERATION_TERMINATED) {
response.setLastRow((int) batch.getEndRow() - 1);
} else {
response.setLastRow((int) batch.getEndRow());
}
response.setUpdateResult(this.returnsUpdateCount);
if (this.returnsUpdateCount) {
// batch updates can have special exceptions in addition to update count results
Throwable t = this.processor.getContext().getBatchUpdateException();
if (t != null) {
t = logError(t);
response.setException(t);
}
// swap the result for the generated keys
if (this.processor.getContext().isReturnAutoGeneratedKeys() && finalRowCount == 1 && this.processor.getContext().getGeneratedKeys() != null && handleGeneratedKeys(response)) {
finalRowCount = response.getLastRow();
} else if (finalRowCount == 0 && response.getException() == null) {
// anon block or other construct not setting an explicit update count
response.setResults(Arrays.asList(Arrays.asList(0)));
finalRowCount = 1;
}
}
// set final row
response.setFinalRow((int) Math.min(finalRowCount, Integer.MAX_VALUE));
if (response.getLastRow() == finalRowCount) {
response.setDelayDeserialization(false);
}
setWarnings(response);
// If it is stored procedure, set parameters
if (originalCommand instanceof StoredProcedure) {
StoredProcedure proc = (StoredProcedure) originalCommand;
if (proc.returnParameters()) {
response.setParameters(getParameterInfo(proc));
}
}
/*
* mark the results sent at this point.
* communication exceptions will be treated as non-recoverable
*/
receiver = this.resultsReceiver;
this.resultsReceiver = null;
}
cancelCancelTask();
if ((!this.dqpWorkContext.getSession().isEmbedded() && requestMsg.isDelaySerialization() && this.requestMsg.getShowPlan() == ShowPlan.ON) || this.requestMsg.getShowPlan() == ShowPlan.DEBUG || LogManager.isMessageToBeRecorded(LogConstants.CTX_COMMANDLOGGING, MessageLevel.TRACE)) {
int bytes;
try {
boolean keep = !this.dqpWorkContext.getSession().isEmbedded() && requestMsg.isDelaySerialization();
bytes = response.serialize(keep);
if (keep) {
response.setDelayDeserialization(true);
}
dataBytes.addAndGet(bytes);
// $NON-NLS-1$ //$NON-NLS-2$
LogManager.logDetail(// $NON-NLS-1$ //$NON-NLS-2$
LogConstants.CTX_DQP, // $NON-NLS-1$ //$NON-NLS-2$
"Sending results for", // $NON-NLS-1$ //$NON-NLS-2$
requestID, // $NON-NLS-1$ //$NON-NLS-2$
"start row", response.getFirstRow(), "end row", response.getLastRow(), bytes, // $NON-NLS-1$ //$NON-NLS-2$
"bytes");
} catch (Exception e) {
// do nothing. there is a low level serialization error that we will let happen
// later since it would be inconvenient here
}
}
setAnalysisRecords(response);
receiver.receiveResults(response);
return result;
}
use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.
the class RequestWorkItem method processNew.
protected void processNew() throws TeiidProcessingException, TeiidComponentException {
planningStart = System.currentTimeMillis();
SessionAwareCache<CachedResults> rsCache = dqpCore.getRsCache();
boolean cachable = false;
CacheID cacheId = null;
if (rsCache != null) {
boolean canUseCache = true;
if (requestMsg.getRequestOptions().isContinuous()) {
canUseCache = false;
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Command is continuous, result set caching will not be used");
} else if (!requestMsg.useResultSetCache() && getCacheHint() == null) {
canUseCache = false;
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Command has no cache hint and result set cache mode is not on.");
}
if (canUseCache) {
ParseInfo pi = Request.createParseInfo(requestMsg, this.dqpWorkContext.getSession());
cacheId = new CacheID(this.dqpWorkContext, pi, requestMsg.getCommandString());
cachable = cacheId.setParameters(requestMsg.getParameterValues());
if (cachable) {
// allow cache to be transactionally aware
if (rsCache.isTransactional()) {
TransactionContext tc = request.getTransactionContext(false);
if (tc != null && tc.getTransactionType() != Scope.NONE) {
initTransactionState(tc);
resume();
}
}
CachedResults cr = rsCache.get(cacheId);
// TODO: possibly ignore max rows for caching
if (cr != null && (cr.getRowLimit() == 0 || (requestMsg.getRowLimit() != 0 && requestMsg.getRowLimit() <= cr.getRowLimit()))) {
request.initMetadata();
this.originalCommand = cr.getCommand(requestMsg.getCommandString(), request.metadata, pi);
if (!request.validateAccess(requestMsg.getCommands(), this.originalCommand, CommandType.CACHED)) {
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Using result set cached results", cacheId);
this.resultsBuffer = cr.getResults();
doneProducingBatches();
return;
}
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Cached result command to be modified, will not use the cached results", cacheId);
}
} else {
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Parameters are not serializable - cache cannot be used for", cacheId);
}
}
} else {
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Result set caching is disabled.");
}
try {
request.processRequest();
} finally {
analysisRecord = request.analysisRecord;
}
originalCommand = request.userCommand;
if (cachable && (requestMsg.useResultSetCache() || originalCommand.getCacheHint() != null) && rsCache != null && originalCommand.areResultsCachable()) {
this.cid = cacheId;
// turn on the collection of data objects used
request.processor.getContext().setDataObjects(new HashSet<Object>(4));
}
request.processor.getContext().setWorkItem(this);
processor = request.processor;
planningEnd = System.currentTimeMillis();
this.dqpCore.logMMCommand(this, Event.PLAN, null, null);
collector = new BatchCollector(processor, processor.getBufferManager(), this.request.context, isForwardOnly()) {
int maxRows = 0;
@Override
protected void flushBatchDirect(TupleBatch batch, boolean add) throws TeiidComponentException, TeiidProcessingException {
resultsBuffer = getTupleBuffer();
if (maxRows == 0) {
maxRows = OUTPUT_BUFFER_MAX_BATCHES * resultsBuffer.getBatchSize();
}
if (cid != null) {
super.flushBatchDirect(batch, add);
}
synchronized (lobStreams) {
if (cid == null && resultsBuffer.isLobs()) {
super.flushBatchDirect(batch, false);
}
if (batch.getTerminationFlag()) {
done();
}
add = sendResultsIfNeeded(batch);
if (cid != null) {
return;
}
super.flushBatchDirect(batch, add);
if (!add && !processor.hasBuffer()) {
resultsBuffer.setRowCount(batch.getEndRow());
}
if (transactionState != TransactionState.ACTIVE && (requestMsg.getRequestOptions().isContinuous() || (useCallingThread && isForwardOnly()))) {
synchronized (this) {
if (resultsReceiver == null) {
// $NON-NLS-1$
throw BlockedException.block(requestID, "Blocking to allow asynch processing");
}
}
if (add && !returnsUpdateCount) {
// $NON-NLS-1$
throw new AssertionError("Should not add batch to buffer");
}
}
if (add) {
flowControl(batch);
}
}
}
private void flowControl(TupleBatch batch) throws BlockedException {
if (processor.hasBuffer() || batch.getTerminationFlag() || transactionState == TransactionState.ACTIVE) {
return;
}
synchronized (this) {
if (!isForwardOnly() && resultsReceiver != null && begin > resultsBuffer.getRowCount()) {
// a valid request beyond the processed range
return;
}
}
if (resultsBuffer.getManagedRowCount() < maxRows) {
// continue to buffer
return;
}
int timeOut = 500;
if (!connectorInfo.isEmpty()) {
if (explicitSourceClose) {
for (DataTierTupleSource ts : getConnectorRequests()) {
if (!ts.isExplicitClose()) {
timeOut = 100;
break;
}
}
} else {
timeOut = 100;
}
}
if (dqpCore.blockOnOutputBuffer(RequestWorkItem.this)) {
if (moreWorkTask != null) {
moreWorkTask.cancel(false);
moreWorkTask = null;
}
if (getThreadState() != ThreadState.MORE_WORK) {
// we schedule the work to ensure that an idle client won't just indefinitely hold resources
moreWorkTask = scheduleWork(timeOut);
}
throw // $NON-NLS-1$
BlockedException.block(// $NON-NLS-1$
requestID, // $NON-NLS-1$
"Blocking due to full results TupleBuffer", this.getTupleBuffer(), "rows", this.getTupleBuffer().getManagedRowCount(), "batch size", // $NON-NLS-1$ //$NON-NLS-2$
this.getTupleBuffer().getBatchSize());
}
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, requestID, "Exceeding buffer limit since there are pending active plans or this is using the calling thread.");
}
}
};
if (!request.addedLimit && this.requestMsg.getRowLimit() > 0 && this.requestMsg.getRowLimit() < Integer.MAX_VALUE) {
// covers maxrows for commands that already have a limit, are prepared, or are a stored procedure
this.collector.setRowLimit(this.requestMsg.getRowLimit());
this.collector.setSaveLastRow(request.isReturingParams());
}
this.resultsBuffer = collector.getTupleBuffer();
if (this.resultsBuffer == null) {
// This is just a dummy result it will get replaced by collector source
resultsBuffer = this.processor.getBufferManager().createTupleBuffer(this.originalCommand.getProjectedSymbols(), this.request.context.getConnectionId(), TupleSourceType.FINAL);
} else if (this.requestMsg.getRequestOptions().isContinuous()) {
// TODO: this is based upon continuous being an embedded connection otherwise we have to do something like
// forcing inlining, but truncating or erroring over a given size (similar to odbc handling)
resultsBuffer.removeLobTracking();
}
initTransactionState(request.transactionContext);
if (requestMsg.isNoExec()) {
doneProducingBatches();
resultsBuffer.close();
this.cid = null;
}
this.returnsUpdateCount = request.returnsUpdateCount;
if (this.returnsUpdateCount && this.requestMsg.getRequestOptions().isContinuous()) {
// $NON-NLS-1$
throw new IllegalStateException("Continuous requests are not allowed to be updates.");
}
request = null;
}
use of org.teiid.common.buffer.TupleBatch in project teiid by teiid.
the class XMLTableNode method nextBatchDirect.
@Override
protected synchronized TupleBatch nextBatchDirect() throws BlockedException, TeiidComponentException, TeiidProcessingException {
evaluate(false);
if (streaming) {
while (state == State.BUILDING) {
try {
this.wait();
} catch (InterruptedException e) {
throw new TeiidRuntimeException(QueryPlugin.Event.TEIID30169, e);
}
}
unwrapException(asynchException);
TupleBatch batch = this.buffer.getBatch(outputRow);
outputRow = batch.getEndRow() + 1;
if (state != State.DONE && !batch.getTerminationFlag()) {
state = hasNextBatch() ? State.AVAILABLE : State.BUILDING;
}
return batch;
}
while (!isBatchFull() && !isLastBatch()) {
if (item == null) {
try {
item = result.iter.next();
} catch (XPathException e) {
throw new TeiidProcessingException(QueryPlugin.Event.TEIID30170, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30170, e.getMessage()));
}
rowCount++;
if (item == null) {
terminateBatches();
break;
}
}
addBatchRow(processRow());
if (rowCount == rowLimit) {
terminateBatches();
break;
}
}
return pullBatch();
}
Aggregations