Examples with BlockedException org.teiid.common.buffer.BlockedException used on opensource projects

Example 31 with BlockedException

use of org.teiid.common.buffer.BlockedException in project teiid by teiid.

the class SortUtility method doMerge.

protected void doMerge(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
    long desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
    int toForce = (int) Math.min(desiredSpace, Math.max(2 * schemaSize, this.bufferManager.getMaxProcessingSize()));
    int reserved = 0;
    if (desiredSpace > toForce) {
        try {
            int subLists = Math.max(2, this.bufferManager.getMaxProcessingSize() / schemaSize);
            int twoPass = subLists * subLists;
            if (twoPass < activeTupleBuffers.size()) {
                // wait for 2-pass
                int needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), .5));
                while (activeTupleBuffers.size() / needed + activeTupleBuffers.size() % needed > needed) {
                    needed++;
                }
                reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, false);
                if (reserved == 0 && twoPass * subLists < activeTupleBuffers.size()) {
                    // force 3-pass
                    needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), 1 / 3d));
                    while (activeTupleBuffers.size() / (needed * needed) + activeTupleBuffers.size() % needed > needed) {
                        needed++;
                    }
                    reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, true);
                    // $NON-NLS-1$
                    LogManager.logWarning(LogConstants.CTX_DQP, "performing three pass sort");
                }
            } else if (desiredSpace < Integer.MAX_VALUE) {
                // wait for 1-pass
                reserved += bufferManager.reserveBuffersBlocking((int) desiredSpace - toForce, attempts, false);
            }
        } catch (BlockedException be) {
            if (!nonBlocking) {
                throw be;
            }
        }
    }
    int total = reserved + toForce;
    if (total > schemaSize) {
        toForce -= total % schemaSize;
    }
    reserved += bufferManager.reserveBuffers(toForce, BufferReserveMode.FORCE);
    try {
        while (this.activeTupleBuffers.size() > 1) {
            ArrayList<SortedSublist> sublists = new ArrayList<SortedSublist>(activeTupleBuffers.size());
            TupleBuffer merged = createTupleBuffer();
            desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
            if (desiredSpace < reserved) {
                bufferManager.releaseBuffers(reserved - (int) desiredSpace);
                reserved = (int) desiredSpace;
            }
            // always allow progress
            int maxSortIndex = Math.max(2, reserved / schemaSize);
            if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_DQP, MessageLevel.TRACE)) {
                // $NON-NLS-1$ //$NON-NLS-2$
                LogManager.logTrace(org.teiid.logging.LogConstants.CTX_DQP, "Merging", maxSortIndex, "sublists out of", activeTupleBuffers.size());
            }
            // initialize the sublists with the min value
            for (int i = 0; i < maxSortIndex; i++) {
                TupleBuffer activeID = activeTupleBuffers.get(i);
                SortedSublist sortedSublist = new SortedSublist();
                sortedSublist.its = activeID.createIndexedTupleSource();
                sortedSublist.its.setNoBlocking(true);
                sortedSublist.index = i;
                incrementWorkingTuple(sublists, sortedSublist);
            }
            boolean checkLimit = maxSortIndex == activeTupleBuffers.size() && rowLimit > -1;
            // iteratively process the lowest tuple
            while (sublists.size() > 0) {
                SortedSublist sortedSublist = sublists.remove(sublists.size() - 1);
                merged.addTuple(sortedSublist.tuple);
                incrementWorkingTuple(sublists, sortedSublist);
                if (checkLimit && merged.getRowCount() == rowLimit) {
                    // early exit for row limit
                    break;
                }
            }
            // Remove merged sublists
            for (int i = 0; i < maxSortIndex; i++) {
                TupleBuffer id = activeTupleBuffers.remove(0);
                id.remove();
            }
            merged.saveBatch();
            this.activeTupleBuffers.add(merged);
        }
    } finally {
        this.bufferManager.releaseBuffers(reserved);
    }
}

Example 32 with BlockedException

use of org.teiid.common.buffer.BlockedException in project teiid by teiid.

the class SortUtility method initialSort.

/**
 * creates sorted sublists stored in tuplebuffers
 */
protected void initialSort(boolean onePass, boolean lowLatency, int rowLimit) throws TeiidComponentException, TeiidProcessingException {
    long end = Long.MAX_VALUE;
    if (!nonBlocking) {
        // obey the timeslice
        CommandContext cc = CommandContext.getThreadLocalContext();
        if (cc != null) {
            end = System.nanoTime() + (cc.getTimeSliceEnd() - System.currentTimeMillis()) * 1000000;
        }
    }
    if (source == null) {
        doneReading = true;
    }
    outer: while (!doneReading) {
        // sub-phase 1 - build up a working buffer of tuples
        if (this.workingBuffer == null) {
            this.workingBuffer = createTupleBuffer();
        }
        while (!doneReading) {
            try {
                List<?> tuple = source.nextTuple();
                if (tuple == null) {
                    doneReading = true;
                    break;
                }
                this.workingBuffer.addTuple(tuple);
                if (onePass && lowLatency && this.workingBuffer.getRowCount() > 2 * this.targetRowCount) {
                    break outer;
                } else if (end != Long.MAX_VALUE && (this.workingBuffer.getRowCount() % 32) == 1 && System.nanoTime() > end) {
                    CommandContext.getThreadLocalContext().getWorkItem().moreWork();
                    // $NON-NLS-1$
                    throw BlockedException.block("Blocking on large sort");
                }
            } catch (BlockedException e) {
                /*there are three cases here
	            	 * 1. a fully blocking sort (optionally dup removal)
	            	 * 2. a streaming dup removal
	            	 * 3. a one pass sort (for grace join like processing)
	            	 */
                if (!onePass) {
                    // read fully before processing
                    throw e;
                }
                // we're trying to create intermediate buffers that will comfortably be small memory sorts
                if (this.workingBuffer.getRowCount() < this.targetRowCount) {
                    throw e;
                }
                // there's processing that we can do
                break outer;
            }
        }
    }
    long rowCount = workingBuffer.getRowCount();
    if (!nonBlocking && !onePass) {
        CommandContext cc = CommandContext.getThreadLocalContext();
        if (cc != null && cc.isParallel() && ((rowCount > (1 << 21) && rowCount > (this.targetRowCount << 3)) || rowCount > (this.targetRowCount << 5))) {
            // potentially long running sort, so let it be async
            workAsync(rowLimit, cc);
        }
    }
    sortWorking(rowLimit);
}

Example 33 with BlockedException

use of org.teiid.common.buffer.BlockedException in project teiid by teiid.

the class UnionAllNode method nextBatchDirectInternal.

public TupleBatch nextBatchDirectInternal() throws BlockedException, TeiidComponentException, TeiidProcessingException {
    // Walk through all children and for each one that isn't done, try to retrieve a batch
    // When all sources are done, set the termination flag on that batch
    RelationalNode[] children = getChildren();
    int childCount = getChildCount();
    int activeSources = 0;
    TupleBatch batch = null;
    boolean additionalSources = false;
    for (int i = 0; i < childCount; i++) {
        if (children[i] != null && !sourceDone[i]) {
            if (sourceOpen != null && !sourceOpen[i]) {
                additionalSources = true;
                continue;
            }
            activeSources++;
            if (batch == null) {
                try {
                    batch = children[i].nextBatch();
                    // Got a batch
                    if (batch.getTerminationFlag() == true) {
                        // Mark source as being done and decrement the activeSources counter
                        sourceDone[i] = true;
                        activeSources--;
                        if (reserved > 0) {
                            getBufferManager().releaseBuffers(schemaSize);
                            reserved -= schemaSize;
                        }
                    }
                } catch (BlockedException e) {
                // no problem - try the next one
                }
            } else {
                // This is sufficient to break the loop - we won't learn anything new after this
                break;
            }
        }
    }
    // Determine what to return
    TupleBatch outputBatch = null;
    if (batch != null) {
        // Rebuild the batch to reset the output row
        outputBatch = new TupleBatch(outputRow, batch.getTuples());
        // This is the last unioned batch if:
        // 1) This batch is a termination batch from the child
        // 2) No other active sources exist
        outputBatch.setTerminationFlag(batch.getTerminationFlag() && activeSources == 0 && !additionalSources);
        // Update output row for next batch
        outputRow += outputBatch.getRowCount();
    } else if (activeSources > 0) {
        // $NON-NLS-1$
        throw BlockedException.block(getContext().getRequestId(), "Blocking on union source.", getID());
    } else {
        boolean openedAny = false;
        int toOpen = 0;
        if (sourceOpen != null) {
            for (int i = 0; i < childCount; i++) {
                if (sourceOpen[i] && sourceDone[i]) {
                    toOpen++;
                }
            }
            for (int i = 0; i < childCount && toOpen > 0; i++) {
                if (!sourceOpen[i]) {
                    getBufferManager().reserveBuffers(schemaSize, BufferReserveMode.FORCE);
                    reserved += schemaSize;
                    children[i].open();
                    sourceOpen[i] = true;
                    openedAny = true;
                    toOpen--;
                }
            }
        }
        if (openedAny) {
            return nextBatchDirect();
        }
        // No batch and no active sources - return empty termination batch (should never happen but just in case)
        outputBatch = new TupleBatch(outputRow, Collections.EMPTY_LIST);
        outputBatch.setTerminationFlag(true);
    }
    return outputBatch;
}

Example 34 with BlockedException

use of org.teiid.common.buffer.BlockedException in project teiid by teiid.

the class DependentAccessNode method nextCommand.

@Override
protected Command nextCommand() throws TeiidProcessingException, TeiidComponentException {
    if (rewrittenCommand == null) {
        Command atomicCommand = super.nextCommand();
        try {
            rewriteAndEvaluate(atomicCommand, getEvaluator(Collections.emptyMap()), this.getContext(), this.getContext().getMetadata());
        } catch (BlockedException e) {
            // be too late
            if (sort && ((Query) atomicCommand).getOrderBy() != null) {
                declineSort();
            }
            throw e;
        }
        rewrittenCommand = atomicCommand;
        nextCommand = null;
    }
    if (nextCommand == null && rewrittenCommand != null) {
        nextCommand = (Command) rewrittenCommand.clone();
    }
    return super.nextCommand();
}

Example 35 with BlockedException

use of org.teiid.common.buffer.BlockedException in project teiid by teiid.

the class BatchedUpdatePlan method nextBatch.

/**
 * @see org.teiid.query.processor.ProcessorPlan#nextBatch()
 * @since 4.2
 */
public TupleBatch nextBatch() throws BlockedException, TeiidComponentException, TeiidProcessingException {
    for (; planIndex < updatePlans.length && (getContext() == null || getContext().getBatchUpdateException() == null); ) {
        try {
            if (!planOpened[planIndex]) {
                // Open the plan only once
                /* Defect 16166
	                 * Some commands in a batch may depend on updates by previous commands in the same batch. A call
	                 * to open() usually submits an atomic command, so calling open() on all the child plans at the same time
	                 * will mean that the datasource may not be in the state expected by a later command within the batch. So,
	                 * for a batch of commands, we only open() a later plan when we are finished with the previous plan to
	                 * guarantee that the commands in the previous plan are completed before the commands in any subsequent
	                 * plans are executed.
	                 */
                openPlan();
            } else if (this.planContexts[planIndex] != null) {
                this.getContext().getTransactionServer().resume(this.planContexts[planIndex]);
            }
            // Execute nextBatch() on each plan in sequence
            TupleBatch nextBatch = null;
            do {
                // Can throw BlockedException
                nextBatch = updatePlans[planIndex].nextBatch();
                List<List<?>> currentBatch = nextBatch.getTuples();
                for (int i = 0; i < currentBatch.size(); i++, commandIndex++) {
                    updateCounts[commandIndex] = currentBatch.get(i);
                }
            } while (!nextBatch.getTerminationFlag());
            // since we are done with the plan explicitly close it.
            updatePlans[planIndex].close();
            if (this.planContexts[planIndex] != null) {
                TransactionService ts = this.getContext().getTransactionServer();
                ts.commit(this.planContexts[planIndex]);
                this.planContexts[planIndex] = null;
            }
            planIndex++;
        } catch (BlockedException e) {
            throw e;
        } catch (TeiidComponentException | TeiidProcessingException e) {
            if (singleResult) {
                throw e;
            }
            Throwable cause = e;
            if (e.getCause() instanceof TranslatorBatchException) {
                TranslatorBatchException tbe = (TranslatorBatchException) e.getCause();
                for (int i = 0; i < tbe.getUpdateCounts().length; i++) {
                    updateCounts[commandIndex++] = Arrays.asList(tbe.getUpdateCounts()[i]);
                }
            }
            updateCounts = Arrays.copyOf(updateCounts, commandIndex);
            getContext().setBatchUpdateException(cause);
        } finally {
            if (planIndex < updatePlans.length && this.planContexts[planIndex] != null) {
                this.getContext().getTransactionServer().suspend(this.planContexts[planIndex]);
            }
        }
    }
    if (singleResult) {
        long result = 0;
        for (int i = 0; i < updateCounts.length; i++) {
            int value = (Integer) updateCounts[i].get(0);
            if (value == Statement.EXECUTE_FAILED) {
                // the batch results rather than throwing an exception
                throw new TeiidProcessingException(QueryPlugin.Event.TEIID31199, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31198));
            }
            if (value > 0) {
                result += value;
            }
        }
        TupleBatch batch = new TupleBatch(1, new List<?>[] { Arrays.asList((int) Math.min(Integer.MAX_VALUE, result)) });
        batch.setTerminationFlag(true);
        return batch;
    }
    // Add tuples to current batch
    TupleBatch batch = new TupleBatch(1, updateCounts);
    batch.setTerminationFlag(true);
    return batch;
}