Examples with TupleBuffer org.teiid.common.buffer.TupleBuffer used on opensource projects

Example 6 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class WindowFunctionProjectNode method buildResults.

/**
 * Build the results by maintaining indexes that either map
 * rowid->values
 * or
 * rowid->partitionid and partitionid->values
 *
 * TODO use the size hint for tree balancing
 */
private void buildResults() throws TeiidComponentException, TeiidProcessingException, FunctionExecutionException, ExpressionEvaluationException {
    List<Map.Entry<WindowSpecification, WindowSpecificationInfo>> specs = new ArrayList<Map.Entry<WindowSpecification, WindowSpecificationInfo>>(windows.entrySet());
    for (int specIndex = 0; specIndex < specs.size(); specIndex++) {
        Map.Entry<WindowSpecification, WindowSpecificationInfo> entry = specs.get(specIndex);
        WindowSpecificationInfo info = entry.getValue();
        IndexedTupleSource specificationTs = tb.createIndexedTupleSource();
        boolean multiGroup = false;
        int[] partitionIndexes = null;
        int[] orderIndexes = null;
        // if there is partitioning or ordering, then sort
        if (!info.orderType.isEmpty()) {
            multiGroup = true;
            int[] sortKeys = new int[info.orderType.size()];
            int i = 0;
            if (!info.groupIndexes.isEmpty()) {
                for (Integer sortIndex : info.groupIndexes) {
                    sortKeys[i++] = sortIndex;
                }
                partitionIndexes = Arrays.copyOf(sortKeys, info.groupIndexes.size());
            }
            if (!info.sortIndexes.isEmpty()) {
                for (Integer sortIndex : info.sortIndexes) {
                    sortKeys[i++] = sortIndex;
                }
                orderIndexes = Arrays.copyOfRange(sortKeys, info.groupIndexes.size(), info.groupIndexes.size() + info.sortIndexes.size());
            }
            if (!info.functions.isEmpty()) {
                // $NON-NLS-1$
                ElementSymbol key = new ElementSymbol("rowId");
                key.setType(DataTypeManager.DefaultDataClasses.INTEGER);
                // $NON-NLS-1$
                ElementSymbol value = new ElementSymbol("partitionId");
                value.setType(DataTypeManager.DefaultDataClasses.INTEGER);
                List<ElementSymbol> elements = Arrays.asList(key, value);
                partitionMapping[specIndex] = this.getBufferManager().createSTree(elements, this.getConnectionID(), 1);
            }
            SortUtility su = new SortUtility(null, Mode.SORT, this.getBufferManager(), this.getConnectionID(), tb.getSchema(), info.orderType, info.nullOrderings, sortKeys);
            su.setWorkingBuffer(tb);
            su.setNonBlocking(true);
            TupleBuffer sorted = su.sort();
            specificationTs = sorted.createIndexedTupleSource(true);
        }
        List<AggregateFunction> aggs = initializeAccumulators(info.functions, specIndex, false);
        List<AggregateFunction> rowValueAggs = initializeAccumulators(info.rowValuefunctions, specIndex, true);
        int groupId = 0;
        List<?> lastRow = null;
        while (specificationTs.hasNext()) {
            List<?> tuple = specificationTs.nextTuple();
            if (multiGroup) {
                if (lastRow != null) {
                    boolean samePartition = GroupingNode.sameGroup(partitionIndexes, tuple, lastRow) == -1;
                    if (!aggs.isEmpty() && (!samePartition || GroupingNode.sameGroup(orderIndexes, tuple, lastRow) != -1)) {
                        saveValues(specIndex, aggs, groupId, samePartition, false);
                        groupId++;
                    }
                    saveValues(specIndex, rowValueAggs, lastRow.get(lastRow.size() - 1), samePartition, true);
                }
                if (!aggs.isEmpty()) {
                    List<Object> partitionTuple = Arrays.asList(tuple.get(tuple.size() - 1), groupId);
                    partitionMapping[specIndex].insert(partitionTuple, InsertMode.NEW, -1);
                }
            }
            for (AggregateFunction function : aggs) {
                function.addInput(tuple, getContext());
            }
            for (AggregateFunction function : rowValueAggs) {
                function.addInput(tuple, getContext());
            }
            lastRow = tuple;
        }
        if (lastRow != null) {
            saveValues(specIndex, aggs, groupId, true, false);
            saveValues(specIndex, rowValueAggs, lastRow.get(lastRow.size() - 1), true, true);
        }
    }
}

Example 7 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class AccessNode method nextCommand.

@SuppressWarnings("unused")
protected Command nextCommand() throws TeiidProcessingException, TeiidComponentException {
    // to ensure that the subquery ids remain stable
    if (nextCommand == null) {
        nextCommand = (Command) processingCommand.clone();
        if (evaluatedPlans != null) {
            for (WithQueryCommand with : ((QueryCommand) nextCommand).getWith()) {
                TupleBuffer tb = evaluatedPlans.get(with.getGroupSymbol()).collector.getTupleBuffer();
                with.setTupleBuffer(tb);
            }
        }
    }
    return nextCommand;
}

Example 8 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class JoinNode method nextBatchDirectInternal.

/**
 * @see org.teiid.query.processor.relational.RelationalNode#nextBatchDirect()
 * @since 4.2
 */
protected TupleBatch nextBatchDirectInternal() throws BlockedException, TeiidComponentException, TeiidProcessingException {
    try {
        if (state == State.LOAD_LEFT) {
            boolean rightDep = false;
            // dependent semi joins are processed right first
            if (isDependent() && (this.joinType == JoinType.JOIN_ANTI_SEMI || this.joinType == JoinType.JOIN_SEMI)) {
                rightDep = true;
                this.joinStrategy.openRight();
                this.joinStrategy.loadRight();
                TupleBuffer buffer = this.joinStrategy.rightSource.getTupleBuffer();
                // the tuplebuffer may be from a lower node, so pass in the schema
                dvs = new DependentValueSource(buffer, this.joinStrategy.rightSource.getSource().getElements());
                dvs.setDistinct(this.joinStrategy.rightSource.isExpresssionDistinct());
                this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, dvs);
            }
            if (this.joinType != JoinType.JOIN_FULL_OUTER || this.getJoinCriteria() == null) {
                this.joinStrategy.leftSource.setImplicitBuffer(ImplicitBuffer.NONE);
            }
            this.joinStrategy.openLeft();
            this.joinStrategy.loadLeft();
            if (isDependent() && !rightDep) {
                TupleBuffer buffer = this.joinStrategy.leftSource.getTupleBuffer();
                // the tuplebuffer may be from a lower node, so pass in the schema
                dvs = new DependentValueSource(buffer, this.joinStrategy.leftSource.getSource().getElements());
                dvs.setDistinct(this.joinStrategy.leftSource.isExpresssionDistinct());
                this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, dvs);
            }
            state = State.LOAD_RIGHT;
        }
    } catch (BlockedException e) {
        if (!isDependent()) {
            if (getJoinType() != JoinType.JOIN_FULL_OUTER && this.joinStrategy.leftSource.getSortUtility() == null && this.joinStrategy.leftSource.rowCountLE(0)) {
                this.terminateBatches();
                return pullBatch();
            }
            this.joinStrategy.openRight();
            this.joinStrategy.loadRight();
            prefetch(this.joinStrategy.rightSource, this.joinStrategy.leftSource);
        }
        throw e;
    }
    try {
        if (state == State.LOAD_RIGHT) {
            if (getJoinType() != JoinType.JOIN_FULL_OUTER && this.joinStrategy.leftSource.getSortUtility() == null && this.joinStrategy.leftSource.rowCountLE(0)) {
                this.terminateBatches();
                return pullBatch();
            }
            this.joinStrategy.openRight();
            this.joinStrategy.loadRight();
            state = State.EXECUTE;
        }
        this.joinStrategy.process();
        this.terminateBatches();
    } catch (BatchAvailableException e) {
    // pull the batch
    } catch (BlockedException e) {
        // could track which side is blocking
        try {
            prefetch(this.joinStrategy.leftSource, this.joinStrategy.rightSource);
        } catch (BlockedException e1) {
        }
        prefetch(this.joinStrategy.rightSource, this.joinStrategy.leftSource);
        throw e;
    }
    return pullBatch();
}

Example 9 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class RelationalNode method getBuffer.

/**
 * return the final tuple buffer or null if not available
 * @return
 * @throws TeiidProcessingException
 * @throws TeiidComponentException
 * @throws BlockedException
 */
public final TupleBuffer getBuffer(int maxRows) throws BlockedException, TeiidComponentException, TeiidProcessingException {
    CommandContext context = this.getContext();
    if (context != null && context.isCancelled()) {
        throw new TeiidProcessingException(QueryPlugin.Event.TEIID30160, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30160, getContext().getRequestId()));
    }
    boolean recordStats = context != null && context.getCollectNodeStatistics() && !isLastBatch();
    try {
        // start timer for this batch
        if (recordStats) {
            this.getProcessingState().nodeStatistics.startBatchTimer();
        }
        TupleBuffer buffer = getBufferDirect(maxRows);
        terminateBatches();
        if (recordStats) {
            // stop timer for this batch (normal)
            this.getProcessingState().nodeStatistics.stopBatchTimer();
            this.getProcessingState().nodeStatistics.collectCumulativeNodeStats(buffer.getRowCount(), RelationalNodeStatistics.BATCHCOMPLETE_STOP);
            this.getProcessingState().nodeStatistics.collectNodeStats(this.getChildren());
            if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_DQP, MessageLevel.TRACE) && !buffer.isForwardOnly()) {
                for (long i = 1; i <= buffer.getRowCount(); i += buffer.getBatchSize()) {
                    TupleBatch tb = buffer.getBatch(i);
                    recordBatch(tb);
                }
            }
            recordStats = false;
        }
        return buffer;
    } catch (BlockedException e) {
        if (recordStats) {
            // stop timer for this batch (BlockedException)
            this.getProcessingState().nodeStatistics.stopBatchTimer();
            this.getProcessingState().nodeStatistics.collectCumulativeNodeStats(null, RelationalNodeStatistics.BLOCKEDEXCEPTION_STOP);
            recordStats = false;
        }
        throw e;
    } finally {
        if (recordStats) {
            this.getProcessingState().nodeStatistics.stopBatchTimer();
        }
    }
}

Example 10 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class BatchCollector method collectTuples.

public TupleBuffer collectTuples(boolean singleBatch) throws TeiidComponentException, TeiidProcessingException {
    TupleBatch batch = null;
    while (!done) {
        if (this.hasFinalBuffer) {
            if (this.buffer == null) {
                TupleBuffer finalBuffer = this.sourceNode.getBuffer(rowLimit);
                Assertion.isNotNull(finalBuffer);
                this.buffer = finalBuffer;
            }
            if (this.buffer.isFinal()) {
                this.buffer.setForwardOnly(forwardOnly);
                done = true;
                break;
            }
        }
        batch = sourceNode.nextBatch();
        if (rowLimit > 0 && rowLimit <= batch.getEndRow()) {
            if (!done) {
                this.sourceNode.close();
            }
            List<?> lastTuple = null;
            if (saveLastRow) {
                if (batch.getTerminationFlag()) {
                    lastTuple = batch.getTuples().get(batch.getTuples().size() - 1);
                } else if (rowLimit < batch.getBeginRow()) {
                    // skip until end
                    continue;
                }
            }
            boolean modified = false;
            if (rowLimit < batch.getEndRow()) {
                // we know row limit must be smaller than max int, so an int cast is safe here
                int firstRow = (int) Math.min(rowLimit + 1, batch.getBeginRow());
                List<List<?>> tuples = batch.getTuples().subList(0, rowLimit - firstRow + 1);
                batch = new TupleBatch(firstRow, tuples);
                modified = true;
            }
            if (lastTuple != null) {
                if (!modified) {
                    batch = new TupleBatch(batch.getBeginRow(), batch.getTuples());
                }
                batch.getTuples().add(lastTuple);
            }
            batch.setTerminationFlag(true);
        }
        flushBatch(batch);
        // Check for termination condition
        if (batch.getTerminationFlag()) {
            done = true;
            if (!this.sourceNode.hasBuffer()) {
                buffer.close();
            }
            break;
        }
        if (singleBatch) {
            return null;
        }
    }
    return buffer;
}