Examples with TupleBuffer org.teiid.common.buffer.TupleBuffer used on opensource projects

Example 26 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SortUtility method sortWorking.

private void sortWorking(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
    // sub-phase 2 - perform a memory sort on the workingbuffer/source
    int totalReservedBuffers = 0;
    try {
        int maxRows = this.batchSize;
        Collection<List<?>> workingTuples = null;
        boolean done = false;
        /*
			 * we can balance the work between the initial / multi-pass sort based upon the row count
			 * and an updated estimate of the batch memory size 
			 */
        this.workingBuffer.close();
        schemaSize = Math.max(1, this.workingBuffer.getRowSizeEstimate() * this.batchSize);
        long rowCount = workingBuffer.getRowCount();
        long memorySpaceNeeded = rowCount * this.workingBuffer.getRowSizeEstimate();
        totalReservedBuffers = bufferManager.reserveBuffers(Math.min(bufferManager.getMaxProcessingSize(), (int) Math.min(memorySpaceNeeded, Integer.MAX_VALUE)), BufferReserveMode.FORCE);
        if (totalReservedBuffers != memorySpaceNeeded) {
            int processingSublists = Math.max(2, bufferManager.getMaxProcessingSize() / schemaSize);
            int desiredSpace = (int) Math.min(Integer.MAX_VALUE, (workingBuffer.getRowCount() / processingSublists + (workingBuffer.getRowCount() % processingSublists)) * this.workingBuffer.getRowSizeEstimate());
            if (desiredSpace > totalReservedBuffers) {
                totalReservedBuffers += bufferManager.reserveBuffers(desiredSpace - totalReservedBuffers, BufferReserveMode.NO_WAIT);
            // TODO: wait to force 2/3 pass processing
            } else if (memorySpaceNeeded <= Integer.MAX_VALUE) {
                totalReservedBuffers += bufferManager.reserveBuffers((int) memorySpaceNeeded - totalReservedBuffers, BufferReserveMode.NO_WAIT);
            }
            if (totalReservedBuffers > schemaSize) {
                int additional = totalReservedBuffers % schemaSize;
                totalReservedBuffers -= additional;
                // release any excess
                bufferManager.releaseBuffers(additional);
            }
        }
        TupleBufferTupleSource ts = workingBuffer.createIndexedTupleSource(source != null);
        ts.setReverse(!stableSort && workingBuffer.getRowCount() > this.batchSize);
        maxRows = Math.max(1, (totalReservedBuffers / schemaSize)) * batchSize;
        boolean checkLimit = rowLimit > -1 && rowCount <= maxRows;
        if (mode == Mode.SORT) {
            workingTuples = new AccessibleArrayList<>();
        } else {
            workingTuples = new TreeSet<List<?>>(comparator);
        }
        outer: while (!done) {
            while (!done) {
                if (workingTuples.size() >= maxRows) {
                    break;
                }
                List<?> tuple = ts.nextTuple();
                if (tuple == null) {
                    done = true;
                    if (workingTuples.isEmpty()) {
                        break outer;
                    }
                    break;
                }
                workingTuples.add(tuple);
            }
            TupleBuffer sublist = createTupleBuffer();
            activeTupleBuffers.add(sublist);
            if (this.mode == Mode.SORT) {
                // perform a stable sort
                if (workingTuples.size() > (1 << 18)) {
                    Arrays.parallelSort(((AccessibleArrayList) workingTuples).elementData, 0, workingTuples.size(), comparator);
                } else {
                    Collections.sort((List<List<?>>) workingTuples, comparator);
                }
            }
            for (List<?> list : workingTuples) {
                sublist.addTuple(list);
                if (checkLimit && sublist.getRowCount() == rowLimit) {
                    sublist.saveBatch();
                    break outer;
                }
            }
            workingTuples.clear();
            sublist.saveBatch();
        }
    } catch (BlockedException e) {
        // $NON-NLS-1$
        Assertion.failed("should not block during memory sublist sorting");
    } finally {
        bufferManager.releaseBuffers(totalReservedBuffers);
        if (this.workingBuffer != null) {
            if (this.source != null) {
                this.workingBuffer.remove();
            }
            this.workingBuffer = null;
        }
    }
    if (this.activeTupleBuffers.isEmpty()) {
        activeTupleBuffers.add(createTupleBuffer());
    }
    this.phase = MERGE;
}

Example 27 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SortUtility method createTupleBuffer.

private TupleBuffer createTupleBuffer() throws TeiidComponentException {
    TupleBuffer tb = bufferManager.createTupleBuffer(this.schema, this.groupName, TupleSourceType.PROCESSOR);
    if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
        // $NON-NLS-1$
        LogManager.logDetail(LogConstants.CTX_DQP, "Created intermediate sort buffer", tb);
    }
    tb.setForwardOnly(true);
    return tb;
}

Example 28 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SortUtility method doMerge.

protected void doMerge(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
    long desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
    int toForce = (int) Math.min(desiredSpace, Math.max(2 * schemaSize, this.bufferManager.getMaxProcessingSize()));
    int reserved = 0;
    if (desiredSpace > toForce) {
        try {
            int subLists = Math.max(2, this.bufferManager.getMaxProcessingSize() / schemaSize);
            int twoPass = subLists * subLists;
            if (twoPass < activeTupleBuffers.size()) {
                // wait for 2-pass
                int needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), .5));
                while (activeTupleBuffers.size() / needed + activeTupleBuffers.size() % needed > needed) {
                    needed++;
                }
                reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, false);
                if (reserved == 0 && twoPass * subLists < activeTupleBuffers.size()) {
                    // force 3-pass
                    needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), 1 / 3d));
                    while (activeTupleBuffers.size() / (needed * needed) + activeTupleBuffers.size() % needed > needed) {
                        needed++;
                    }
                    reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, true);
                    // $NON-NLS-1$
                    LogManager.logWarning(LogConstants.CTX_DQP, "performing three pass sort");
                }
            } else if (desiredSpace < Integer.MAX_VALUE) {
                // wait for 1-pass
                reserved += bufferManager.reserveBuffersBlocking((int) desiredSpace - toForce, attempts, false);
            }
        } catch (BlockedException be) {
            if (!nonBlocking) {
                throw be;
            }
        }
    }
    int total = reserved + toForce;
    if (total > schemaSize) {
        toForce -= total % schemaSize;
    }
    reserved += bufferManager.reserveBuffers(toForce, BufferReserveMode.FORCE);
    try {
        while (this.activeTupleBuffers.size() > 1) {
            ArrayList<SortedSublist> sublists = new ArrayList<SortedSublist>(activeTupleBuffers.size());
            TupleBuffer merged = createTupleBuffer();
            desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
            if (desiredSpace < reserved) {
                bufferManager.releaseBuffers(reserved - (int) desiredSpace);
                reserved = (int) desiredSpace;
            }
            // always allow progress
            int maxSortIndex = Math.max(2, reserved / schemaSize);
            if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_DQP, MessageLevel.TRACE)) {
                // $NON-NLS-1$ //$NON-NLS-2$
                LogManager.logTrace(org.teiid.logging.LogConstants.CTX_DQP, "Merging", maxSortIndex, "sublists out of", activeTupleBuffers.size());
            }
            // initialize the sublists with the min value
            for (int i = 0; i < maxSortIndex; i++) {
                TupleBuffer activeID = activeTupleBuffers.get(i);
                SortedSublist sortedSublist = new SortedSublist();
                sortedSublist.its = activeID.createIndexedTupleSource();
                sortedSublist.its.setNoBlocking(true);
                sortedSublist.index = i;
                incrementWorkingTuple(sublists, sortedSublist);
            }
            boolean checkLimit = maxSortIndex == activeTupleBuffers.size() && rowLimit > -1;
            // iteratively process the lowest tuple
            while (sublists.size() > 0) {
                SortedSublist sortedSublist = sublists.remove(sublists.size() - 1);
                merged.addTuple(sortedSublist.tuple);
                incrementWorkingTuple(sublists, sortedSublist);
                if (checkLimit && merged.getRowCount() == rowLimit) {
                    // early exit for row limit
                    break;
                }
            }
            // Remove merged sublists
            for (int i = 0; i < maxSortIndex; i++) {
                TupleBuffer id = activeTupleBuffers.remove(0);
                id.remove();
            }
            merged.saveBatch();
            this.activeTupleBuffers.add(merged);
        }
    } finally {
        this.bufferManager.releaseBuffers(reserved);
    }
}

Example 29 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SourceState method close.

public void close() {
    closeBuffer();
    if (buffers != null) {
        for (TupleBuffer tb : buffers) {
            tb.remove();
        }
    }
    this.buffers = null;
    this.open = false;
    if (this.sortUtility != null) {
        this.sortUtility.remove();
        this.sortUtility = null;
    }
}

Example 30 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SourceState method sort.

public void sort(SortOption sortOption) throws TeiidComponentException, TeiidProcessingException {
    if (sortOption == SortOption.ALREADY_SORTED) {
        return;
    }
    if (this.sortUtility == null) {
        TupleSource ts = null;
        if (source.hasBuffer()) {
            this.buffer = getTupleBuffer();
        } else if (this.buffer == null && this.collector != null) {
            if (sortOption == SortOption.NOT_SORTED) {
                // pass the buffer and the source
                this.buffer = this.collector.getTupleBuffer();
                ts = new BatchCollector.BatchProducerTupleSource(this.source, this.buffer.getRowCount() + 1);
            } else {
                // fully read
                this.buffer = this.collector.collectTuples();
            }
        }
        if (this.buffer != null) {
            this.buffer.setForwardOnly(true);
        } else {
            ts = new BatchIterator(this.source);
        }
        this.sortUtility = new SortUtility(ts, expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), sortOption == SortOption.SORT_DISTINCT ? Mode.DUP_REMOVE_SORT : Mode.SORT, this.source.getBufferManager(), this.source.getConnectionID(), source.getElements());
        this.markExpressionsDistinct(sortOption == SortOption.SORT_DISTINCT && expressions.size() == this.getOuterVals().size());
        if (this.buffer != null) {
            this.sortUtility.setWorkingBuffer(this.buffer);
        }
    }
    TupleBuffer sorted = null;
    if (sortOption == SortOption.NOT_SORTED) {
        if (this.buffers != null || sortUtility.isDoneReading()) {
            return;
        }
        this.buffers = sortUtility.onePassSort(limited);
        if (this.buffers.size() != 1 || !sortUtility.isDoneReading()) {
            nextBuffer();
            return;
        }
        sorted = this.buffers.get(0);
        this.buffers = null;
    } else {
        sorted = sortUtility.sort();
    }
    // only remove the buffer if this is the first time through
    if (this.buffer != null && this.buffer != sorted) {
        this.buffer.remove();
    }
    this.buffer = sorted;
    this.markExpressionsDistinct(sortUtility.isDistinct());
}