Search in sources :

Example 26 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SortUtility method sortWorking.

private void sortWorking(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
    // sub-phase 2 - perform a memory sort on the workingbuffer/source
    int totalReservedBuffers = 0;
    try {
        int maxRows = this.batchSize;
        Collection<List<?>> workingTuples = null;
        boolean done = false;
        /*
			 * we can balance the work between the initial / multi-pass sort based upon the row count
			 * and an updated estimate of the batch memory size 
			 */
        this.workingBuffer.close();
        schemaSize = Math.max(1, this.workingBuffer.getRowSizeEstimate() * this.batchSize);
        long rowCount = workingBuffer.getRowCount();
        long memorySpaceNeeded = rowCount * this.workingBuffer.getRowSizeEstimate();
        totalReservedBuffers = bufferManager.reserveBuffers(Math.min(bufferManager.getMaxProcessingSize(), (int) Math.min(memorySpaceNeeded, Integer.MAX_VALUE)), BufferReserveMode.FORCE);
        if (totalReservedBuffers != memorySpaceNeeded) {
            int processingSublists = Math.max(2, bufferManager.getMaxProcessingSize() / schemaSize);
            int desiredSpace = (int) Math.min(Integer.MAX_VALUE, (workingBuffer.getRowCount() / processingSublists + (workingBuffer.getRowCount() % processingSublists)) * this.workingBuffer.getRowSizeEstimate());
            if (desiredSpace > totalReservedBuffers) {
                totalReservedBuffers += bufferManager.reserveBuffers(desiredSpace - totalReservedBuffers, BufferReserveMode.NO_WAIT);
            // TODO: wait to force 2/3 pass processing
            } else if (memorySpaceNeeded <= Integer.MAX_VALUE) {
                totalReservedBuffers += bufferManager.reserveBuffers((int) memorySpaceNeeded - totalReservedBuffers, BufferReserveMode.NO_WAIT);
            }
            if (totalReservedBuffers > schemaSize) {
                int additional = totalReservedBuffers % schemaSize;
                totalReservedBuffers -= additional;
                // release any excess
                bufferManager.releaseBuffers(additional);
            }
        }
        TupleBufferTupleSource ts = workingBuffer.createIndexedTupleSource(source != null);
        ts.setReverse(!stableSort && workingBuffer.getRowCount() > this.batchSize);
        maxRows = Math.max(1, (totalReservedBuffers / schemaSize)) * batchSize;
        boolean checkLimit = rowLimit > -1 && rowCount <= maxRows;
        if (mode == Mode.SORT) {
            workingTuples = new AccessibleArrayList<>();
        } else {
            workingTuples = new TreeSet<List<?>>(comparator);
        }
        outer: while (!done) {
            while (!done) {
                if (workingTuples.size() >= maxRows) {
                    break;
                }
                List<?> tuple = ts.nextTuple();
                if (tuple == null) {
                    done = true;
                    if (workingTuples.isEmpty()) {
                        break outer;
                    }
                    break;
                }
                workingTuples.add(tuple);
            }
            TupleBuffer sublist = createTupleBuffer();
            activeTupleBuffers.add(sublist);
            if (this.mode == Mode.SORT) {
                // perform a stable sort
                if (workingTuples.size() > (1 << 18)) {
                    Arrays.parallelSort(((AccessibleArrayList) workingTuples).elementData, 0, workingTuples.size(), comparator);
                } else {
                    Collections.sort((List<List<?>>) workingTuples, comparator);
                }
            }
            for (List<?> list : workingTuples) {
                sublist.addTuple(list);
                if (checkLimit && sublist.getRowCount() == rowLimit) {
                    sublist.saveBatch();
                    break outer;
                }
            }
            workingTuples.clear();
            sublist.saveBatch();
        }
    } catch (BlockedException e) {
        // $NON-NLS-1$
        Assertion.failed("should not block during memory sublist sorting");
    } finally {
        bufferManager.releaseBuffers(totalReservedBuffers);
        if (this.workingBuffer != null) {
            if (this.source != null) {
                this.workingBuffer.remove();
            }
            this.workingBuffer = null;
        }
    }
    if (this.activeTupleBuffers.isEmpty()) {
        activeTupleBuffers.add(createTupleBuffer());
    }
    this.phase = MERGE;
}
Also used : TupleBufferTupleSource(org.teiid.common.buffer.TupleBuffer.TupleBufferTupleSource) TupleBuffer(org.teiid.common.buffer.TupleBuffer) AbstractList(java.util.AbstractList) ArrayList(java.util.ArrayList) List(java.util.List) BlockedException(org.teiid.common.buffer.BlockedException)

Example 27 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SortUtility method createTupleBuffer.

private TupleBuffer createTupleBuffer() throws TeiidComponentException {
    TupleBuffer tb = bufferManager.createTupleBuffer(this.schema, this.groupName, TupleSourceType.PROCESSOR);
    if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
        // $NON-NLS-1$
        LogManager.logDetail(LogConstants.CTX_DQP, "Created intermediate sort buffer", tb);
    }
    tb.setForwardOnly(true);
    return tb;
}
Also used : TupleBuffer(org.teiid.common.buffer.TupleBuffer)

Example 28 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SortUtility method doMerge.

protected void doMerge(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
    long desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
    int toForce = (int) Math.min(desiredSpace, Math.max(2 * schemaSize, this.bufferManager.getMaxProcessingSize()));
    int reserved = 0;
    if (desiredSpace > toForce) {
        try {
            int subLists = Math.max(2, this.bufferManager.getMaxProcessingSize() / schemaSize);
            int twoPass = subLists * subLists;
            if (twoPass < activeTupleBuffers.size()) {
                // wait for 2-pass
                int needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), .5));
                while (activeTupleBuffers.size() / needed + activeTupleBuffers.size() % needed > needed) {
                    needed++;
                }
                reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, false);
                if (reserved == 0 && twoPass * subLists < activeTupleBuffers.size()) {
                    // force 3-pass
                    needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), 1 / 3d));
                    while (activeTupleBuffers.size() / (needed * needed) + activeTupleBuffers.size() % needed > needed) {
                        needed++;
                    }
                    reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, true);
                    // $NON-NLS-1$
                    LogManager.logWarning(LogConstants.CTX_DQP, "performing three pass sort");
                }
            } else if (desiredSpace < Integer.MAX_VALUE) {
                // wait for 1-pass
                reserved += bufferManager.reserveBuffersBlocking((int) desiredSpace - toForce, attempts, false);
            }
        } catch (BlockedException be) {
            if (!nonBlocking) {
                throw be;
            }
        }
    }
    int total = reserved + toForce;
    if (total > schemaSize) {
        toForce -= total % schemaSize;
    }
    reserved += bufferManager.reserveBuffers(toForce, BufferReserveMode.FORCE);
    try {
        while (this.activeTupleBuffers.size() > 1) {
            ArrayList<SortedSublist> sublists = new ArrayList<SortedSublist>(activeTupleBuffers.size());
            TupleBuffer merged = createTupleBuffer();
            desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
            if (desiredSpace < reserved) {
                bufferManager.releaseBuffers(reserved - (int) desiredSpace);
                reserved = (int) desiredSpace;
            }
            // always allow progress
            int maxSortIndex = Math.max(2, reserved / schemaSize);
            if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_DQP, MessageLevel.TRACE)) {
                // $NON-NLS-1$ //$NON-NLS-2$
                LogManager.logTrace(org.teiid.logging.LogConstants.CTX_DQP, "Merging", maxSortIndex, "sublists out of", activeTupleBuffers.size());
            }
            // initialize the sublists with the min value
            for (int i = 0; i < maxSortIndex; i++) {
                TupleBuffer activeID = activeTupleBuffers.get(i);
                SortedSublist sortedSublist = new SortedSublist();
                sortedSublist.its = activeID.createIndexedTupleSource();
                sortedSublist.its.setNoBlocking(true);
                sortedSublist.index = i;
                incrementWorkingTuple(sublists, sortedSublist);
            }
            boolean checkLimit = maxSortIndex == activeTupleBuffers.size() && rowLimit > -1;
            // iteratively process the lowest tuple
            while (sublists.size() > 0) {
                SortedSublist sortedSublist = sublists.remove(sublists.size() - 1);
                merged.addTuple(sortedSublist.tuple);
                incrementWorkingTuple(sublists, sortedSublist);
                if (checkLimit && merged.getRowCount() == rowLimit) {
                    // early exit for row limit
                    break;
                }
            }
            // Remove merged sublists
            for (int i = 0; i < maxSortIndex; i++) {
                TupleBuffer id = activeTupleBuffers.remove(0);
                id.remove();
            }
            merged.saveBatch();
            this.activeTupleBuffers.add(merged);
        }
    } finally {
        this.bufferManager.releaseBuffers(reserved);
    }
}
Also used : ArrayList(java.util.ArrayList) TupleBuffer(org.teiid.common.buffer.TupleBuffer) BlockedException(org.teiid.common.buffer.BlockedException)

Example 29 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SourceState method close.

public void close() {
    closeBuffer();
    if (buffers != null) {
        for (TupleBuffer tb : buffers) {
            tb.remove();
        }
    }
    this.buffers = null;
    this.open = false;
    if (this.sortUtility != null) {
        this.sortUtility.remove();
        this.sortUtility = null;
    }
}
Also used : TupleBuffer(org.teiid.common.buffer.TupleBuffer)

Example 30 with TupleBuffer

use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.

the class SourceState method sort.

public void sort(SortOption sortOption) throws TeiidComponentException, TeiidProcessingException {
    if (sortOption == SortOption.ALREADY_SORTED) {
        return;
    }
    if (this.sortUtility == null) {
        TupleSource ts = null;
        if (source.hasBuffer()) {
            this.buffer = getTupleBuffer();
        } else if (this.buffer == null && this.collector != null) {
            if (sortOption == SortOption.NOT_SORTED) {
                // pass the buffer and the source
                this.buffer = this.collector.getTupleBuffer();
                ts = new BatchCollector.BatchProducerTupleSource(this.source, this.buffer.getRowCount() + 1);
            } else {
                // fully read
                this.buffer = this.collector.collectTuples();
            }
        }
        if (this.buffer != null) {
            this.buffer.setForwardOnly(true);
        } else {
            ts = new BatchIterator(this.source);
        }
        this.sortUtility = new SortUtility(ts, expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), sortOption == SortOption.SORT_DISTINCT ? Mode.DUP_REMOVE_SORT : Mode.SORT, this.source.getBufferManager(), this.source.getConnectionID(), source.getElements());
        this.markExpressionsDistinct(sortOption == SortOption.SORT_DISTINCT && expressions.size() == this.getOuterVals().size());
        if (this.buffer != null) {
            this.sortUtility.setWorkingBuffer(this.buffer);
        }
    }
    TupleBuffer sorted = null;
    if (sortOption == SortOption.NOT_SORTED) {
        if (this.buffers != null || sortUtility.isDoneReading()) {
            return;
        }
        this.buffers = sortUtility.onePassSort(limited);
        if (this.buffers.size() != 1 || !sortUtility.isDoneReading()) {
            nextBuffer();
            return;
        }
        sorted = this.buffers.get(0);
        this.buffers = null;
    } else {
        sorted = sortUtility.sort();
    }
    // only remove the buffer if this is the first time through
    if (this.buffer != null && this.buffer != sorted) {
        this.buffer.remove();
    }
    this.buffer = sorted;
    this.markExpressionsDistinct(sortUtility.isDistinct());
}
Also used : IndexedTupleSource(org.teiid.common.buffer.IndexedTupleSource) TupleSource(org.teiid.common.buffer.TupleSource) TupleBuffer(org.teiid.common.buffer.TupleBuffer) BatchIterator(org.teiid.query.processor.BatchIterator)

Aggregations

TupleBuffer (org.teiid.common.buffer.TupleBuffer)43 ElementSymbol (org.teiid.query.sql.symbol.ElementSymbol)19 BufferManager (org.teiid.common.buffer.BufferManager)16 List (java.util.List)15 Test (org.junit.Test)15 ArrayList (java.util.ArrayList)11 BlockedException (org.teiid.common.buffer.BlockedException)10 TupleSource (org.teiid.common.buffer.TupleSource)10 CommandContext (org.teiid.query.util.CommandContext)7 TeiidProcessingException (org.teiid.core.TeiidProcessingException)6 FakeRelationalNode (org.teiid.query.processor.relational.FakeRelationalNode)6 TeiidComponentException (org.teiid.core.TeiidComponentException)5 TupleBatch (org.teiid.common.buffer.TupleBatch)4 Map (java.util.Map)3 CollectionTupleSource (org.teiid.query.processor.CollectionTupleSource)3 Expression (org.teiid.query.sql.symbol.Expression)3 HashMap (java.util.HashMap)2 LinkedHashMap (java.util.LinkedHashMap)2 LinkedList (java.util.LinkedList)2 IndexedTupleSource (org.teiid.common.buffer.IndexedTupleSource)2