Examples with TupleBufferTupleSource org.teiid.common.buffer.TupleBuffer.TupleBufferTupleSource used on opensource projects

Example 1 with TupleBufferTupleSource

use of org.teiid.common.buffer.TupleBuffer.TupleBufferTupleSource in project teiid by teiid.

the class SortUtility method sortWorking.

private void sortWorking(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
    // sub-phase 2 - perform a memory sort on the workingbuffer/source
    int totalReservedBuffers = 0;
    try {
        int maxRows = this.batchSize;
        Collection<List<?>> workingTuples = null;
        boolean done = false;
        /*
			 * we can balance the work between the initial / multi-pass sort based upon the row count
			 * and an updated estimate of the batch memory size 
			 */
        this.workingBuffer.close();
        schemaSize = Math.max(1, this.workingBuffer.getRowSizeEstimate() * this.batchSize);
        long rowCount = workingBuffer.getRowCount();
        long memorySpaceNeeded = rowCount * this.workingBuffer.getRowSizeEstimate();
        totalReservedBuffers = bufferManager.reserveBuffers(Math.min(bufferManager.getMaxProcessingSize(), (int) Math.min(memorySpaceNeeded, Integer.MAX_VALUE)), BufferReserveMode.FORCE);
        if (totalReservedBuffers != memorySpaceNeeded) {
            int processingSublists = Math.max(2, bufferManager.getMaxProcessingSize() / schemaSize);
            int desiredSpace = (int) Math.min(Integer.MAX_VALUE, (workingBuffer.getRowCount() / processingSublists + (workingBuffer.getRowCount() % processingSublists)) * this.workingBuffer.getRowSizeEstimate());
            if (desiredSpace > totalReservedBuffers) {
                totalReservedBuffers += bufferManager.reserveBuffers(desiredSpace - totalReservedBuffers, BufferReserveMode.NO_WAIT);
            // TODO: wait to force 2/3 pass processing
            } else if (memorySpaceNeeded <= Integer.MAX_VALUE) {
                totalReservedBuffers += bufferManager.reserveBuffers((int) memorySpaceNeeded - totalReservedBuffers, BufferReserveMode.NO_WAIT);
            }
            if (totalReservedBuffers > schemaSize) {
                int additional = totalReservedBuffers % schemaSize;
                totalReservedBuffers -= additional;
                // release any excess
                bufferManager.releaseBuffers(additional);
            }
        }
        TupleBufferTupleSource ts = workingBuffer.createIndexedTupleSource(source != null);
        ts.setReverse(!stableSort && workingBuffer.getRowCount() > this.batchSize);
        maxRows = Math.max(1, (totalReservedBuffers / schemaSize)) * batchSize;
        boolean checkLimit = rowLimit > -1 && rowCount <= maxRows;
        if (mode == Mode.SORT) {
            workingTuples = new AccessibleArrayList<>();
        } else {
            workingTuples = new TreeSet<List<?>>(comparator);
        }
        outer: while (!done) {
            while (!done) {
                if (workingTuples.size() >= maxRows) {
                    break;
                }
                List<?> tuple = ts.nextTuple();
                if (tuple == null) {
                    done = true;
                    if (workingTuples.isEmpty()) {
                        break outer;
                    }
                    break;
                }
                workingTuples.add(tuple);
            }
            TupleBuffer sublist = createTupleBuffer();
            activeTupleBuffers.add(sublist);
            if (this.mode == Mode.SORT) {
                // perform a stable sort
                if (workingTuples.size() > (1 << 18)) {
                    Arrays.parallelSort(((AccessibleArrayList) workingTuples).elementData, 0, workingTuples.size(), comparator);
                } else {
                    Collections.sort((List<List<?>>) workingTuples, comparator);
                }
            }
            for (List<?> list : workingTuples) {
                sublist.addTuple(list);
                if (checkLimit && sublist.getRowCount() == rowLimit) {
                    sublist.saveBatch();
                    break outer;
                }
            }
            workingTuples.clear();
            sublist.saveBatch();
        }
    } catch (BlockedException e) {
        // $NON-NLS-1$
        Assertion.failed("should not block during memory sublist sorting");
    } finally {
        bufferManager.releaseBuffers(totalReservedBuffers);
        if (this.workingBuffer != null) {
            if (this.source != null) {
                this.workingBuffer.remove();
            }
            this.workingBuffer = null;
        }
    }
    if (this.activeTupleBuffers.isEmpty()) {
        activeTupleBuffers.add(createTupleBuffer());
    }
    this.phase = MERGE;
}

Example 2 with TupleBufferTupleSource

use of org.teiid.common.buffer.TupleBuffer.TupleBufferTupleSource in project teiid by teiid.

the class TestTupleBuffer method testReverseIteration.

@Test
public void testReverseIteration() throws Exception {
    // $NON-NLS-1$
    ElementSymbol x = new ElementSymbol("x");
    x.setType(DataTypeManager.DefaultDataClasses.INTEGER);
    List<ElementSymbol> schema = Arrays.asList(x);
    // $NON-NLS-1$
    TupleBuffer tb = BufferManagerFactory.getStandaloneBufferManager().createTupleBuffer(schema, "x", TupleSourceType.PROCESSOR);
    tb.addTuple(Arrays.asList(1));
    tb.addTuple(Arrays.asList(2));
    TupleBufferTupleSource tbts = tb.createIndexedTupleSource();
    tbts.setReverse(true);
    assertTrue(tbts.hasNext());
    assertEquals(2, tbts.nextTuple().get(0));
    assertEquals(1, tbts.nextTuple().get(0));
    assertFalse(tbts.hasNext());
}