use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.
the class SortUtility method sortWorking.
private void sortWorking(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
// sub-phase 2 - perform a memory sort on the workingbuffer/source
int totalReservedBuffers = 0;
try {
int maxRows = this.batchSize;
Collection<List<?>> workingTuples = null;
boolean done = false;
/*
* we can balance the work between the initial / multi-pass sort based upon the row count
* and an updated estimate of the batch memory size
*/
this.workingBuffer.close();
schemaSize = Math.max(1, this.workingBuffer.getRowSizeEstimate() * this.batchSize);
long rowCount = workingBuffer.getRowCount();
long memorySpaceNeeded = rowCount * this.workingBuffer.getRowSizeEstimate();
totalReservedBuffers = bufferManager.reserveBuffers(Math.min(bufferManager.getMaxProcessingSize(), (int) Math.min(memorySpaceNeeded, Integer.MAX_VALUE)), BufferReserveMode.FORCE);
if (totalReservedBuffers != memorySpaceNeeded) {
int processingSublists = Math.max(2, bufferManager.getMaxProcessingSize() / schemaSize);
int desiredSpace = (int) Math.min(Integer.MAX_VALUE, (workingBuffer.getRowCount() / processingSublists + (workingBuffer.getRowCount() % processingSublists)) * this.workingBuffer.getRowSizeEstimate());
if (desiredSpace > totalReservedBuffers) {
totalReservedBuffers += bufferManager.reserveBuffers(desiredSpace - totalReservedBuffers, BufferReserveMode.NO_WAIT);
// TODO: wait to force 2/3 pass processing
} else if (memorySpaceNeeded <= Integer.MAX_VALUE) {
totalReservedBuffers += bufferManager.reserveBuffers((int) memorySpaceNeeded - totalReservedBuffers, BufferReserveMode.NO_WAIT);
}
if (totalReservedBuffers > schemaSize) {
int additional = totalReservedBuffers % schemaSize;
totalReservedBuffers -= additional;
// release any excess
bufferManager.releaseBuffers(additional);
}
}
TupleBufferTupleSource ts = workingBuffer.createIndexedTupleSource(source != null);
ts.setReverse(!stableSort && workingBuffer.getRowCount() > this.batchSize);
maxRows = Math.max(1, (totalReservedBuffers / schemaSize)) * batchSize;
boolean checkLimit = rowLimit > -1 && rowCount <= maxRows;
if (mode == Mode.SORT) {
workingTuples = new AccessibleArrayList<>();
} else {
workingTuples = new TreeSet<List<?>>(comparator);
}
outer: while (!done) {
while (!done) {
if (workingTuples.size() >= maxRows) {
break;
}
List<?> tuple = ts.nextTuple();
if (tuple == null) {
done = true;
if (workingTuples.isEmpty()) {
break outer;
}
break;
}
workingTuples.add(tuple);
}
TupleBuffer sublist = createTupleBuffer();
activeTupleBuffers.add(sublist);
if (this.mode == Mode.SORT) {
// perform a stable sort
if (workingTuples.size() > (1 << 18)) {
Arrays.parallelSort(((AccessibleArrayList) workingTuples).elementData, 0, workingTuples.size(), comparator);
} else {
Collections.sort((List<List<?>>) workingTuples, comparator);
}
}
for (List<?> list : workingTuples) {
sublist.addTuple(list);
if (checkLimit && sublist.getRowCount() == rowLimit) {
sublist.saveBatch();
break outer;
}
}
workingTuples.clear();
sublist.saveBatch();
}
} catch (BlockedException e) {
// $NON-NLS-1$
Assertion.failed("should not block during memory sublist sorting");
} finally {
bufferManager.releaseBuffers(totalReservedBuffers);
if (this.workingBuffer != null) {
if (this.source != null) {
this.workingBuffer.remove();
}
this.workingBuffer = null;
}
}
if (this.activeTupleBuffers.isEmpty()) {
activeTupleBuffers.add(createTupleBuffer());
}
this.phase = MERGE;
}
use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.
the class SortUtility method createTupleBuffer.
private TupleBuffer createTupleBuffer() throws TeiidComponentException {
TupleBuffer tb = bufferManager.createTupleBuffer(this.schema, this.groupName, TupleSourceType.PROCESSOR);
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
// $NON-NLS-1$
LogManager.logDetail(LogConstants.CTX_DQP, "Created intermediate sort buffer", tb);
}
tb.setForwardOnly(true);
return tb;
}
use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.
the class SortUtility method doMerge.
protected void doMerge(int rowLimit) throws TeiidComponentException, TeiidProcessingException {
long desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
int toForce = (int) Math.min(desiredSpace, Math.max(2 * schemaSize, this.bufferManager.getMaxProcessingSize()));
int reserved = 0;
if (desiredSpace > toForce) {
try {
int subLists = Math.max(2, this.bufferManager.getMaxProcessingSize() / schemaSize);
int twoPass = subLists * subLists;
if (twoPass < activeTupleBuffers.size()) {
// wait for 2-pass
int needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), .5));
while (activeTupleBuffers.size() / needed + activeTupleBuffers.size() % needed > needed) {
needed++;
}
reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, false);
if (reserved == 0 && twoPass * subLists < activeTupleBuffers.size()) {
// force 3-pass
needed = (int) Math.ceil(Math.pow(activeTupleBuffers.size(), 1 / 3d));
while (activeTupleBuffers.size() / (needed * needed) + activeTupleBuffers.size() % needed > needed) {
needed++;
}
reserved += bufferManager.reserveBuffersBlocking(needed * schemaSize - toForce, attempts, true);
// $NON-NLS-1$
LogManager.logWarning(LogConstants.CTX_DQP, "performing three pass sort");
}
} else if (desiredSpace < Integer.MAX_VALUE) {
// wait for 1-pass
reserved += bufferManager.reserveBuffersBlocking((int) desiredSpace - toForce, attempts, false);
}
} catch (BlockedException be) {
if (!nonBlocking) {
throw be;
}
}
}
int total = reserved + toForce;
if (total > schemaSize) {
toForce -= total % schemaSize;
}
reserved += bufferManager.reserveBuffers(toForce, BufferReserveMode.FORCE);
try {
while (this.activeTupleBuffers.size() > 1) {
ArrayList<SortedSublist> sublists = new ArrayList<SortedSublist>(activeTupleBuffers.size());
TupleBuffer merged = createTupleBuffer();
desiredSpace = activeTupleBuffers.size() * (long) schemaSize;
if (desiredSpace < reserved) {
bufferManager.releaseBuffers(reserved - (int) desiredSpace);
reserved = (int) desiredSpace;
}
// always allow progress
int maxSortIndex = Math.max(2, reserved / schemaSize);
if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_DQP, MessageLevel.TRACE)) {
// $NON-NLS-1$ //$NON-NLS-2$
LogManager.logTrace(org.teiid.logging.LogConstants.CTX_DQP, "Merging", maxSortIndex, "sublists out of", activeTupleBuffers.size());
}
// initialize the sublists with the min value
for (int i = 0; i < maxSortIndex; i++) {
TupleBuffer activeID = activeTupleBuffers.get(i);
SortedSublist sortedSublist = new SortedSublist();
sortedSublist.its = activeID.createIndexedTupleSource();
sortedSublist.its.setNoBlocking(true);
sortedSublist.index = i;
incrementWorkingTuple(sublists, sortedSublist);
}
boolean checkLimit = maxSortIndex == activeTupleBuffers.size() && rowLimit > -1;
// iteratively process the lowest tuple
while (sublists.size() > 0) {
SortedSublist sortedSublist = sublists.remove(sublists.size() - 1);
merged.addTuple(sortedSublist.tuple);
incrementWorkingTuple(sublists, sortedSublist);
if (checkLimit && merged.getRowCount() == rowLimit) {
// early exit for row limit
break;
}
}
// Remove merged sublists
for (int i = 0; i < maxSortIndex; i++) {
TupleBuffer id = activeTupleBuffers.remove(0);
id.remove();
}
merged.saveBatch();
this.activeTupleBuffers.add(merged);
}
} finally {
this.bufferManager.releaseBuffers(reserved);
}
}
use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.
the class SourceState method close.
public void close() {
closeBuffer();
if (buffers != null) {
for (TupleBuffer tb : buffers) {
tb.remove();
}
}
this.buffers = null;
this.open = false;
if (this.sortUtility != null) {
this.sortUtility.remove();
this.sortUtility = null;
}
}
use of org.teiid.common.buffer.TupleBuffer in project teiid by teiid.
the class SourceState method sort.
public void sort(SortOption sortOption) throws TeiidComponentException, TeiidProcessingException {
if (sortOption == SortOption.ALREADY_SORTED) {
return;
}
if (this.sortUtility == null) {
TupleSource ts = null;
if (source.hasBuffer()) {
this.buffer = getTupleBuffer();
} else if (this.buffer == null && this.collector != null) {
if (sortOption == SortOption.NOT_SORTED) {
// pass the buffer and the source
this.buffer = this.collector.getTupleBuffer();
ts = new BatchCollector.BatchProducerTupleSource(this.source, this.buffer.getRowCount() + 1);
} else {
// fully read
this.buffer = this.collector.collectTuples();
}
}
if (this.buffer != null) {
this.buffer.setForwardOnly(true);
} else {
ts = new BatchIterator(this.source);
}
this.sortUtility = new SortUtility(ts, expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), sortOption == SortOption.SORT_DISTINCT ? Mode.DUP_REMOVE_SORT : Mode.SORT, this.source.getBufferManager(), this.source.getConnectionID(), source.getElements());
this.markExpressionsDistinct(sortOption == SortOption.SORT_DISTINCT && expressions.size() == this.getOuterVals().size());
if (this.buffer != null) {
this.sortUtility.setWorkingBuffer(this.buffer);
}
}
TupleBuffer sorted = null;
if (sortOption == SortOption.NOT_SORTED) {
if (this.buffers != null || sortUtility.isDoneReading()) {
return;
}
this.buffers = sortUtility.onePassSort(limited);
if (this.buffers.size() != 1 || !sortUtility.isDoneReading()) {
nextBuffer();
return;
}
sorted = this.buffers.get(0);
this.buffers = null;
} else {
sorted = sortUtility.sort();
}
// only remove the buffer if this is the first time through
if (this.buffer != null && this.buffer != sorted) {
this.buffer.remove();
}
this.buffer = sorted;
this.markExpressionsDistinct(sortUtility.isDistinct());
}
Aggregations