Examples with VariableFrameMemoryManager org.apache.hyracks.dataflow.std.buffermanager.VariableFrameMemoryManager used on opensource projects

Example 1 with VariableFrameMemoryManager

use of org.apache.hyracks.dataflow.std.buffermanager.VariableFrameMemoryManager in project asterixdb by apache.

the class InMemorySortRuntimeFactory method createOneOutputPushRuntime.

@Override
public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException {
    return new AbstractOneInputOneOutputPushRuntime() {

        FrameSorterMergeSort frameSorter = null;

        @Override
        public void open() throws HyracksDataException {
            writer.open();
            if (frameSorter == null) {
                IFrameBufferManager manager = new VariableFrameMemoryManager(new VariableFramePool(ctx, VariableFramePool.UNLIMITED_MEMORY), FrameFreeSlotPolicyFactory.createFreeSlotPolicy(EnumFreeSlotPolicy.LAST_FIT));
                frameSorter = new FrameSorterMergeSort(ctx, manager, sortFields, firstKeyNormalizerFactory, comparatorFactories, outputRecordDesc);
            }
            frameSorter.reset();
        }

        @Override
        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
            frameSorter.insertFrame(buffer);
        }

        @Override
        public void fail() throws HyracksDataException {
            writer.fail();
        }

        @Override
        public void close() throws HyracksDataException {
            try {
                frameSorter.sort();
                frameSorter.flush(writer);
            } finally {
                writer.close();
            }
        }
    };
}

Example 2 with VariableFrameMemoryManager

use of org.apache.hyracks.dataflow.std.buffermanager.VariableFrameMemoryManager in project asterixdb by apache.

the class HybridTopKSortRunGenerator method nextFrame.

@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
    if (topK <= 0) {
        return;
    }
    inAccessor.reset(buffer);
    if (tupleSorter != null) {
        boolean isBadK = false;
        for (int i = 0; i < inAccessor.getTupleCount(); i++) {
            if (!tupleSorter.insertTuple(inAccessor, i)) {
                flushFramesToRun();
                isBadK = true;
                if (!tupleSorter.insertTuple(inAccessor, i)) {
                    throw new HyracksDataException("The given tuple is too big to insert into the sorting memory.");
                }
            }
        }
        if (isBadK) {
            tupleSorterFlushedTimes++;
            if (tupleSorterFlushedTimes > SWITCH_TO_FRAME_SORTER_THRESHOLD) {
                if (tupleSorter.hasRemaining()) {
                    flushFramesToRun();
                }
                tupleSorter.close();
                tupleSorter = null;
                if (LOG.isLoggable(Level.FINE)) {
                    LOG.fine("clear tupleSorter");
                }
            }
        }
    } else {
        if (frameSorter == null) {
            VariableFrameMemoryManager bufferManager = new VariableFrameMemoryManager(new VariableFramePool(ctx, (frameLimit - 1) * ctx.getInitialFrameSize()), FrameFreeSlotPolicyFactory.createFreeSlotPolicy(EnumFreeSlotPolicy.BIGGEST_FIT, frameLimit - 1));
            frameSorter = new FrameSorterMergeSort(ctx, bufferManager, sortFields, nmkFactory, comparatorFactories, recordDescriptor, topK);
            if (LOG.isLoggable(Level.FINE)) {
                LOG.fine("create frameSorter");
            }
        }
        if (!frameSorter.insertFrame(buffer)) {
            flushFramesToRun();
            if (!frameSorter.insertFrame(buffer)) {
                throw new HyracksDataException("The given frame is too big to insert into the sorting memory.");
            }
        }
    }
}