Examples with IFrameReader org.apache.hyracks.api.comm.IFrameReader used on opensource projects

Example 1 with IFrameReader

use of org.apache.hyracks.api.comm.IFrameReader in project asterixdb by apache.

the class RunMergingFrameReaderTest method testRunFileReader.

@Test
public void testRunFileReader() throws HyracksDataException {
    int pageSize = 128;
    int numRuns = 4;
    int numFramesPerRun = 4;
    int minRecordSize = pageSize / 10;
    int maxRecordSize = pageSize / 2;
    IHyracksTaskContext ctx = testUtils.create(pageSize);
    ExternalSortRunGenerator runGenerator = new ExternalSortRunGenerator(ctx, SortFields, null, ComparatorFactories, RecordDesc, Algorithm.MERGE_SORT, numFramesPerRun);
    runGenerator.open();
    Map<Integer, String> keyValuePair = new HashMap<>();
    List<IFrame> frameList = new ArrayList<>();
    prepareData(ctx, frameList, pageSize * numFramesPerRun * numRuns, minRecordSize, maxRecordSize, null, keyValuePair);
    for (IFrame frame : frameList) {
        runGenerator.nextFrame(frame.getBuffer());
    }
    numFramesPerRun = 2;
    minRecordSize = pageSize;
    maxRecordSize = pageSize;
    frameList.clear();
    prepareData(ctx, frameList, pageSize * numFramesPerRun * numRuns, minRecordSize, maxRecordSize, null, keyValuePair);
    for (IFrame frame : frameList) {
        runGenerator.nextFrame(frame.getBuffer());
    }
    runGenerator.close();
    List<IFrame> inFrame = new ArrayList<>(runGenerator.getRuns().size());
    for (GeneratedRunFileReader max : runGenerator.getRuns()) {
        inFrame.add(new GroupVSizeFrame(ctx, max.getMaxFrameSize()));
    }
    // Let each run file reader not delete the run file when it is read and closed.
    for (GeneratedRunFileReader run : runGenerator.getRuns()) {
        PA.setValue(run, "deleteAfterClose", false);
    }
    matchResult(ctx, runGenerator.getRuns(), keyValuePair);
    List<IFrameReader> runs = new ArrayList<>();
    for (GeneratedRunFileReader run : runGenerator.getRuns()) {
        runs.add(run);
    }
    RunMergingFrameReader reader = new RunMergingFrameReader(ctx, runs, inFrame, SortFields, Comparators, null, RecordDesc);
    IFrame outFrame = new VSizeFrame(ctx);
    reader.open();
    while (reader.nextFrame(outFrame)) {
        assertFrameIsSorted(outFrame, Arrays.asList(keyValuePair));
    }
    reader.close();
    assertAllKeyValueIsConsumed(Arrays.asList(keyValuePair));
}

Example 2 with IFrameReader

use of org.apache.hyracks.api.comm.IFrameReader in project asterixdb by apache.

the class MToNPartitioningMergingConnectorDescriptor method createPartitionCollector.

@Override
public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc, int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
    IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
    for (int i = 0; i < comparatorFactories.length; ++i) {
        comparators[i] = comparatorFactories[i].createBinaryComparator();
    }
    INormalizedKeyComputer nmkComputer = nkcFactory == null ? null : nkcFactory.createNormalizedKeyComputer();
    IPartitionBatchManager pbm = new NonDeterministicPartitionBatchManager(nProducerPartitions);
    IFrameReader sortMergeFrameReader = new SortMergeFrameReader(ctx, nProducerPartitions, nProducerPartitions, sortFields, comparators, nmkComputer, recordDesc, pbm);
    BitSet expectedPartitions = new BitSet();
    expectedPartitions.set(0, nProducerPartitions);
    return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, sortMergeFrameReader, pbm);
}

Example 3 with IFrameReader

use of org.apache.hyracks.api.comm.IFrameReader in project asterixdb by apache.

the class Task method pushFrames.

private void pushFrames(IPartitionCollector collector, List<PartitionChannel> inputChannels, IFrameWriter writer) throws HyracksDataException {
    if (aborted) {
        return;
    }
    try {
        collector.open();
        try {
            if (inputChannels.size() <= 0) {
                joblet.advertisePartitionRequest(taskAttemptId, collector.getRequiredPartitionIds(), collector, PartitionState.STARTED);
            } else {
                collector.addPartitions(inputChannels);
            }
            IFrameReader reader = collector.getReader();
            reader.open();
            try {
                writer.open();
                try {
                    VSizeFrame frame = new VSizeFrame(this);
                    while (reader.nextFrame(frame)) {
                        if (aborted) {
                            return;
                        }
                        ByteBuffer buffer = frame.getBuffer();
                        writer.nextFrame(buffer);
                        buffer.compact();
                    }
                } catch (Exception e) {
                    writer.fail();
                    throw e;
                } finally {
                    writer.close();
                }
            } finally {
                reader.close();
            }
        } finally {
            collector.close();
        }
    } catch (Exception e) {
        throw HyracksDataException.create(e);
    }
}

Example 4 with IFrameReader

use of org.apache.hyracks.api.comm.IFrameReader in project asterixdb by apache.

the class SortMergeFrameReader method open.

@Override
public void open() throws HyracksDataException {
    if (maxConcurrentMerges >= nSenders) {
        List<IFrame> inFrames = new ArrayList<>(nSenders);
        for (int i = 0; i < nSenders; ++i) {
            inFrames.add(new VSizeFrame(ctx));
        }
        List<IFrameReader> batch = new ArrayList<IFrameReader>(nSenders);
        pbm.getNextBatch(batch, nSenders);
        merger = new RunMergingFrameReader(ctx, batch, inFrames, sortFields, comparators, nmkComputer, recordDescriptor);
    } else {
        // multi level merge.
        throw new HyracksDataException("Not yet supported");
    }
    merger.open();
}