Examples with IFrame org.apache.hyracks.api.comm.IFrame used on opensource projects

Example 6 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class RunMergingFrameReaderTest method testRunFileReader.

@Test
public void testRunFileReader() throws HyracksDataException {
    int pageSize = 128;
    int numRuns = 4;
    int numFramesPerRun = 4;
    int minRecordSize = pageSize / 10;
    int maxRecordSize = pageSize / 2;
    IHyracksTaskContext ctx = testUtils.create(pageSize);
    ExternalSortRunGenerator runGenerator = new ExternalSortRunGenerator(ctx, SortFields, null, ComparatorFactories, RecordDesc, Algorithm.MERGE_SORT, numFramesPerRun);
    runGenerator.open();
    Map<Integer, String> keyValuePair = new HashMap<>();
    List<IFrame> frameList = new ArrayList<>();
    prepareData(ctx, frameList, pageSize * numFramesPerRun * numRuns, minRecordSize, maxRecordSize, null, keyValuePair);
    for (IFrame frame : frameList) {
        runGenerator.nextFrame(frame.getBuffer());
    }
    numFramesPerRun = 2;
    minRecordSize = pageSize;
    maxRecordSize = pageSize;
    frameList.clear();
    prepareData(ctx, frameList, pageSize * numFramesPerRun * numRuns, minRecordSize, maxRecordSize, null, keyValuePair);
    for (IFrame frame : frameList) {
        runGenerator.nextFrame(frame.getBuffer());
    }
    runGenerator.close();
    List<IFrame> inFrame = new ArrayList<>(runGenerator.getRuns().size());
    for (GeneratedRunFileReader max : runGenerator.getRuns()) {
        inFrame.add(new GroupVSizeFrame(ctx, max.getMaxFrameSize()));
    }
    // Let each run file reader not delete the run file when it is read and closed.
    for (GeneratedRunFileReader run : runGenerator.getRuns()) {
        PA.setValue(run, "deleteAfterClose", false);
    }
    matchResult(ctx, runGenerator.getRuns(), keyValuePair);
    List<IFrameReader> runs = new ArrayList<>();
    for (GeneratedRunFileReader run : runGenerator.getRuns()) {
        runs.add(run);
    }
    RunMergingFrameReader reader = new RunMergingFrameReader(ctx, runs, inFrame, SortFields, Comparators, null, RecordDesc);
    IFrame outFrame = new VSizeFrame(ctx);
    reader.open();
    while (reader.nextFrame(outFrame)) {
        assertFrameIsSorted(outFrame, Arrays.asList(keyValuePair));
    }
    reader.close();
    assertAllKeyValueIsConsumed(Arrays.asList(keyValuePair));
}

Example 7 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class AbstractRunGeneratorTest method testSortRecords.

protected List<GeneratedRunFileReader> testSortRecords(int pageSize, int frameLimit, int numRuns, int minRecordSize, int maxRecordSize, HashMap<Integer, String> specialData) throws HyracksDataException {
    IHyracksTaskContext ctx = testUtils.create(pageSize);
    HashMap<Integer, String> keyValuePair = new HashMap<>();
    List<IFrame> frameList = new ArrayList<>();
    prepareData(ctx, frameList, pageSize * frameLimit * numRuns, minRecordSize, maxRecordSize, specialData, keyValuePair);
    AbstractSortRunGenerator runGenerator = getSortRunGenerator(ctx, frameLimit, keyValuePair.size());
    runGenerator.open();
    for (IFrame frame : frameList) {
        runGenerator.nextFrame(frame.getBuffer());
    }
    runGenerator.close();
    matchResult(ctx, runGenerator.getRuns(), keyValuePair);
    return runGenerator.getRuns();
}

Example 8 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class VariableFramesMemoryManagerTest method prepareRandomTuples.

private HashMap<Integer, Integer> prepareRandomTuples() throws HyracksDataException {
    frameList.clear();
    HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
    int[] fieldSlot = { 0 };
    int id = 0;
    int size = 0;
    while (size < BUDGET) {
        int tupleLength = random.nextInt(BUDGET / 3) + 4;
        IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(FrameHelper.calcAlignedFrameSizeToStore(1, tupleLength, MIN_FRAME_SIZE)));
        IFrameTupleAppender appender = new FrameTupleAppender();
        appender.reset(frame, true);
        //            System.out.println("id:" + id + ",frameSize:" + frame.getInitialFrameSize() / MIN_FRAME_SIZE);
        ByteBuffer buffer = ByteBuffer.allocate(tupleLength);
        buffer.putInt(0, id);
        assertTrue(appender.append(fieldSlot, buffer.array(), 0, buffer.capacity()));
        set.put(id++, tupleLength);
        size += frame.getFrameSize();
        frameList.add(frame);
    }
    return set;
}

Example 9 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class VariableFramesMemoryManagerTest method prepareTuples.

private HashMap<Integer, Integer> prepareTuples() throws HyracksDataException {
    frameList.clear();
    HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
    for (int i = 0; i < NUM_MIN_FRAME; ++i) {
        IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(MIN_FRAME_SIZE));
        IFrameTupleAppender appender = new FrameTupleAppender();
        appender.reset(frame, true);
        int[] fieldSlot = { 0 };
        ByteBuffer buffer = ByteBuffer.allocate(MIN_FRAME_SIZE / 2);
        buffer.putInt(0, i);
        appender.append(fieldSlot, buffer.array(), 0, buffer.capacity());
        set.put(i, buffer.capacity());
        frameList.add(frame);
    }
    return set;
}

Example 10 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class FeedFrameUtil method removeBadTuple.

public static ByteBuffer removeBadTuple(IHyracksTaskContext ctx, int tupleIndex, FrameTupleAccessor fta) throws HyracksDataException {
    FrameTupleAppender appender = new FrameTupleAppender();
    IFrame slicedFrame = new VSizeFrame(ctx);
    appender.reset(slicedFrame, true);
    int totalTuples = fta.getTupleCount();
    for (int ti = 0; ti < totalTuples; ti++) {
        if (ti != tupleIndex) {
            appender.append(fta, ti);
        }
    }
    return slicedFrame.getBuffer();
}