Examples with IFrame org.apache.hyracks.api.comm.IFrame used on opensource projects

Example 16 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class AbstractExternalGroupbyTest method testBuildAndMerge.

private void testBuildAndMerge(int tableSize, int numFrames, int frameSize, int minDataSize, int minRecordSize, int maxRecordSize, Map<Integer, String> specialData) throws HyracksDataException {
    IHyracksTaskContext ctx = TestUtils.create(frameSize);
    initial(ctx, tableSize, numFrames);
    ArrayList<IFrame> input = new ArrayList<>();
    Map<Integer, String> keyValueMap = new HashMap<>();
    AbstractRunGeneratorTest.prepareData(ctx, input, minDataSize, minRecordSize, maxRecordSize, specialData, keyValueMap);
    ResultValidateWriter writer = new ResultValidateWriter(keyValueMap);
    try {
        getBuilder().open();
        for (IFrame frame : input) {
            getBuilder().nextFrame(frame.getBuffer());
        }
    } finally {
        getBuilder().close();
    }
    getMerger().setOutputFrameWriter(0, writer, outputRec);
    getMerger().initialize();
}

Example 17 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class RunMergingFrameReaderTest method testOneLargeRunMerge.

@Test
public void testOneLargeRunMerge() throws HyracksDataException {
    int pageSize = 64;
    int numRuns = 2;
    int numFramesPerRun = 1;
    int minRecordSize = pageSize / 10;
    int maxRecordSize = pageSize / 8;
    IHyracksTaskContext ctx = testUtils.create(pageSize);
    List<Map<Integer, String>> keyValueMap = new ArrayList<>();
    List<TestFrameReader> readerList = new ArrayList<>();
    List<IFrame> frameList = new ArrayList<>();
    prepareRandomInputRunList(ctx, pageSize, numRuns, numFramesPerRun, minRecordSize, maxRecordSize, readerList, frameList, keyValueMap);
    minRecordSize = pageSize;
    maxRecordSize = pageSize;
    numFramesPerRun = 4;
    prepareRandomInputRunList(ctx, pageSize, numRuns, numFramesPerRun, minRecordSize, maxRecordSize, readerList, frameList, keyValueMap);
    minRecordSize = pageSize * 2;
    maxRecordSize = pageSize * 2;
    numFramesPerRun = 6;
    prepareRandomInputRunList(ctx, pageSize, numRuns, numFramesPerRun, minRecordSize, maxRecordSize, readerList, frameList, keyValueMap);
    RunMergingFrameReader reader = new RunMergingFrameReader(ctx, readerList, frameList, SortFields, Comparators, null, RecordDesc);
    testMergeSucceed(ctx, reader, keyValueMap);
}

Example 18 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class TopKRunGeneratorTest method prepareSortedData.

private ByteBuffer prepareSortedData(Map<Integer, String> keyValuePair) throws HyracksDataException {
    ByteBuffer buffer = ByteBuffer.allocate(PAGE_SIZE * NUM_PAGES);
    IFrame inputFrame = new FixedSizeFrame(buffer);
    FrameTupleAppender appender = new FrameTupleAppender();
    appender.reset(inputFrame, true);
    ArrayTupleBuilder builder = new ArrayTupleBuilder(RecordDesc.getFieldCount());
    for (Map.Entry<Integer, String> entry : keyValuePair.entrySet()) {
        builder.reset();
        builder.addField(SerDers[0], entry.getKey());
        builder.addField(SerDers[1], entry.getValue());
        appender.append(builder.getFieldEndOffsets(), builder.getByteArray(), 0, builder.getSize());
    }
    return buffer;
}

Example 19 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class MultiThreadTaskEmulator method runInParallel.

public void runInParallel(final IFrameWriter[] writers, final List<IFrame>[] inputFrames) throws Exception {
    final Semaphore sem = new Semaphore(writers.length - 1);
    List<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
    for (int i = 1; i < writers.length; i++) {
        sem.acquire();
        final IFrameWriter writer = writers[i];
        final List<IFrame> inputFrame = inputFrames[i];
        executor.execute(() -> {
            executeOneWriter(writer, inputFrame, exceptions);
            sem.release();
        });
    }
    final IFrameWriter writer = writers[0];
    final List<IFrame> inputFrame = inputFrames[0];
    executeOneWriter(writer, inputFrame, exceptions);
    sem.acquire(writers.length - 1);
    for (int i = 0; i < exceptions.size(); i++) {
        exceptions.get(i).printStackTrace();
        if (i == exceptions.size() - 1) {
            throw exceptions.get(i);
        }
    }
}

Example 20 with IFrame

use of org.apache.hyracks.api.comm.IFrame in project asterixdb by apache.

the class RunMergingFrameReaderTest method testNormalRunMerge.

@Test
public void testNormalRunMerge() throws HyracksDataException {
    int pageSize = 128;
    int numRuns = 2;
    int numFramesPerRun = 2;
    int minRecordSize = pageSize / 10;
    int maxRecordSize = pageSize / 8;
    IHyracksTaskContext ctx = testUtils.create(pageSize);
    List<Map<Integer, String>> keyValueMapList = new ArrayList<>(numRuns);
    List<TestFrameReader> readerList = new ArrayList<>(numRuns);
    List<IFrame> frameList = new ArrayList<>(numRuns);
    prepareRandomInputRunList(ctx, pageSize, numRuns, numFramesPerRun, minRecordSize, maxRecordSize, readerList, frameList, keyValueMapList);
    RunMergingFrameReader reader = new RunMergingFrameReader(ctx, readerList, frameList, SortFields, Comparators, null, RecordDesc);
    testMergeSucceed(ctx, reader, keyValueMapList);
}