Examples with VSizeFrame org.apache.hyracks.api.comm.VSizeFrame used on opensource projects

Example 6 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameWithSpillNoDiscard.

/*
     * Spill = true;
     * Discard = false;
     * Fixed size frames
     */
@Test
public void testMemoryFixedSizeFrameWithSpillNoDiscard() {
    try {
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // Spill budget = Memory budget, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, DEFAULT_FRAME_SIZE * NUM_FRAMES, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
        writer.freeze();
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Next call should NOT block. we will do it in a different thread
        Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
        result.get();
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that one frame is spilled
        Assert.assertEquals(handler.getNumSpilled(), 1);
        // consume memory frames
        writer.unfreeze();
        handler.close();
        Assert.assertEquals(handler.framesOnDisk(), 0);
    // exit
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 7 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class InputHandlerTest method testMemoryFixedSizeFrameNoDiskNoDiscardFastConsumer.

/*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Very fast next operator
     */
@Test
public void testMemoryFixedSizeFrameNoDiskNoDiscardFastConsumer() {
    try {
        int numRounds = 10;
        IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
        // No spill, No discard
        FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
        // Non-Active Writer
        TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(), false);
        // FramePool
        ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
        FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
        handler.open();
        VSizeFrame frame = new VSizeFrame(ctx);
        // add NUM_FRAMES times
        for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
            handler.nextFrame(frame.getBuffer());
        }
        // Check that no records were discarded
        Assert.assertEquals(handler.getNumDiscarded(), 0);
        // Check that no records were spilled
        Assert.assertEquals(handler.getNumSpilled(), 0);
        writer.validate(false);
        handler.close();
        // Check that nextFrame was called
        Assert.assertEquals(NUM_FRAMES * numRounds, writer.nextFrameCount());
        writer.validate(true);
    } catch (Throwable th) {
        th.printStackTrace();
        Assert.fail();
    }
    Assert.assertNull(cause);
}

Example 8 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class FrameFixedFieldTupleAppenderTest method testResetShouldWork.

@Test
public void testResetShouldWork() throws HyracksDataException {
    testAppendFieldShouldSucceed();
    appender.reset(new VSizeFrame(new FrameManager(TEST_FRAME_SIZE)), true);
    testAppendFieldShouldSucceed();
}

Example 9 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class AbstractIntegrationTest method readResults.

protected List<String> readResults(JobSpecification spec, JobId jobId, ResultSetId resultSetId) throws Exception {
    int nReaders = 1;
    IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
    IHyracksDataset hyracksDataset = new HyracksDataset(hcc, spec.getFrameSize(), nReaders);
    IHyracksDatasetReader reader = hyracksDataset.createReader(jobId, resultSetId);
    List<String> resultRecords = new ArrayList<>();
    ByteBufferInputStream bbis = new ByteBufferInputStream();
    FrameManager resultDisplayFrameMgr = new FrameManager(spec.getFrameSize());
    VSizeFrame frame = new VSizeFrame(resultDisplayFrameMgr);
    int readSize = reader.read(frame);
    while (readSize > 0) {
        try {
            frameTupleAccessor.reset(frame.getBuffer());
            for (int tIndex = 0; tIndex < frameTupleAccessor.getTupleCount(); tIndex++) {
                int start = frameTupleAccessor.getTupleStartOffset(tIndex);
                int length = frameTupleAccessor.getTupleEndOffset(tIndex) - start;
                bbis.setByteBuffer(frame.getBuffer(), start);
                byte[] recordBytes = new byte[length];
                bbis.read(recordBytes, 0, length);
                resultRecords.add(new String(recordBytes, 0, length));
            }
        } finally {
            bbis.close();
        }
        readSize = reader.read(frame);
    }
    return resultRecords;
}

Example 10 with VSizeFrame

use of org.apache.hyracks.api.comm.VSizeFrame in project asterixdb by apache.

the class InputFrameGenerator method generateDataFrame.

public List<IFrame> generateDataFrame(RecordDescriptor recordDescriptor, List<Object[]> listOfObject) throws HyracksDataException {
    List<IFrame> listFrame = new ArrayList<>();
    VSizeFrame frame = new VSizeFrame(manager);
    FrameFixedFieldAppender appender = new FrameFixedFieldAppender(recordDescriptor.getFieldCount());
    appender.reset(frame, true);
    ByteArrayOutputStream baos = new ByteArrayOutputStream(manager.getInitialFrameSize());
    DataOutputStream ds = new DataOutputStream(baos);
    for (Object[] objs : listOfObject) {
        for (int i = 0; i < recordDescriptor.getFieldCount(); i++) {
            baos.reset();
            recordDescriptor.getFields()[i].serialize(objs[i], ds);
            if (!appender.appendField(baos.toByteArray(), 0, baos.size())) {
                listFrame.add(frame);
                frame = new VSizeFrame(manager);
                appender.resetWithLeftOverData(frame);
                if (!appender.appendField(baos.toByteArray(), 0, baos.size())) {
                    throw new HyracksDataException("Should never happen!");
                }
            }
        }
    }
    listFrame.add(frame);
    return listFrame;
}