Examples with SpilledBufferOrEventSequence org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence used on opensource projects

Example 6 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testBufferSequenceWithIncompleteBuffer.

@Test
public void testBufferSequenceWithIncompleteBuffer() {
    try {
        writeBuffer(fileChannel, 1672, 7);
        // write an incomplete buffer
        ByteBuffer data = ByteBuffer.allocate(615);
        data.order(ByteOrder.LITTLE_ENDIAN);
        data.putInt(2);
        data.putInt(999);
        data.put((byte) 0);
        data.position(0);
        data.limit(312);
        fileChannel.write(data);
        fileChannel.position(0L);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        // first one is valid
        validateBuffer(seq.getNext(), 1672, 7);
        // next one should fail
        try {
            seq.getNext();
            fail("should fail with an exception");
        } catch (IOException e) {
        // expected
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 7 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testEventSequence.

@Test
public void testEventSequence() {
    try {
        final Random rnd = new Random();
        final int numEvents = 3000;
        final int numChannels = 1656;
        final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(numEvents);
        for (int i = 0; i < numEvents; i++) {
            events.add(generateAndWriteEvent(fileChannel, rnd, numChannels));
        }
        fileChannel.position(0L);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        int i = 0;
        BufferOrEvent boe;
        while ((boe = seq.getNext()) != null) {
            BufferOrEvent expected = events.get(i);
            assertTrue(boe.isEvent());
            assertEquals(expected.getEvent(), boe.getEvent());
            assertEquals(expected.getChannelIndex(), boe.getChannelIndex());
            i++;
        }
        assertEquals(numEvents, i);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 8 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testCleanup.

@Test
public void testCleanup() {
    try {
        ByteBuffer data = ByteBuffer.allocate(157);
        data.order(ByteOrder.LITTLE_ENDIAN);
        fileChannel.write(data);
        fileChannel.position(54);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        seq.cleanup();
        assertFalse(fileChannel.isOpen());
        assertFalse(tempFile.exists());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}