Examples with SpilledBufferOrEventSequence org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence used on opensource projects

Example 1 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testIncompleteHeaderOnFirstElement.

@Test
public void testIncompleteHeaderOnFirstElement() {
    try {
        ByteBuffer buf = ByteBuffer.allocate(7);
        buf.order(ByteOrder.LITTLE_ENDIAN);
        fileChannel.write(buf);
        fileChannel.position(0);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        try {
            seq.getNext();
            fail("should fail with an exception");
        } catch (IOException e) {
        // expected
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testMultipleSequences.

@Test
public void testMultipleSequences() {
    File secondFile = null;
    FileChannel secondChannel = null;
    try {
        // create the second file channel
        secondFile = File.createTempFile("testdata", "tmp");
        secondChannel = new RandomAccessFile(secondFile, "rw").getChannel();
        final Random rnd = new Random();
        final Random bufferRnd = new Random();
        final long bufferSeed = rnd.nextLong();
        bufferRnd.setSeed(bufferSeed);
        final int numEventsAndBuffers1 = 272;
        final int numEventsAndBuffers2 = 151;
        final int numChannels = 1656;
        final ArrayList<BufferOrEvent> events1 = new ArrayList<BufferOrEvent>(128);
        final ArrayList<BufferOrEvent> events2 = new ArrayList<BufferOrEvent>(128);
        for (int i = 0; i < numEventsAndBuffers1; i++) {
            boolean isEvent = rnd.nextDouble() < 0.05d;
            if (isEvent) {
                events1.add(generateAndWriteEvent(fileChannel, rnd, numChannels));
            } else {
                writeBuffer(fileChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        for (int i = 0; i < numEventsAndBuffers2; i++) {
            boolean isEvent = rnd.nextDouble() < 0.05d;
            if (isEvent) {
                events2.add(generateAndWriteEvent(secondChannel, rnd, numChannels));
            } else {
                writeBuffer(secondChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        // reset and create reader
        fileChannel.position(0L);
        secondChannel.position(0L);
        bufferRnd.setSeed(bufferSeed);
        SpilledBufferOrEventSequence seq1 = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        SpilledBufferOrEventSequence seq2 = new SpilledBufferOrEventSequence(secondFile, secondChannel, buffer, pageSize);
        // read and validate the sequence 1
        seq1.open();
        int numEvent = 0;
        for (int i = 0; i < numEventsAndBuffers1; i++) {
            BufferOrEvent next = seq1.getNext();
            if (next.isEvent()) {
                BufferOrEvent expected = events1.get(numEvent++);
                assertEquals(expected.getEvent(), next.getEvent());
                assertEquals(expected.getChannelIndex(), next.getChannelIndex());
            } else {
                validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        assertNull(seq1.getNext());
        assertEquals(events1.size(), numEvent);
        // read and validate the sequence 2
        seq2.open();
        numEvent = 0;
        for (int i = 0; i < numEventsAndBuffers2; i++) {
            BufferOrEvent next = seq2.getNext();
            if (next.isEvent()) {
                BufferOrEvent expected = events2.get(numEvent++);
                assertEquals(expected.getEvent(), next.getEvent());
                assertEquals(expected.getChannelIndex(), next.getChannelIndex());
            } else {
                validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        assertNull(seq2.getNext());
        assertEquals(events2.size(), numEvent);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (secondChannel != null) {
            try {
                secondChannel.close();
            } catch (IOException e) {
            // ignore here
            }
        }
        if (secondFile != null) {
            //noinspection ResultOfMethodCallIgnored
            secondFile.delete();
        }
    }
}

Example 3 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testEmptyChannel.

// ------------------------------------------------------------------------
//  Tests
// ------------------------------------------------------------------------
@Test
public void testEmptyChannel() {
    try {
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        assertNull(seq.getNext());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testBufferSequence.

@Test
public void testBufferSequence() {
    try {
        final Random rnd = new Random();
        final long seed = rnd.nextLong();
        final int numBuffers = 325;
        final int numChannels = 671;
        rnd.setSeed(seed);
        for (int i = 0; i < numBuffers; i++) {
            writeBuffer(fileChannel, rnd.nextInt(pageSize) + 1, rnd.nextInt(numChannels));
        }
        fileChannel.position(0L);
        rnd.setSeed(seed);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        for (int i = 0; i < numBuffers; i++) {
            validateBuffer(seq.getNext(), rnd.nextInt(pageSize) + 1, rnd.nextInt(numChannels));
        }
        // should have no more data
        assertNull(seq.getNext());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 5 with SpilledBufferOrEventSequence

use of org.apache.flink.streaming.runtime.io.BufferSpiller.SpilledBufferOrEventSequence in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testMixedSequence.

@Test
public void testMixedSequence() {
    try {
        final Random rnd = new Random();
        final Random bufferRnd = new Random();
        final long bufferSeed = rnd.nextLong();
        bufferRnd.setSeed(bufferSeed);
        final int numEventsAndBuffers = 3000;
        final int numChannels = 1656;
        final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);
        for (int i = 0; i < numEventsAndBuffers; i++) {
            boolean isEvent = rnd.nextDouble() < 0.05d;
            if (isEvent) {
                events.add(generateAndWriteEvent(fileChannel, rnd, numChannels));
            } else {
                writeBuffer(fileChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        // reset and create reader
        fileChannel.position(0L);
        bufferRnd.setSeed(bufferSeed);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        // read and validate the sequence
        int numEvent = 0;
        for (int i = 0; i < numEventsAndBuffers; i++) {
            BufferOrEvent next = seq.getNext();
            if (next.isEvent()) {
                BufferOrEvent expected = events.get(numEvent++);
                assertEquals(expected.getEvent(), next.getEvent());
                assertEquals(expected.getChannelIndex(), next.getChannelIndex());
            } else {
                validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        // no further data
        assertNull(seq.getNext());
        // all events need to be consumed
        assertEquals(events.size(), numEvent);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}