Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 41 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method validateAlignmentBuffered.

private static void validateAlignmentBuffered(long actualBytesBuffered, BufferOrEvent... sequence) {
    long expectedBuffered = 0;
    for (BufferOrEvent boe : sequence) {
        if (boe.isBuffer()) {
            expectedBuffered += BufferSpiller.HEADER_SIZE + boe.getBuffer().getSize();
        }
    }
    assertEquals("Wrong alignment buffered bytes", actualBytesBuffered, expectedBuffered);
}

Example 42 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testMultiChannelNoBarriers.

/**
	 * Validates that the buffer behaves correctly if no checkpoint barriers come,
	 * for an input with multiple input channels.
	 */
@Test
public void testMultiChannelNoBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(2), createBuffer(2), createBuffer(0), createBuffer(1), createBuffer(0), createEndOfPartition(0), createBuffer(3), createBuffer(1), createEndOfPartition(3), createBuffer(1), createEndOfPartition(1), createBuffer(2), createEndOfPartition(2) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 4, Arrays.asList(sequence));
        BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
        for (BufferOrEvent boe : sequence) {
            assertEquals(boe, buffer.getNextNonBlocked());
        }
        assertEquals(0L, buffer.getAlignmentDurationNanos());
        assertNull(buffer.getNextNonBlocked());
        assertNull(buffer.getNextNonBlocked());
        buffer.cleanup();
        checkNoTempFilesRemain();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 43 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testMultiChannelSkippingCheckpoints.

/**
	 * Validates that the buffer skips over the current checkpoint if it
	 * receives a barrier from a later checkpoint on a non-blocked input.
	 */
@Test
public void testMultiChannelSkippingCheckpoints() {
    try {
        BufferOrEvent[] sequence = { // checkpoint 1 - with blocked data
        createBuffer(0), createBuffer(2), createBuffer(0), createBarrier(1, 1), createBarrier(1, 2), createBuffer(2), createBuffer(1), createBuffer(0), createBarrier(1, 0), createBuffer(1), createBuffer(0), // checkpoint 2 will not complete: pre-mature barrier from checkpoint 3
        createBarrier(2, 1), createBuffer(1), createBuffer(2), createBarrier(2, 0), createBuffer(2), createBuffer(0), createBarrier(3, 2), createBuffer(2), createBuffer(1), createEndOfPartition(1), createBuffer(2), createEndOfPartition(2), createBuffer(0), createEndOfPartition(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
        BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
        StatefulTask toNotify = mock(StatefulTask.class);
        buffer.registerCheckpointEventHandler(toNotify);
        long startTs;
        // initial data
        check(sequence[0], buffer.getNextNonBlocked());
        check(sequence[1], buffer.getNextNonBlocked());
        check(sequence[2], buffer.getNextNonBlocked());
        // align checkpoint 1
        startTs = System.nanoTime();
        check(sequence[7], buffer.getNextNonBlocked());
        assertEquals(1L, buffer.getCurrentCheckpointId());
        // checkpoint done - replay buffered
        check(sequence[5], buffer.getNextNonBlocked());
        validateAlignmentTime(startTs, buffer);
        verify(toNotify).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
        check(sequence[6], buffer.getNextNonBlocked());
        check(sequence[9], buffer.getNextNonBlocked());
        check(sequence[10], buffer.getNextNonBlocked());
        // alignment of checkpoint 2
        startTs = System.nanoTime();
        check(sequence[13], buffer.getNextNonBlocked());
        check(sequence[15], buffer.getNextNonBlocked());
        // checkpoint 2 aborted, checkpoint 3 started
        check(sequence[12], buffer.getNextNonBlocked());
        assertEquals(3L, buffer.getCurrentCheckpointId());
        validateAlignmentTime(startTs, buffer);
        verify(toNotify).abortCheckpointOnBarrier(eq(2L), any(CheckpointDeclineSubsumedException.class));
        check(sequence[16], buffer.getNextNonBlocked());
        // checkpoint 3 alignment in progress
        check(sequence[19], buffer.getNextNonBlocked());
        // checkpoint 3 aborted (end of partition)
        check(sequence[20], buffer.getNextNonBlocked());
        verify(toNotify).abortCheckpointOnBarrier(eq(3L), any(CheckpointDeclineSubsumedException.class));
        // replay buffered data from checkpoint 3
        check(sequence[18], buffer.getNextNonBlocked());
        // all the remaining messages
        check(sequence[21], buffer.getNextNonBlocked());
        check(sequence[22], buffer.getNextNonBlocked());
        check(sequence[23], buffer.getNextNonBlocked());
        check(sequence[24], buffer.getNextNonBlocked());
        assertNull(buffer.getNextNonBlocked());
        assertNull(buffer.getNextNonBlocked());
        buffer.cleanup();
        checkNoTempFilesRemain();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 44 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testSingleChannelNoBarriers.

// ------------------------------------------------------------------------
//  Tests
// ------------------------------------------------------------------------
/**
	 * Validates that the buffer behaves correctly if no checkpoint barriers come,
	 * for a single input channel.
	 */
@Test
public void testSingleChannelNoBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(0), createBuffer(0), createBuffer(0), createEndOfPartition(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
        BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
        for (BufferOrEvent boe : sequence) {
            assertEquals(boe, buffer.getNextNonBlocked());
        }
        assertEquals(0L, buffer.getAlignmentDurationNanos());
        assertNull(buffer.getNextNonBlocked());
        assertNull(buffer.getNextNonBlocked());
        buffer.cleanup();
        checkNoTempFilesRemain();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 45 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testAbortViaQueuedBarriers.

@Test
public void testAbortViaQueuedBarriers() throws Exception {
    BufferOrEvent[] sequence = { /* 0 */
    createBuffer(1), /* 1 */
    createBarrier(1, 1), createBarrier(1, 2), /* 3 */
    createBuffer(2), createBuffer(0), createBuffer(1), /* 6 */
    createCancellationBarrier(2, 2), /* 7 */
    createBarrier(2, 1), /* 8 */
    createBuffer(0), createBuffer(1), createBuffer(2), /* 11 */
    createBarrier(1, 0), /* 12 */
    createBuffer(2), createBuffer(1), createBuffer(0), /* 15 */
    createBarrier(2, 0), /* 16 */
    createBuffer(0), createBuffer(1), createBuffer(2) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
    StatefulTask toNotify = mock(StatefulTask.class);
    buffer.registerCheckpointEventHandler(toNotify);
    long startTs;
    check(sequence[0], buffer.getNextNonBlocked());
    // starting first checkpoint
    startTs = System.nanoTime();
    check(sequence[4], buffer.getNextNonBlocked());
    check(sequence[8], buffer.getNextNonBlocked());
    // finished first checkpoint
    check(sequence[3], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    validateAlignmentTime(startTs, buffer);
    check(sequence[5], buffer.getNextNonBlocked());
    // re-read the queued cancellation barriers
    check(sequence[9], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(2L), any(CheckpointDeclineOnCancellationBarrierException.class));
    assertEquals(0L, buffer.getAlignmentDurationNanos());
    check(sequence[10], buffer.getNextNonBlocked());
    check(sequence[12], buffer.getNextNonBlocked());
    check(sequence[13], buffer.getNextNonBlocked());
    check(sequence[14], buffer.getNextNonBlocked());
    check(sequence[16], buffer.getNextNonBlocked());
    check(sequence[17], buffer.getNextNonBlocked());
    check(sequence[18], buffer.getNextNonBlocked());
    // no further alignment should have happened
    assertEquals(0L, buffer.getAlignmentDurationNanos());
    // no further checkpoint (abort) notifications
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(any(CheckpointMetaData.class), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    verify(toNotify, times(1)).abortCheckpointOnBarrier(anyLong(), any(CheckpointDeclineOnCancellationBarrierException.class));
    // all done
    assertNull(buffer.getNextNonBlocked());
    assertNull(buffer.getNextNonBlocked());
    buffer.cleanup();
    checkNoTempFilesRemain();
}