Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 36 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferAlignmentLimitTest method testAlignmentLimitWithQueuedAlignments.

/**
	 * This tests the following case:
	 *   - an alignment starts
	 *   - barriers from a second checkpoint queue before the first completes
	 *   - together they are larger than the threshold
	 *   - after the first checkpoint (with second checkpoint data queued) aborts, the second completes 
	 */
@Test
public void testAlignmentLimitWithQueuedAlignments() throws Exception {
    BufferOrEvent[] sequence = { /*  0 */
    createBuffer(1, 100), createBuffer(2, 70), /*  2 */
    createBarrier(3, 2), /*  3 */
    createBuffer(1, 100), createBuffer(2, 100), /*  5 */
    createBarrier(3, 0), /*  6 */
    createBuffer(0, 100), createBuffer(1, 100), /*  8 */
    createBarrier(4, 0), /*  9 */
    createBuffer(0, 100), createBuffer(0, 120), createBuffer(1, 100), /* 12 */
    createBuffer(2, 100), /* 13 */
    createBarrier(3, 1), /* 14 */
    createBarrier(4, 1), /* 15 */
    createBuffer(0, 100), createBuffer(1, 100), createBuffer(2, 100), /* 18 */
    createBarrier(4, 2), /* 19 */
    createBuffer(0, 100), createBuffer(1, 100), createBuffer(2, 100) };
    // the barrier buffer has a limit that only 1000 bytes may be spilled in alignment
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER, 500);
    StatefulTask toNotify = mock(StatefulTask.class);
    buffer.registerCheckpointEventHandler(toNotify);
    // validating the sequence of buffers
    long startTs;
    check(sequence[0], buffer.getNextNonBlocked());
    check(sequence[1], buffer.getNextNonBlocked());
    // start of checkpoint
    startTs = System.nanoTime();
    check(sequence[3], buffer.getNextNonBlocked());
    check(sequence[7], buffer.getNextNonBlocked());
    // next checkpoint also in progress
    check(sequence[11], buffer.getNextNonBlocked());
    // checkpoint alignment aborted due to too much data
    check(sequence[4], buffer.getNextNonBlocked());
    validateAlignmentTime(startTs, buffer);
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(3L), any(AlignmentLimitExceededException.class));
    // replay buffered data - in the middle, the alignment for checkpoint 4 starts
    check(sequence[6], buffer.getNextNonBlocked());
    startTs = System.nanoTime();
    check(sequence[12], buffer.getNextNonBlocked());
    // only checkpoint 4 is pending now - the last checkpoint 3 barrier will not trigger success 
    check(sequence[17], buffer.getNextNonBlocked());
    // checkpoint 4 completed - check and validate buffered replay
    check(sequence[9], buffer.getNextNonBlocked());
    validateAlignmentTime(startTs, buffer);
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(4L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    check(sequence[10], buffer.getNextNonBlocked());
    check(sequence[15], buffer.getNextNonBlocked());
    check(sequence[16], buffer.getNextNonBlocked());
    // trailing data
    check(sequence[19], buffer.getNextNonBlocked());
    check(sequence[20], buffer.getNextNonBlocked());
    check(sequence[21], buffer.getNextNonBlocked());
    // only checkpoint 4 was successfully completed, not checkpoint 3
    verify(toNotify, times(0)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(3L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    assertNull(buffer.getNextNonBlocked());
    assertNull(buffer.getNextNonBlocked());
    buffer.cleanup();
    checkNoTempFilesRemain();
}

Example 37 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferAlignmentLimitTest method testBreakCheckpointAtAlignmentLimit.

// ------------------------------------------------------------------------
//  Tests
// ------------------------------------------------------------------------
/**
	 * This tests that a single alignment that buffers too much data cancels
	 */
@Test
public void testBreakCheckpointAtAlignmentLimit() throws Exception {
    BufferOrEvent[] sequence = { /*  0 */
    createBuffer(1, 100), createBuffer(2, 70), /*  2 */
    createBuffer(0, 42), createBuffer(2, 111), /*  4 */
    createBarrier(7, 1), /*  5 */
    createBuffer(1, 100), createBuffer(2, 200), createBuffer(1, 300), createBuffer(0, 50), /*  9 */
    createBarrier(7, 0), /* 10 */
    createBuffer(2, 100), createBuffer(0, 100), createBuffer(1, 200), createBuffer(0, 200), /* 14 */
    createBuffer(0, 101), /* 15 */
    createBuffer(0, 100), createBuffer(1, 100), createBuffer(2, 100), /* 18 */
    createBarrier(7, 2), /* 19 */
    createBuffer(0, 100), createBuffer(1, 100), createBuffer(2, 100) };
    // the barrier buffer has a limit that only 1000 bytes may be spilled in alignment
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER, 1000);
    StatefulTask toNotify = mock(StatefulTask.class);
    buffer.registerCheckpointEventHandler(toNotify);
    // validating the sequence of buffers
    check(sequence[0], buffer.getNextNonBlocked());
    check(sequence[1], buffer.getNextNonBlocked());
    check(sequence[2], buffer.getNextNonBlocked());
    check(sequence[3], buffer.getNextNonBlocked());
    // start of checkpoint
    long startTs = System.nanoTime();
    check(sequence[6], buffer.getNextNonBlocked());
    check(sequence[8], buffer.getNextNonBlocked());
    check(sequence[10], buffer.getNextNonBlocked());
    // trying to pull the next makes the alignment overflow - so buffered buffers are replayed
    check(sequence[5], buffer.getNextNonBlocked());
    validateAlignmentTime(startTs, buffer);
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(7L), any(AlignmentLimitExceededException.class));
    // playing back buffered events
    check(sequence[7], buffer.getNextNonBlocked());
    check(sequence[11], buffer.getNextNonBlocked());
    check(sequence[12], buffer.getNextNonBlocked());
    check(sequence[13], buffer.getNextNonBlocked());
    check(sequence[14], buffer.getNextNonBlocked());
    // the additional data
    check(sequence[15], buffer.getNextNonBlocked());
    check(sequence[16], buffer.getNextNonBlocked());
    check(sequence[17], buffer.getNextNonBlocked());
    check(sequence[19], buffer.getNextNonBlocked());
    check(sequence[20], buffer.getNextNonBlocked());
    check(sequence[21], buffer.getNextNonBlocked());
    // no call for a completed checkpoint must have happened
    verify(toNotify, times(0)).triggerCheckpointOnBarrier(any(CheckpointMetaData.class), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    assertNull(buffer.getNextNonBlocked());
    assertNull(buffer.getNextNonBlocked());
    buffer.cleanup();
    checkNoTempFilesRemain();
}

Example 38 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferAlignmentLimitTest method createBuffer.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
private static BufferOrEvent createBuffer(int channel, int size) {
    byte[] bytes = new byte[size];
    RND.nextBytes(bytes);
    MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
    memory.put(0, bytes);
    Buffer buf = new Buffer(memory, FreeingBufferRecycler.INSTANCE);
    buf.setSize(size);
    // retain an additional time so it does not get disposed after being read by the input gate
    buf.retain();
    return new BufferOrEvent(buf, channel);
}

Example 39 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testIgnoreCancelBarrierIfCheckpointSubsumed.

/**
	 * This tests the where a cancellation barrier is received for a checkpoint already
	 * canceled due to receiving a newer checkpoint barrier.
	 */
@Test
public void testIgnoreCancelBarrierIfCheckpointSubsumed() throws Exception {
    BufferOrEvent[] sequence = { /*  0 */
    createBuffer(2), /*  1 */
    createBarrier(3, 1), createBarrier(3, 0), /*  3 */
    createBuffer(0), createBuffer(1), createBuffer(2), /*  6 */
    createBarrier(5, 2), /*  7 */
    createBuffer(2), createBuffer(1), createBuffer(0), /* 10 */
    createCancellationBarrier(3, 2), /* 11 */
    createBuffer(2), createBuffer(0), createBuffer(1), /* 14 */
    createBarrier(5, 0), createBarrier(5, 1), /* 16 */
    createBuffer(0), createBuffer(1), createBuffer(2) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
    StatefulTask toNotify = mock(StatefulTask.class);
    buffer.registerCheckpointEventHandler(toNotify);
    long startTs;
    // validate the sequence
    check(sequence[0], buffer.getNextNonBlocked());
    // beginning of first checkpoint
    check(sequence[5], buffer.getNextNonBlocked());
    // future barrier aborts checkpoint
    startTs = System.nanoTime();
    check(sequence[3], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(3L), any(CheckpointDeclineSubsumedException.class));
    check(sequence[4], buffer.getNextNonBlocked());
    // alignment of next checkpoint
    check(sequence[8], buffer.getNextNonBlocked());
    check(sequence[9], buffer.getNextNonBlocked());
    check(sequence[12], buffer.getNextNonBlocked());
    check(sequence[13], buffer.getNextNonBlocked());
    // checkpoint finished
    check(sequence[7], buffer.getNextNonBlocked());
    validateAlignmentTime(startTs, buffer);
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    check(sequence[11], buffer.getNextNonBlocked());
    // remaining data
    check(sequence[16], buffer.getNextNonBlocked());
    check(sequence[17], buffer.getNextNonBlocked());
    check(sequence[18], buffer.getNextNonBlocked());
    // all done
    assertNull(buffer.getNextNonBlocked());
    assertNull(buffer.getNextNonBlocked());
    buffer.cleanup();
    checkNoTempFilesRemain();
    // check overall notifications
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(any(CheckpointMetaData.class), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    verify(toNotify, times(1)).abortCheckpointOnBarrier(anyLong(), any(Throwable.class));
}

Example 40 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method createBuffer.

private static BufferOrEvent createBuffer(int channel) {
    final int size = SIZE_COUNTER++;
    byte[] bytes = new byte[size];
    RND.nextBytes(bytes);
    MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
    memory.put(0, bytes);
    Buffer buf = new Buffer(memory, FreeingBufferRecycler.INSTANCE);
    buf.setSize(size);
    // retain an additional time so it does not get disposed after being read by the input gate
    buf.retain();
    return new BufferOrEvent(buf, channel);
}