Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 76 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method testAlignedAfterTimedOut.

@Test
public void testAlignedAfterTimedOut() throws Exception {
    int numChannels = 1;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    long alignmentTimeOut = 100L;
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        Buffer barrier1 = barrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut));
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(barrier1.retainBuffer(), 0, 0);
        assertAnnouncement(gate);
        clock.advanceTime(alignmentTimeOut + 1, TimeUnit.MILLISECONDS);
        assertBarrier(gate);
        assertEquals(1, target.getTriggeredCheckpointCounter());
        Buffer barrier2 = barrier(2, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut));
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(barrier2.retainBuffer(), 1, 0);
        assertAnnouncement(gate);
        assertBarrier(gate);
        assertEquals(2, target.getTriggeredCheckpointCounter());
        assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault()), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut)));
    }
}

Example 77 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnAnnouncementForSecondCheckpoint.

/**
 * This test tries to make sure that the first time out happens after processing {@link
 * EventAnnouncement} but before/during processing the first {@link CheckpointBarrier} of at
 * least second checkpoint.
 */
@Test
public void testTimeoutAlignmentOnAnnouncementForSecondCheckpoint() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build();
    long alignmentTimeout = 100;
    performFirstCheckpoint(numChannels, target, gate, alignmentTimeout);
    assertEquals(1, target.getTriggeredCheckpointCounter());
    Buffer checkpointBarrier = withTimeout(2, alignmentTimeout);
    for (int i = 0; i < numChannels; i++) {
        (getChannel(gate, i)).onBuffer(dataBuffer(), 1, 0);
        (getChannel(gate, i)).onBuffer(checkpointBarrier.retainBuffer(), 2, 0);
    }
    assertEquals(1, target.getTriggeredCheckpointCounter());
    for (int i = 0; i < numChannels; i++) {
        assertAnnouncement(gate);
    }
    assertEquals(1, target.getTriggeredCheckpointCounter());
    clock.advanceTime(alignmentTimeout * 4, TimeUnit.MILLISECONDS);
    // the barrier should overtake the data buffers
    assertBarrier(gate);
    assertEquals(2, target.getTriggeredCheckpointCounter());
}

Example 78 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method testActiveTimeoutAlignmentOnAnnouncement.

@Test
public void testActiveTimeoutAlignmentOnAnnouncement() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        long alignmentTimeout = 10;
        Buffer checkpointBarrier = withTimeout(alignmentTimeout);
        getChannel(gate, 0).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 0).onBuffer(dataBuffer(), 1, 0);
        getChannel(gate, 0).onBuffer(checkpointBarrier.retainBuffer(), 2, 0);
        getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 1).onBuffer(checkpointBarrier.retainBuffer(), 1, 0);
        assertEquals(0, target.getTriggeredCheckpointCounter());
        assertAnnouncement(gate);
        assertAnnouncement(gate);
        // the announcement should time out causing the barriers to overtake the data buffers
        clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
        assertBarrier(gate);
        assertBarrier(gate);
        assertEquals(1, target.getTriggeredCheckpointCounter());
        assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault())));
        // Followed by overtaken buffers
        assertData(gate);
        assertData(gate);
        assertData(gate);
    }
}

Example 79 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentBeforeFirstBarrier.

@Test
public void testTimeoutAlignmentBeforeFirstBarrier() throws Exception {
    // given: Local channels.
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withTestChannels().withMailboxExecutor().build();
    long alignedCheckpointTimeout = 100;
    // when: Aligned checkpoint timeout expired before the first barrier received.
    Buffer checkpointBarrier = withTimeout(1, alignedCheckpointTimeout);
    clock.advanceTime(alignedCheckpointTimeout + 1, TimeUnit.MILLISECONDS);
    ((TestInputChannel) gate.getChannel(0)).read(checkpointBarrier.retainBuffer());
    // then: The UC is triggered as soon as the first barrier is received.
    assertBarrier(gate);
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 80 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AlternatingCheckpointsTest method testNoActiveTimeoutAlignmentAfterAbort.

@Test
public void testNoActiveTimeoutAlignmentAfterAbort() throws Exception {
    int numberOfChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numberOfChannels, getTestBarrierHandlerFactory(target)).withTestChannels().withSyncExecutor().build();
    long alignmentTimeout = 100;
    Buffer checkpointBarrier = withTimeout(alignmentTimeout);
    send(checkpointBarrier, 0, gate);
    send(toBuffer(new CancelCheckpointMarker(1L), true), 0, gate);
    send(toBuffer(new CancelCheckpointMarker(1L), true), 1, gate);
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    assertThat(target.getTriggeredCheckpointOptions().size(), equalTo(0));
}