Examples with InputChannelInfo org.apache.flink.runtime.checkpoint.channel.InputChannelInfo used on opensource projects

Example 26 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class UnalignedCheckpointsCancellationTest method test.

@Test
public void test() throws Exception {
    TestInvokable invokable = new TestInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(numChannels).setChannelFactory(InputChannelBuilder::buildLocalChannel).build();
    SingleCheckpointBarrierHandler unaligner = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), true, inputGate);
    for (RuntimeEvent e : events) {
        if (e instanceof CancelCheckpointMarker) {
            unaligner.processCancellationBarrier((CancelCheckpointMarker) e, new InputChannelInfo(0, channel));
        } else if (e instanceof CheckpointBarrier) {
            unaligner.processBarrier((CheckpointBarrier) e, new InputChannelInfo(0, channel), false);
        } else {
            throw new IllegalArgumentException("unexpected event type: " + e);
        }
    }
    assertEquals("expectAbortCheckpoint", expectAbortCheckpoint, invokable.checkpointAborted);
    assertEquals("expectTriggerCheckpoint", expectTriggerCheckpoint, invokable.checkpointTriggered);
}

Example 27 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class UnalignedCheckpointsTest method testProcessCancellationBarrierBeforeProcessAndReceiveBarrier.

@Test
public void testProcessCancellationBarrierBeforeProcessAndReceiveBarrier() throws Exception {
    final ValidatingCheckpointInvokable invokable = new ValidatingCheckpointInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildLocalChannel).build();
    final SingleCheckpointBarrierHandler handler = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), true, inputGate);
    handler.processCancellationBarrier(new CancelCheckpointMarker(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0));
    verifyTriggeredCheckpoint(handler, invokable, DEFAULT_CHECKPOINT_ID);
    // it would not trigger checkpoint since the respective cancellation barrier already
    // happened before
    handler.processBarrier(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0), false);
    verifyTriggeredCheckpoint(handler, invokable, DEFAULT_CHECKPOINT_ID);
}

Example 28 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class UnalignedCheckpointsTest method testProcessCancellationBarrierAfterProcessBarrier.

/**
 * Tests {@link
 * SingleCheckpointBarrierHandler#processCancellationBarrier(CancelCheckpointMarker,
 * InputChannelInfo)} abort the current pending checkpoint triggered by {@link
 * CheckpointBarrierHandler#processBarrier(CheckpointBarrier, InputChannelInfo)}.
 */
@Test
public void testProcessCancellationBarrierAfterProcessBarrier() throws Exception {
    final ValidatingCheckpointInvokable invokable = new ValidatingCheckpointInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).setChannelFactory(InputChannelBuilder::buildLocalChannel).build();
    final SingleCheckpointBarrierHandler handler = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), true, inputGate);
    // should trigger respective checkpoint
    handler.processBarrier(buildCheckpointBarrier(DEFAULT_CHECKPOINT_ID), new InputChannelInfo(0, 0), false);
    assertTrue(handler.isCheckpointPending());
    assertEquals(DEFAULT_CHECKPOINT_ID, handler.getLatestCheckpointId());
    testProcessCancellationBarrier(handler, invokable);
}

Example 29 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class RecordWriterTest method parseBuffer.

// ---------------------------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------------------------
static BufferOrEvent parseBuffer(Buffer buffer, int targetChannel) throws IOException {
    if (buffer.isBuffer()) {
        return new BufferOrEvent(buffer, new InputChannelInfo(0, targetChannel));
    } else {
        // is event:
        AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
        // the buffer is not needed anymore
        buffer.recycleBuffer();
        return new BufferOrEvent(event, new InputChannelInfo(0, targetChannel));
    }
}

Example 30 with InputChannelInfo

use of org.apache.flink.runtime.checkpoint.channel.InputChannelInfo in project flink by apache.

the class CheckpointCoordinatorFailureTest method testFailingCompletedCheckpointStoreAdd.

/**
 * Tests that a failure while storing a completed checkpoint in the completed checkpoint store
 * will properly fail the originating pending checkpoint and clean upt the completed checkpoint.
 */
@Test
public void testFailingCompletedCheckpointStoreAdd() throws Exception {
    JobVertexID jobVertexId = new JobVertexID();
    final ManuallyTriggeredScheduledExecutor manuallyTriggeredScheduledExecutor = new ManuallyTriggeredScheduledExecutor();
    ExecutionGraph testGraph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexId).build();
    ExecutionVertex vertex = testGraph.getJobVertex(jobVertexId).getTaskVertices()[0];
    // set up the coordinator and validate the initial state
    CheckpointCoordinator coord = new CheckpointCoordinatorBuilder().setExecutionGraph(testGraph).setCompletedCheckpointStore(new FailingCompletedCheckpointStore(new Exception("The failing completed checkpoint store failed again... :-("))).setTimer(manuallyTriggeredScheduledExecutor).build();
    coord.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    assertEquals(1, coord.getNumberOfPendingCheckpoints());
    PendingCheckpoint pendingCheckpoint = coord.getPendingCheckpoints().values().iterator().next();
    assertFalse(pendingCheckpoint.isDisposed());
    final long checkpointId = coord.getPendingCheckpoints().keySet().iterator().next();
    KeyedStateHandle managedKeyedHandle = mock(KeyedStateHandle.class);
    KeyedStateHandle rawKeyedHandle = mock(KeyedStateHandle.class);
    OperatorStateHandle managedOpHandle = mock(OperatorStreamStateHandle.class);
    OperatorStateHandle rawOpHandle = mock(OperatorStreamStateHandle.class);
    InputChannelStateHandle inputChannelStateHandle = new InputChannelStateHandle(new InputChannelInfo(0, 1), mock(StreamStateHandle.class), Collections.singletonList(1L));
    ResultSubpartitionStateHandle resultSubpartitionStateHandle = new ResultSubpartitionStateHandle(new ResultSubpartitionInfo(0, 1), mock(StreamStateHandle.class), Collections.singletonList(1L));
    final OperatorSubtaskState operatorSubtaskState = spy(OperatorSubtaskState.builder().setManagedOperatorState(managedOpHandle).setRawOperatorState(rawOpHandle).setManagedKeyedState(managedKeyedHandle).setRawKeyedState(rawKeyedHandle).setInputChannelState(StateObjectCollection.singleton(inputChannelStateHandle)).setResultSubpartitionState(StateObjectCollection.singleton(resultSubpartitionStateHandle)).build());
    TaskStateSnapshot subtaskState = spy(new TaskStateSnapshot());
    subtaskState.putSubtaskStateByOperatorID(new OperatorID(), operatorSubtaskState);
    when(subtaskState.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(vertex.getJobvertexId()))).thenReturn(operatorSubtaskState);
    AcknowledgeCheckpoint acknowledgeMessage = new AcknowledgeCheckpoint(testGraph.getJobID(), vertex.getCurrentExecutionAttempt().getAttemptId(), checkpointId, new CheckpointMetrics(), subtaskState);
    try {
        coord.receiveAcknowledgeMessage(acknowledgeMessage, "Unknown location");
        fail("Expected a checkpoint exception because the completed checkpoint store could not " + "store the completed checkpoint.");
    } catch (CheckpointException e) {
    // ignore because we expected this exception
    }
    // make sure that the pending checkpoint has been discarded after we could not complete it
    assertTrue(pendingCheckpoint.isDisposed());
    // make sure that the subtask state has been discarded after we could not complete it.
    verify(operatorSubtaskState).discardState();
    verify(operatorSubtaskState.getManagedOperatorState().iterator().next()).discardState();
    verify(operatorSubtaskState.getRawOperatorState().iterator().next()).discardState();
    verify(operatorSubtaskState.getManagedKeyedState().iterator().next()).discardState();
    verify(operatorSubtaskState.getRawKeyedState().iterator().next()).discardState();
    verify(operatorSubtaskState.getInputChannelState().iterator().next().getDelegate()).discardState();
    verify(operatorSubtaskState.getResultSubpartitionState().iterator().next().getDelegate()).discardState();
}