Examples with OperatorSnapshotFutures org.apache.flink.streaming.api.operators.OperatorSnapshotFutures used on opensource projects

Example 6 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class AbstractStreamOperatorTestHarness method snapshotWithLocalState.

/**
 * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions,
 * org.apache.flink.runtime.state.CheckpointStreamFactory)}.
 */
public OperatorSnapshotFinalizer snapshotWithLocalState(long checkpointId, long timestamp, SnapshotType checkpointType) throws Exception {
    CheckpointStorageLocationReference locationReference = CheckpointStorageLocationReference.getDefault();
    OperatorSnapshotFutures operatorStateResult = operator.snapshotState(checkpointId, timestamp, new CheckpointOptions(checkpointType, locationReference), checkpointStorageAccess.resolveCheckpointStorageLocation(checkpointId, locationReference));
    return new OperatorSnapshotFinalizer(operatorStateResult);
}

Example 7 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class StreamTaskTest method testRestorePerformedOnlyOnce.

@Test
public void testRestorePerformedOnlyOnce() throws Exception {
    // given: the operator with empty snapshot result (all state handles are null)
    OneInputStreamOperator<String, String> statelessOperator = streamOperatorWithSnapshot(new OperatorSnapshotFutures());
    DummyEnvironment dummyEnvironment = new DummyEnvironment();
    // when: Invoke the restore explicitly before launching the task.
    RunningTask<MockStreamTask> task = runTask(() -> {
        MockStreamTask mockStreamTask = createMockStreamTask(dummyEnvironment, operatorChain(statelessOperator));
        mockStreamTask.restore();
        return mockStreamTask;
    });
    waitTaskIsRunning(task.streamTask, task.invocationFuture);
    task.streamTask.cancel();
    // then: 'restore' was called only once.
    assertThat(task.streamTask.restoreInvocationCount, is(1));
}

Example 8 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class StreamTaskTest method testDecliningCheckpointStreamOperator.

@Test
public void testDecliningCheckpointStreamOperator() throws Exception {
    DummyEnvironment dummyEnvironment = new DummyEnvironment();
    // mock the returned snapshots
    OperatorSnapshotFutures operatorSnapshotResult1 = mock(OperatorSnapshotFutures.class);
    OperatorSnapshotFutures operatorSnapshotResult2 = mock(OperatorSnapshotFutures.class);
    final Exception testException = new ExpectedTestException();
    RunningTask<MockStreamTask> task = runTask(() -> createMockStreamTask(dummyEnvironment, operatorChain(streamOperatorWithSnapshotException(testException), streamOperatorWithSnapshot(operatorSnapshotResult1), streamOperatorWithSnapshot(operatorSnapshotResult2))));
    MockStreamTask streamTask = task.streamTask;
    waitTaskIsRunning(streamTask, task.invocationFuture);
    streamTask.triggerCheckpointAsync(new CheckpointMetaData(42L, 1L), CheckpointOptions.forCheckpointWithDefaultLocation());
    try {
        task.waitForTaskCompletion(false);
    } catch (Exception ex) {
        if (!ExceptionUtils.findThrowable(ex, ExpectedTestException.class).isPresent()) {
            throw ex;
        }
    }
    verify(operatorSnapshotResult1).cancel();
    verify(operatorSnapshotResult2).cancel();
}

Example 9 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class StreamTaskTest method testAsyncCheckpointingConcurrentCloseBeforeAcknowledge.

/**
 * FLINK-5667
 *
 * <p>Tests that a concurrent cancel operation discards the state handles of a not yet
 * acknowledged checkpoint and prevents sending an acknowledge message to the
 * CheckpointCoordinator. The situation can only happen if the cancel call is executed before
 * Environment.acknowledgeCheckpoint().
 */
@Test
public void testAsyncCheckpointingConcurrentCloseBeforeAcknowledge() throws Exception {
    final TestingKeyedStateHandle managedKeyedStateHandle = new TestingKeyedStateHandle();
    final TestingKeyedStateHandle rawKeyedStateHandle = new TestingKeyedStateHandle();
    final TestingOperatorStateHandle managedOperatorStateHandle = new TestingOperatorStateHandle();
    final TestingOperatorStateHandle rawOperatorStateHandle = new TestingOperatorStateHandle();
    final BlockingRunnableFuture<SnapshotResult<KeyedStateHandle>> rawKeyedStateHandleFuture = new BlockingRunnableFuture<>(2, SnapshotResult.of(rawKeyedStateHandle));
    OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(DoneFuture.of(SnapshotResult.of(managedKeyedStateHandle)), rawKeyedStateHandleFuture, DoneFuture.of(SnapshotResult.of(managedOperatorStateHandle)), DoneFuture.of(SnapshotResult.of(rawOperatorStateHandle)), DoneFuture.of(SnapshotResult.empty()), DoneFuture.of(SnapshotResult.empty()));
    final OneInputStreamOperator<String, String> streamOperator = streamOperatorWithSnapshot(operatorSnapshotResult);
    final AcknowledgeDummyEnvironment mockEnvironment = new AcknowledgeDummyEnvironment();
    RunningTask<MockStreamTask> task = runTask(() -> createMockStreamTask(mockEnvironment, operatorChain(streamOperator)));
    waitTaskIsRunning(task.streamTask, task.invocationFuture);
    final long checkpointId = 42L;
    task.streamTask.triggerCheckpointAsync(new CheckpointMetaData(checkpointId, 1L), CheckpointOptions.forCheckpointWithDefaultLocation());
    rawKeyedStateHandleFuture.awaitRun();
    task.streamTask.cancel();
    final FutureUtils.ConjunctFuture<Void> discardFuture = FutureUtils.waitForAll(asList(managedKeyedStateHandle.getDiscardFuture(), rawKeyedStateHandle.getDiscardFuture(), managedOperatorStateHandle.getDiscardFuture(), rawOperatorStateHandle.getDiscardFuture()));
    // make sure that all state handles have been discarded
    discardFuture.get();
    try {
        mockEnvironment.getAcknowledgeCheckpointFuture().get(10L, TimeUnit.MILLISECONDS);
        fail("The checkpoint should not get acknowledged.");
    } catch (TimeoutException expected) {
    // future should not be completed
    }
    task.waitForTaskCompletion(true);
}

Example 10 with OperatorSnapshotFutures

use of org.apache.flink.streaming.api.operators.OperatorSnapshotFutures in project flink by apache.

the class StreamTaskTest method testFailingAsyncCheckpointRunnable.

/**
 * Tests that in case of a failing AsyncCheckpointRunnable all operator snapshot results are
 * cancelled and all non partitioned state handles are discarded.
 */
@Test
public void testFailingAsyncCheckpointRunnable() throws Exception {
    // mock the new state operator snapshots
    OperatorSnapshotFutures operatorSnapshotResult1 = mock(OperatorSnapshotFutures.class);
    OperatorSnapshotFutures operatorSnapshotResult2 = mock(OperatorSnapshotFutures.class);
    OperatorSnapshotFutures operatorSnapshotResult3 = mock(OperatorSnapshotFutures.class);
    RunnableFuture<SnapshotResult<OperatorStateHandle>> failingFuture = mock(RunnableFuture.class);
    when(failingFuture.get()).thenThrow(new ExecutionException(new Exception("Test exception")));
    when(operatorSnapshotResult3.getOperatorStateRawFuture()).thenReturn(failingFuture);
    try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder().build()) {
        RunningTask<MockStreamTask> task = runTask(() -> createMockStreamTask(mockEnvironment, operatorChain(streamOperatorWithSnapshot(operatorSnapshotResult1), streamOperatorWithSnapshot(operatorSnapshotResult2), streamOperatorWithSnapshot(operatorSnapshotResult3))));
        MockStreamTask streamTask = task.streamTask;
        waitTaskIsRunning(streamTask, task.invocationFuture);
        mockEnvironment.setExpectedExternalFailureCause(Throwable.class);
        streamTask.triggerCheckpointAsync(new CheckpointMetaData(42L, 1L), CheckpointOptions.forCheckpointWithDefaultLocation()).get();
        // wait for the completion of the async task
        ExecutorService executor = streamTask.getAsyncOperationsThreadPool();
        executor.shutdown();
        if (!executor.awaitTermination(10000L, TimeUnit.MILLISECONDS)) {
            fail("Executor did not shut down within the given timeout. This indicates that the " + "checkpointing did not resume.");
        }
        assertTrue(mockEnvironment.getActualExternalFailureCause().isPresent());
        verify(operatorSnapshotResult1).cancel();
        verify(operatorSnapshotResult2).cancel();
        verify(operatorSnapshotResult3).cancel();
        streamTask.finishInput();
        task.waitForTaskCompletion(false);
    }
}