Examples with TestCompletedCheckpointStorageLocation org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation used on opensource projects

Example 16 with TestCompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreTest method testAddCheckpointWithFailedRemove.

/**
 * Tests that the checkpoint does not exist in the store when we fail to add it into the store
 * (i.e., there exists an exception thrown by the method).
 */
@Test
public void testAddCheckpointWithFailedRemove() throws Exception {
    final int numCheckpointsToRetain = 1;
    final Configuration configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());
    final CuratorFrameworkWithUnhandledErrorListener curatorFrameworkWrapper = ZooKeeperUtils.startCuratorFramework(configuration, NoOpFatalErrorHandler.INSTANCE);
    final CompletedCheckpointStore store = createZooKeeperCheckpointStore(curatorFrameworkWrapper.asCuratorFramework());
    CountDownLatch discardAttempted = new CountDownLatch(1);
    for (long i = 0; i < numCheckpointsToRetain + 1; ++i) {
        CompletedCheckpoint checkpointToAdd = new CompletedCheckpoint(new JobID(), i, i, i, Collections.emptyMap(), Collections.emptyList(), CheckpointProperties.forCheckpoint(NEVER_RETAIN_AFTER_TERMINATION), new TestCompletedCheckpointStorageLocation());
        // shouldn't fail despite the exception
        store.addCheckpointAndSubsumeOldestOne(checkpointToAdd, new CheckpointsCleaner(), () -> {
            discardAttempted.countDown();
            throw new RuntimeException();
        });
    }
    discardAttempted.await();
}

Example 17 with TestCompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation in project flink by apache.

the class CheckpointMetadataLoadingTest method createSavepointWithOperatorState.

// ------------------------------------------------------------------------
// setup utils
// ------------------------------------------------------------------------
private static CompletedCheckpointStorageLocation createSavepointWithOperatorState(final long checkpointId, final OperatorState state) throws IOException {
    final CheckpointMetadata savepoint = new CheckpointMetadata(checkpointId, Collections.singletonList(state), Collections.emptyList());
    final StreamStateHandle serializedMetadata;
    try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
        Checkpoints.storeCheckpointMetadata(savepoint, os);
        serializedMetadata = new ByteStreamStateHandle("checkpoint", os.toByteArray());
    }
    return new TestCompletedCheckpointStorageLocation(serializedMetadata, "dummy/pointer");
}

Example 18 with TestCompletedCheckpointStorageLocation

use of org.apache.flink.runtime.state.testutils.TestCompletedCheckpointStorageLocation in project flink by apache.

the class CheckpointCoordinatorMasterHooksTest method checkUnMatchedStateOnRestore.

@Test
public void checkUnMatchedStateOnRestore() throws Exception {
    final String id1 = "id1";
    final String id2 = "id2";
    final String state1 = "the-test-string-state";
    final byte[] state1serialized = new StringSerializer().serialize(state1);
    final long state2 = 987654321L;
    final byte[] state2serialized = new LongSerializer().serialize(state2);
    final List<MasterState> masterHookStates = Arrays.asList(new MasterState(id1, state1serialized, StringSerializer.VERSION), new MasterState(id2, state2serialized, LongSerializer.VERSION));
    final MasterTriggerRestoreHook<String> statefulHook = mockGeneric(MasterTriggerRestoreHook.class);
    when(statefulHook.getIdentifier()).thenReturn(id1);
    when(statefulHook.createCheckpointDataSerializer()).thenReturn(new StringSerializer());
    when(statefulHook.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class))).thenThrow(new Exception("not expected"));
    final MasterTriggerRestoreHook<Void> statelessHook = mockGeneric(MasterTriggerRestoreHook.class);
    when(statelessHook.getIdentifier()).thenReturn("some-id");
    final JobID jid = new JobID();
    final long checkpointId = 44L;
    final CompletedCheckpoint checkpoint = new CompletedCheckpoint(jid, checkpointId, 123L, 125L, Collections.<OperatorID, OperatorState>emptyMap(), masterHookStates, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), new TestCompletedCheckpointStorageLocation());
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(new JobVertexID()).build();
    CheckpointCoordinator cc = instantiateCheckpointCoordinator(graph);
    cc.addMasterHook(statefulHook);
    cc.addMasterHook(statelessHook);
    cc.getCheckpointStore().addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    // since we have unmatched state, this should fail
    try {
        cc.restoreLatestCheckpointedStateToAll(Collections.emptySet(), false);
        fail("exception expected");
    } catch (IllegalStateException ignored) {
    }
    // permitting unmatched state should succeed
    cc.restoreLatestCheckpointedStateToAll(Collections.emptySet(), true);
    verify(statefulHook, times(1)).restoreCheckpoint(eq(checkpointId), eq(state1));
    verify(statelessHook, times(1)).restoreCheckpoint(eq(checkpointId), isNull(Void.class));
}