Examples with StringSerializer org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.StringSerializer used on opensource projects

Example 1 with StringSerializer

use of org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.StringSerializer in project flink by apache.

the class CheckpointCoordinatorMasterHooksTest method testHooksAreCalledOnRestore.

@Test
public void testHooksAreCalledOnRestore() throws Exception {
    final String id1 = "id1";
    final String id2 = "id2";
    final String state1 = "the-test-string-state";
    final byte[] state1serialized = new StringSerializer().serialize(state1);
    final long state2 = 987654321L;
    final byte[] state2serialized = new LongSerializer().serialize(state2);
    final List<MasterState> masterHookStates = Arrays.asList(new MasterState(id1, state1serialized, StringSerializer.VERSION), new MasterState(id2, state2serialized, LongSerializer.VERSION));
    final MasterTriggerRestoreHook<String> statefulHook1 = mockGeneric(MasterTriggerRestoreHook.class);
    when(statefulHook1.getIdentifier()).thenReturn(id1);
    when(statefulHook1.createCheckpointDataSerializer()).thenReturn(new StringSerializer());
    when(statefulHook1.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class))).thenThrow(new Exception("not expected"));
    final MasterTriggerRestoreHook<Long> statefulHook2 = mockGeneric(MasterTriggerRestoreHook.class);
    when(statefulHook2.getIdentifier()).thenReturn(id2);
    when(statefulHook2.createCheckpointDataSerializer()).thenReturn(new LongSerializer());
    when(statefulHook2.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class))).thenThrow(new Exception("not expected"));
    final MasterTriggerRestoreHook<Void> statelessHook = mockGeneric(MasterTriggerRestoreHook.class);
    when(statelessHook.getIdentifier()).thenReturn("some-id");
    final JobID jid = new JobID();
    final long checkpointId = 13L;
    final CompletedCheckpoint checkpoint = new CompletedCheckpoint(jid, checkpointId, 123L, 125L, Collections.<OperatorID, OperatorState>emptyMap(), masterHookStates, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), new TestCompletedCheckpointStorageLocation());
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(new JobVertexID()).build();
    CheckpointCoordinator cc = instantiateCheckpointCoordinator(graph);
    cc.addMasterHook(statefulHook1);
    cc.addMasterHook(statelessHook);
    cc.addMasterHook(statefulHook2);
    cc.getCheckpointStore().addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    cc.restoreLatestCheckpointedStateToAll(Collections.emptySet(), false);
    verify(statefulHook1, times(1)).restoreCheckpoint(eq(checkpointId), eq(state1));
    verify(statefulHook2, times(1)).restoreCheckpoint(eq(checkpointId), eq(state2));
    verify(statelessHook, times(1)).restoreCheckpoint(eq(checkpointId), isNull(Void.class));
}

Example 2 with StringSerializer

use of org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.StringSerializer in project flink by apache.

the class CheckpointCoordinatorMasterHooksTest method checkUnMatchedStateOnRestore.

@Test
public void checkUnMatchedStateOnRestore() throws Exception {
    final String id1 = "id1";
    final String id2 = "id2";
    final String state1 = "the-test-string-state";
    final byte[] state1serialized = new StringSerializer().serialize(state1);
    final long state2 = 987654321L;
    final byte[] state2serialized = new LongSerializer().serialize(state2);
    final List<MasterState> masterHookStates = Arrays.asList(new MasterState(id1, state1serialized, StringSerializer.VERSION), new MasterState(id2, state2serialized, LongSerializer.VERSION));
    final MasterTriggerRestoreHook<String> statefulHook = mockGeneric(MasterTriggerRestoreHook.class);
    when(statefulHook.getIdentifier()).thenReturn(id1);
    when(statefulHook.createCheckpointDataSerializer()).thenReturn(new StringSerializer());
    when(statefulHook.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class))).thenThrow(new Exception("not expected"));
    final MasterTriggerRestoreHook<Void> statelessHook = mockGeneric(MasterTriggerRestoreHook.class);
    when(statelessHook.getIdentifier()).thenReturn("some-id");
    final JobID jid = new JobID();
    final long checkpointId = 44L;
    final CompletedCheckpoint checkpoint = new CompletedCheckpoint(jid, checkpointId, 123L, 125L, Collections.<OperatorID, OperatorState>emptyMap(), masterHookStates, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), new TestCompletedCheckpointStorageLocation());
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(new JobVertexID()).build();
    CheckpointCoordinator cc = instantiateCheckpointCoordinator(graph);
    cc.addMasterHook(statefulHook);
    cc.addMasterHook(statelessHook);
    cc.getCheckpointStore().addCheckpointAndSubsumeOldestOne(checkpoint, new CheckpointsCleaner(), () -> {
    });
    // since we have unmatched state, this should fail
    try {
        cc.restoreLatestCheckpointedStateToAll(Collections.emptySet(), false);
        fail("exception expected");
    } catch (IllegalStateException ignored) {
    }
    // permitting unmatched state should succeed
    cc.restoreLatestCheckpointedStateToAll(Collections.emptySet(), true);
    verify(statefulHook, times(1)).restoreCheckpoint(eq(checkpointId), eq(state1));
    verify(statelessHook, times(1)).restoreCheckpoint(eq(checkpointId), isNull(Void.class));
}

Example 3 with StringSerializer

use of org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTestingUtils.StringSerializer in project flink by apache.

the class CheckpointCoordinatorMasterHooksTest method testHooksAreCalledOnTrigger.

// ------------------------------------------------------------------------
// trigger / restore behavior
// ------------------------------------------------------------------------
@Test
public void testHooksAreCalledOnTrigger() throws Exception {
    final String id1 = "id1";
    final String id2 = "id2";
    final String state1 = "the-test-string-state";
    final byte[] state1serialized = new StringSerializer().serialize(state1);
    final long state2 = 987654321L;
    final byte[] state2serialized = new LongSerializer().serialize(state2);
    final MasterTriggerRestoreHook<String> statefulHook1 = mockGeneric(MasterTriggerRestoreHook.class);
    when(statefulHook1.getIdentifier()).thenReturn(id1);
    when(statefulHook1.createCheckpointDataSerializer()).thenReturn(new StringSerializer());
    when(statefulHook1.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class))).thenReturn(CompletableFuture.completedFuture(state1));
    final MasterTriggerRestoreHook<Long> statefulHook2 = mockGeneric(MasterTriggerRestoreHook.class);
    when(statefulHook2.getIdentifier()).thenReturn(id2);
    when(statefulHook2.createCheckpointDataSerializer()).thenReturn(new LongSerializer());
    when(statefulHook2.triggerCheckpoint(anyLong(), anyLong(), any(Executor.class))).thenReturn(CompletableFuture.completedFuture(state2));
    final MasterTriggerRestoreHook<Void> statelessHook = mockGeneric(MasterTriggerRestoreHook.class);
    when(statelessHook.getIdentifier()).thenReturn("some-id");
    // create the checkpoint coordinator
    JobVertexID jobVertexId = new JobVertexID();
    final ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexId).build();
    final ManuallyTriggeredScheduledExecutor manuallyTriggeredScheduledExecutor = new ManuallyTriggeredScheduledExecutor();
    final CheckpointCoordinator cc = instantiateCheckpointCoordinator(graph, manuallyTriggeredScheduledExecutor);
    cc.addMasterHook(statefulHook1);
    cc.addMasterHook(statelessHook);
    cc.addMasterHook(statefulHook2);
    // trigger a checkpoint
    final CompletableFuture<CompletedCheckpoint> checkpointFuture = cc.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    assertFalse(checkpointFuture.isCompletedExceptionally());
    assertEquals(1, cc.getNumberOfPendingCheckpoints());
    verify(statefulHook1, times(1)).triggerCheckpoint(anyLong(), anyLong(), any(Executor.class));
    verify(statefulHook2, times(1)).triggerCheckpoint(anyLong(), anyLong(), any(Executor.class));
    verify(statelessHook, times(1)).triggerCheckpoint(anyLong(), anyLong(), any(Executor.class));
    ExecutionAttemptID attemptID = graph.getJobVertex(jobVertexId).getTaskVertices()[0].getCurrentExecutionAttempt().getAttemptId();
    final long checkpointId = cc.getPendingCheckpoints().values().iterator().next().getCheckpointId();
    cc.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptID, checkpointId), "Unknown location");
    assertEquals(0, cc.getNumberOfPendingCheckpoints());
    assertEquals(1, cc.getNumberOfRetainedSuccessfulCheckpoints());
    final CompletedCheckpoint chk = cc.getCheckpointStore().getLatestCheckpoint();
    final Collection<MasterState> masterStates = chk.getMasterHookStates();
    assertEquals(2, masterStates.size());
    for (MasterState ms : masterStates) {
        if (ms.name().equals(id1)) {
            assertArrayEquals(state1serialized, ms.bytes());
            assertEquals(StringSerializer.VERSION, ms.version());
        } else if (ms.name().equals(id2)) {
            assertArrayEquals(state2serialized, ms.bytes());
            assertEquals(LongSerializer.VERSION, ms.version());
        } else {
            fail("unrecognized state name: " + ms.name());
        }
    }
}