Examples with ExecutionGraph org.apache.flink.runtime.executiongraph.ExecutionGraph used on opensource projects

Example 31 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class CheckpointCoordinatorTest method testMultipleConcurrentCheckpoints.

@Test
public void testMultipleConcurrentCheckpoints() throws Exception {
    JobVertexID jobVertexID1 = new JobVertexID();
    JobVertexID jobVertexID2 = new JobVertexID();
    JobVertexID jobVertexID3 = new JobVertexID();
    CheckpointCoordinatorTestingUtils.CheckpointRecorderTaskManagerGateway gateway = new CheckpointCoordinatorTestingUtils.CheckpointRecorderTaskManagerGateway();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID1).addJobVertex(jobVertexID2).addJobVertex(jobVertexID3, false).setTaskManagerGateway(gateway).build();
    ExecutionVertex vertex1 = graph.getJobVertex(jobVertexID1).getTaskVertices()[0];
    ExecutionVertex vertex2 = graph.getJobVertex(jobVertexID2).getTaskVertices()[0];
    ExecutionVertex vertex3 = graph.getJobVertex(jobVertexID3).getTaskVertices()[0];
    ExecutionAttemptID attemptID1 = vertex1.getCurrentExecutionAttempt().getAttemptId();
    ExecutionAttemptID attemptID2 = vertex2.getCurrentExecutionAttempt().getAttemptId();
    ExecutionAttemptID attemptID3 = vertex3.getCurrentExecutionAttempt().getAttemptId();
    // set up the coordinator and validate the initial state
    CheckpointCoordinator checkpointCoordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setCheckpointCoordinatorConfiguration(CheckpointCoordinatorConfiguration.builder().setMaxConcurrentCheckpoints(Integer.MAX_VALUE).build()).setCompletedCheckpointStore(new StandaloneCompletedCheckpointStore(2)).setTimer(manuallyTriggeredScheduledExecutor).build();
    assertEquals(0, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    // trigger the first checkpoint. this should succeed
    final CompletableFuture<CompletedCheckpoint> checkpointFuture1 = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    FutureUtils.throwIfCompletedExceptionally(checkpointFuture1);
    assertEquals(1, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    PendingCheckpoint pending1 = checkpointCoordinator.getPendingCheckpoints().values().iterator().next();
    long checkpointId1 = pending1.getCheckpointId();
    // trigger messages should have been sent
    for (ExecutionVertex vertex : Arrays.asList(vertex1, vertex2)) {
        ExecutionAttemptID attemptId = vertex.getCurrentExecutionAttempt().getAttemptId();
        assertEquals(checkpointId1, gateway.getOnlyTriggeredCheckpoint(attemptId).checkpointId);
    }
    // acknowledge one of the three tasks
    checkpointCoordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptID2, checkpointId1), TASK_MANAGER_LOCATION_INFO);
    // start the second checkpoint
    // trigger the first checkpoint. this should succeed
    gateway.resetCount();
    final CompletableFuture<CompletedCheckpoint> checkpointFuture2 = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    FutureUtils.throwIfCompletedExceptionally(checkpointFuture2);
    assertEquals(2, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    PendingCheckpoint pending2;
    {
        Iterator<PendingCheckpoint> all = checkpointCoordinator.getPendingCheckpoints().values().iterator();
        PendingCheckpoint cc1 = all.next();
        PendingCheckpoint cc2 = all.next();
        pending2 = pending1 == cc1 ? cc2 : cc1;
    }
    long checkpointId2 = pending2.getCheckpointId();
    // trigger messages should have been sent
    for (ExecutionVertex vertex : Arrays.asList(vertex1, vertex2)) {
        ExecutionAttemptID attemptId = vertex.getCurrentExecutionAttempt().getAttemptId();
        assertEquals(checkpointId2, gateway.getOnlyTriggeredCheckpoint(attemptId).checkpointId);
    }
    // we acknowledge the remaining two tasks from the first
    // checkpoint and two tasks from the second checkpoint
    checkpointCoordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptID3, checkpointId1), TASK_MANAGER_LOCATION_INFO);
    checkpointCoordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptID1, checkpointId2), TASK_MANAGER_LOCATION_INFO);
    checkpointCoordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptID1, checkpointId1), TASK_MANAGER_LOCATION_INFO);
    checkpointCoordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptID2, checkpointId2), TASK_MANAGER_LOCATION_INFO);
    // now, the first checkpoint should be confirmed
    assertEquals(1, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(1, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    assertTrue(pending1.isDisposed());
    // the first confirm message should be out
    for (ExecutionVertex vertex : Arrays.asList(vertex1, vertex2, vertex3)) {
        ExecutionAttemptID attemptId = vertex.getCurrentExecutionAttempt().getAttemptId();
        assertEquals(checkpointId1, gateway.getOnlyNotifiedCompletedCheckpoint(attemptId).checkpointId);
    }
    // send the last remaining ack for the second checkpoint
    gateway.resetCount();
    checkpointCoordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptID3, checkpointId2), TASK_MANAGER_LOCATION_INFO);
    // now, the second checkpoint should be confirmed
    assertEquals(0, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(2, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    assertTrue(pending2.isDisposed());
    // the second commit message should be out
    for (ExecutionVertex vertex : Arrays.asList(vertex1, vertex2, vertex3)) {
        ExecutionAttemptID attemptId = vertex.getCurrentExecutionAttempt().getAttemptId();
        assertEquals(checkpointId2, gateway.getOnlyNotifiedCompletedCheckpoint(attemptId).checkpointId);
    }
    // validate the committed checkpoints
    List<CompletedCheckpoint> scs = checkpointCoordinator.getSuccessfulCheckpoints();
    CompletedCheckpoint sc1 = scs.get(0);
    assertEquals(checkpointId1, sc1.getCheckpointID());
    assertEquals(graph.getJobID(), sc1.getJobId());
    assertEquals(3, sc1.getOperatorStates().size());
    assertTrue(sc1.getOperatorStates().values().stream().allMatch(this::hasNoSubState));
    CompletedCheckpoint sc2 = scs.get(1);
    assertEquals(checkpointId2, sc2.getCheckpointID());
    assertEquals(graph.getJobID(), sc2.getJobId());
    assertEquals(3, sc2.getOperatorStates().size());
    assertTrue(sc2.getOperatorStates().values().stream().allMatch(this::hasNoSubState));
    checkpointCoordinator.shutdown();
}

Example 32 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class CheckpointCoordinatorTest method testTriggerAndDeclineCheckpointComplex.

/**
 * This test triggers two checkpoints and then sends a decline message from one of the tasks for
 * the first checkpoint. This should discard the first checkpoint while not triggering a new
 * checkpoint because a later checkpoint is already in progress.
 */
@Test
public void testTriggerAndDeclineCheckpointComplex() throws Exception {
    JobVertexID jobVertexID1 = new JobVertexID();
    JobVertexID jobVertexID2 = new JobVertexID();
    CheckpointCoordinatorTestingUtils.CheckpointRecorderTaskManagerGateway gateway = new CheckpointCoordinatorTestingUtils.CheckpointRecorderTaskManagerGateway();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID1).addJobVertex(jobVertexID2).setTaskManagerGateway(gateway).build();
    ExecutionVertex vertex1 = graph.getJobVertex(jobVertexID1).getTaskVertices()[0];
    ExecutionVertex vertex2 = graph.getJobVertex(jobVertexID2).getTaskVertices()[0];
    ExecutionAttemptID attemptID1 = vertex1.getCurrentExecutionAttempt().getAttemptId();
    ExecutionAttemptID attemptID2 = vertex2.getCurrentExecutionAttempt().getAttemptId();
    CheckpointCoordinator checkpointCoordinator = getCheckpointCoordinator(graph);
    assertEquals(0, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    assertEquals(0, manuallyTriggeredScheduledExecutor.getActiveScheduledTasks().size());
    // trigger the first checkpoint. this should succeed
    final CompletableFuture<CompletedCheckpoint> checkpointFuture1 = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    FutureUtils.throwIfCompletedExceptionally(checkpointFuture1);
    // trigger second checkpoint, should also succeed
    final CompletableFuture<CompletedCheckpoint> checkpointFuture2 = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    FutureUtils.throwIfCompletedExceptionally(checkpointFuture2);
    // validate that we have a pending checkpoint
    assertEquals(2, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    assertEquals(2, manuallyTriggeredScheduledExecutor.getActiveScheduledTasks().size());
    Iterator<Map.Entry<Long, PendingCheckpoint>> it = checkpointCoordinator.getPendingCheckpoints().entrySet().iterator();
    long checkpoint1Id = it.next().getKey();
    long checkpoint2Id = it.next().getKey();
    PendingCheckpoint checkpoint1 = checkpointCoordinator.getPendingCheckpoints().get(checkpoint1Id);
    PendingCheckpoint checkpoint2 = checkpointCoordinator.getPendingCheckpoints().get(checkpoint2Id);
    assertNotNull(checkpoint1);
    assertEquals(checkpoint1Id, checkpoint1.getCheckpointId());
    assertEquals(graph.getJobID(), checkpoint1.getJobId());
    assertEquals(2, checkpoint1.getNumberOfNonAcknowledgedTasks());
    assertEquals(0, checkpoint1.getNumberOfAcknowledgedTasks());
    assertEquals(0, checkpoint1.getOperatorStates().size());
    assertFalse(checkpoint1.isDisposed());
    assertFalse(checkpoint1.areTasksFullyAcknowledged());
    assertNotNull(checkpoint2);
    assertEquals(checkpoint2Id, checkpoint2.getCheckpointId());
    assertEquals(graph.getJobID(), checkpoint2.getJobId());
    assertEquals(2, checkpoint2.getNumberOfNonAcknowledgedTasks());
    assertEquals(0, checkpoint2.getNumberOfAcknowledgedTasks());
    assertEquals(0, checkpoint2.getOperatorStates().size());
    assertFalse(checkpoint2.isDisposed());
    assertFalse(checkpoint2.areTasksFullyAcknowledged());
    // check that the vertices received the trigger checkpoint message
    for (ExecutionVertex vertex : Arrays.asList(vertex1, vertex2)) {
        List<CheckpointCoordinatorTestingUtils.TriggeredCheckpoint> triggeredCheckpoints = gateway.getTriggeredCheckpoints(vertex.getCurrentExecutionAttempt().getAttemptId());
        assertEquals(2, triggeredCheckpoints.size());
        assertEquals(checkpoint1Id, triggeredCheckpoints.get(0).checkpointId);
        assertEquals(checkpoint2Id, triggeredCheckpoints.get(1).checkpointId);
    }
    // decline checkpoint from one of the tasks, this should cancel the checkpoint
    checkpointCoordinator.receiveDeclineMessage(new DeclineCheckpoint(graph.getJobID(), attemptID1, checkpoint1Id, new CheckpointException(CHECKPOINT_DECLINED)), TASK_MANAGER_LOCATION_INFO);
    for (ExecutionVertex vertex : Arrays.asList(vertex1, vertex2)) {
        assertEquals(checkpoint1Id, gateway.getOnlyNotifiedAbortedCheckpoint(vertex.getCurrentExecutionAttempt().getAttemptId()).checkpointId);
    }
    assertTrue(checkpoint1.isDisposed());
    // validate that we have only one pending checkpoint left
    assertEquals(1, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    assertEquals(1, manuallyTriggeredScheduledExecutor.getActiveScheduledTasks().size());
    // validate that it is the same second checkpoint from earlier
    long checkpointIdNew = checkpointCoordinator.getPendingCheckpoints().entrySet().iterator().next().getKey();
    PendingCheckpoint checkpointNew = checkpointCoordinator.getPendingCheckpoints().get(checkpointIdNew);
    assertEquals(checkpoint2Id, checkpointIdNew);
    assertNotNull(checkpointNew);
    assertEquals(checkpointIdNew, checkpointNew.getCheckpointId());
    assertEquals(graph.getJobID(), checkpointNew.getJobId());
    assertEquals(2, checkpointNew.getNumberOfNonAcknowledgedTasks());
    assertEquals(0, checkpointNew.getNumberOfAcknowledgedTasks());
    assertEquals(0, checkpointNew.getOperatorStates().size());
    assertFalse(checkpointNew.isDisposed());
    assertFalse(checkpointNew.areTasksFullyAcknowledged());
    assertNotEquals(checkpoint1.getCheckpointId(), checkpointNew.getCheckpointId());
    // decline again, nothing should happen
    // decline from the other task, nothing should happen
    checkpointCoordinator.receiveDeclineMessage(new DeclineCheckpoint(graph.getJobID(), attemptID1, checkpoint1Id, new CheckpointException(CHECKPOINT_DECLINED)), TASK_MANAGER_LOCATION_INFO);
    checkpointCoordinator.receiveDeclineMessage(new DeclineCheckpoint(graph.getJobID(), attemptID2, checkpoint1Id, new CheckpointException(CHECKPOINT_DECLINED)), TASK_MANAGER_LOCATION_INFO);
    assertTrue(checkpoint1.isDisposed());
    // will not notify abort message again
    for (ExecutionVertex vertex : Arrays.asList(vertex1, vertex2)) {
        assertEquals(1, gateway.getNotifiedAbortedCheckpoints(vertex.getCurrentExecutionAttempt().getAttemptId()).size());
    }
    checkpointCoordinator.shutdown();
}

Example 33 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class CheckpointCoordinatorTest method testCheckpointAbortsIfTriggerTasksAreFinished.

@Test
public void testCheckpointAbortsIfTriggerTasksAreFinished() throws Exception {
    JobVertexID jobVertexID1 = new JobVertexID();
    JobVertexID jobVertexID2 = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexID1).addJobVertex(jobVertexID2, false).build();
    CheckpointCoordinator checkpointCoordinator = getCheckpointCoordinator(graph);
    Arrays.stream(graph.getJobVertex(jobVertexID1).getTaskVertices()).forEach(task -> task.getCurrentExecutionAttempt().markFinished());
    // nothing should be happening
    assertEquals(0, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    // trigger the first checkpoint. this should not succeed
    final CompletableFuture<CompletedCheckpoint> checkpointFuture = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    assertTrue(checkpointFuture.isCompletedExceptionally());
    // still, nothing should be happening
    assertEquals(0, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    checkpointCoordinator.shutdown();
}

Example 34 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class CheckpointCoordinatorTest method testSharedStateNotDiscaredOnAbort.

@Test
public void testSharedStateNotDiscaredOnAbort() throws Exception {
    JobVertexID v1 = new JobVertexID(), v2 = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(v1).addJobVertex(v2).build();
    CheckpointCoordinator coordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setTimer(manuallyTriggeredScheduledExecutor).build();
    coordinator.startCheckpointScheduler();
    CompletableFuture<CompletedCheckpoint> cpFuture = coordinator.triggerCheckpoint(true);
    manuallyTriggeredScheduledExecutor.triggerAll();
    cpFuture.getNow(null);
    TestingStreamStateHandle metaState = handle();
    TestingStreamStateHandle privateState = handle();
    TestingStreamStateHandle sharedState = handle();
    ackCheckpoint(1L, coordinator, v1, graph, metaState, privateState, sharedState);
    declineCheckpoint(1L, coordinator, v2, graph);
    assertTrue(privateState.isDisposed());
    assertTrue(metaState.isDisposed());
    assertFalse(sharedState.isDisposed());
    cpFuture = coordinator.triggerCheckpoint(true);
    manuallyTriggeredScheduledExecutor.triggerAll();
    cpFuture.getNow(null);
    ackCheckpoint(2L, coordinator, v1, graph, handle(), handle(), handle());
    ackCheckpoint(2L, coordinator, v2, graph, handle(), handle(), handle());
    cpFuture.get();
    assertTrue(sharedState.isDisposed());
}

Example 35 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class CheckpointCoordinatorTest method testReportStatsAfterFailure.

private void testReportStatsAfterFailure(long checkpointId, TriFunctionWithException<CheckpointCoordinator, Execution, CheckpointMetrics, ?, CheckpointException> reportFn) throws Exception {
    JobVertexID decliningVertexID = new JobVertexID();
    JobVertexID lateReportVertexID = new JobVertexID();
    ExecutionGraph executionGraph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(decliningVertexID).addJobVertex(lateReportVertexID).build();
    ExecutionVertex decliningVertex = executionGraph.getJobVertex(decliningVertexID).getTaskVertices()[0];
    ExecutionVertex lateReportVertex = executionGraph.getJobVertex(lateReportVertexID).getTaskVertices()[0];
    CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(Integer.MAX_VALUE, new UnregisteredMetricsGroup());
    CheckpointCoordinator coordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(executionGraph).setTimer(manuallyTriggeredScheduledExecutor).setCheckpointStatsTracker(statsTracker).build();
    CompletableFuture<CompletedCheckpoint> result = coordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    checkState(coordinator.getNumberOfPendingCheckpoints() == 1, "wrong number of pending checkpoints: %s", coordinator.getNumberOfPendingCheckpoints());
    if (result.isDone()) {
        result.get();
    }
    coordinator.receiveDeclineMessage(new DeclineCheckpoint(executionGraph.getJobID(), decliningVertex.getCurrentExecutionAttempt().getAttemptId(), checkpointId, new CheckpointException(CHECKPOINT_DECLINED)), "test");
    CheckpointMetrics lateReportedMetrics = new CheckpointMetricsBuilder().setTotalBytesPersisted(18).setBytesPersistedOfThisCheckpoint(18).setBytesProcessedDuringAlignment(19).setAsyncDurationMillis(20).setAlignmentDurationNanos(123 * 1_000_000).setCheckpointStartDelayNanos(567 * 1_000_000).build();
    reportFn.apply(coordinator, lateReportVertex.getCurrentExecutionAttempt(), lateReportedMetrics);
    assertStatsEqual(checkpointId, lateReportVertex.getJobvertexId(), 0, lateReportedMetrics, statsTracker.createSnapshot().getHistory().getCheckpointById(checkpointId));
}