Examples with ExecutionGraph org.apache.flink.runtime.executiongraph.ExecutionGraph used on opensource projects

Example 51 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class DefaultCheckpointPlanTest method testFulfillFinishedStates.

@Test
public void testFulfillFinishedStates() throws Exception {
    JobVertexID fullyFinishedVertexId = new JobVertexID();
    JobVertexID finishedOnRestoreVertexId = new JobVertexID();
    JobVertexID partiallyFinishedVertexId = new JobVertexID();
    OperatorID fullyFinishedOperatorId = new OperatorID();
    OperatorID finishedOnRestoreOperatorId = new OperatorID();
    OperatorID partiallyFinishedOperatorId = new OperatorID();
    ExecutionGraph executionGraph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(fullyFinishedVertexId, 2, 2, Collections.singletonList(OperatorIDPair.generatedIDOnly(fullyFinishedOperatorId)), true).addJobVertex(finishedOnRestoreVertexId, 2, 2, Collections.singletonList(OperatorIDPair.generatedIDOnly(finishedOnRestoreOperatorId)), true).addJobVertex(partiallyFinishedVertexId, 2, 2, Collections.singletonList(OperatorIDPair.generatedIDOnly(partiallyFinishedOperatorId)), true).build();
    ExecutionVertex[] fullyFinishedVertexTasks = executionGraph.getJobVertex(fullyFinishedVertexId).getTaskVertices();
    ExecutionVertex[] finishedOnRestoreVertexTasks = executionGraph.getJobVertex(finishedOnRestoreVertexId).getTaskVertices();
    ExecutionVertex[] partiallyFinishedVertexTasks = executionGraph.getJobVertex(partiallyFinishedVertexId).getTaskVertices();
    Arrays.stream(fullyFinishedVertexTasks).forEach(task -> task.getCurrentExecutionAttempt().markFinished());
    partiallyFinishedVertexTasks[0].getCurrentExecutionAttempt().markFinished();
    CheckpointPlan checkpointPlan = createCheckpointPlan(executionGraph);
    Arrays.stream(finishedOnRestoreVertexTasks).forEach(checkpointPlan::reportTaskFinishedOnRestore);
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    checkpointPlan.fulfillFinishedTaskStatus(operatorStates);
    assertEquals(3, operatorStates.size());
    assertTrue(operatorStates.get(fullyFinishedOperatorId).isFullyFinished());
    assertTrue(operatorStates.get(finishedOnRestoreOperatorId).isFullyFinished());
    OperatorState operatorState = operatorStates.get(partiallyFinishedOperatorId);
    assertFalse(operatorState.isFullyFinished());
    assertTrue(operatorState.getState(0).isFinished());
}

Example 52 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class DefaultCheckpointPlanTest method testAbortionIfPartiallyFinishedVertexUsedUnionListState.

@Test
public void testAbortionIfPartiallyFinishedVertexUsedUnionListState() throws Exception {
    JobVertexID jobVertexId = new JobVertexID();
    OperatorID operatorId = new OperatorID();
    ExecutionGraph executionGraph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexId, 2, 2, Collections.singletonList(OperatorIDPair.generatedIDOnly(operatorId)), true).build();
    ExecutionVertex[] tasks = executionGraph.getJobVertex(jobVertexId).getTaskVertices();
    tasks[0].getCurrentExecutionAttempt().markFinished();
    CheckpointPlan checkpointPlan = createCheckpointPlan(executionGraph);
    Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
    OperatorState operatorState = new OperatorState(operatorId, 2, 2);
    operatorState.putState(0, createSubtaskStateWithUnionListState(TEMPORARY_FOLDER.newFile()));
    operatorStates.put(operatorId, operatorState);
    expectedException.expect(FlinkRuntimeException.class);
    expectedException.expectMessage(String.format("The vertex %s (id = %s) has " + "used UnionListState, but part of its tasks are FINISHED", executionGraph.getJobVertex(jobVertexId).getName(), jobVertexId));
    checkpointPlan.fulfillFinishedTaskStatus(operatorStates);
}

Example 53 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class DefaultExecutionGraphFactory method createAndRestoreExecutionGraph.

@Override
public ExecutionGraph createAndRestoreExecutionGraph(JobGraph jobGraph, CompletedCheckpointStore completedCheckpointStore, CheckpointsCleaner checkpointsCleaner, CheckpointIDCounter checkpointIdCounter, TaskDeploymentDescriptorFactory.PartitionLocationConstraint partitionLocationConstraint, long initializationTimestamp, VertexAttemptNumberStore vertexAttemptNumberStore, VertexParallelismStore vertexParallelismStore, ExecutionStateUpdateListener executionStateUpdateListener, Logger log) throws Exception {
    ExecutionDeploymentListener executionDeploymentListener = new ExecutionDeploymentTrackerDeploymentListenerAdapter(executionDeploymentTracker);
    ExecutionStateUpdateListener combinedExecutionStateUpdateListener = (execution, previousState, newState) -> {
        executionStateUpdateListener.onStateUpdate(execution, previousState, newState);
        if (newState.isTerminal()) {
            executionDeploymentTracker.stopTrackingDeploymentOf(execution);
        }
    };
    final ExecutionGraph newExecutionGraph = DefaultExecutionGraphBuilder.buildGraph(jobGraph, configuration, futureExecutor, ioExecutor, userCodeClassLoader, completedCheckpointStore, checkpointsCleaner, checkpointIdCounter, rpcTimeout, blobWriter, log, shuffleMaster, jobMasterPartitionTracker, partitionLocationConstraint, executionDeploymentListener, combinedExecutionStateUpdateListener, initializationTimestamp, vertexAttemptNumberStore, vertexParallelismStore, checkpointStatsTrackerFactory, isDynamicGraph);
    final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();
    if (checkpointCoordinator != null) {
        // check whether we find a valid checkpoint
        if (!checkpointCoordinator.restoreInitialCheckpointIfPresent(new HashSet<>(newExecutionGraph.getAllVertices().values()))) {
            // check whether we can restore from a savepoint
            tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
        }
    }
    return newExecutionGraph;
}

Example 54 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class AdaptiveScheduler method tryToAssignSlots.

@Override
public CreatingExecutionGraph.AssignmentResult tryToAssignSlots(CreatingExecutionGraph.ExecutionGraphWithVertexParallelism executionGraphWithVertexParallelism) {
    final ExecutionGraph executionGraph = executionGraphWithVertexParallelism.getExecutionGraph();
    executionGraph.start(componentMainThreadExecutor);
    executionGraph.transitionToRunning();
    executionGraph.setInternalTaskFailuresListener(new UpdateSchedulerNgOnInternalFailuresListener(this));
    final VertexParallelism vertexParallelism = executionGraphWithVertexParallelism.getVertexParallelism();
    return slotAllocator.tryReserveResources(vertexParallelism).map(reservedSlots -> CreatingExecutionGraph.AssignmentResult.success(assignSlotsToExecutionGraph(executionGraph, reservedSlots))).orElseGet(CreatingExecutionGraph.AssignmentResult::notPossible);
}

Example 55 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class AdaptiveScheduler method createExecutionGraphWithAvailableResourcesAsync.

private CompletableFuture<CreatingExecutionGraph.ExecutionGraphWithVertexParallelism> createExecutionGraphWithAvailableResourcesAsync() {
    final VertexParallelism vertexParallelism;
    final VertexParallelismStore adjustedParallelismStore;
    try {
        vertexParallelism = determineParallelism(slotAllocator);
        JobGraph adjustedJobGraph = jobInformation.copyJobGraph();
        for (JobVertex vertex : adjustedJobGraph.getVertices()) {
            JobVertexID id = vertex.getID();
            // use the determined "available parallelism" to use
            // the resources we have access to
            vertex.setParallelism(vertexParallelism.getParallelism(id));
        }
        // use the originally configured max parallelism
        // as the default for consistent runs
        adjustedParallelismStore = computeVertexParallelismStoreForExecution(adjustedJobGraph, executionMode, (vertex) -> {
            VertexParallelismInformation vertexParallelismInfo = initialParallelismStore.getParallelismInfo(vertex.getID());
            return vertexParallelismInfo.getMaxParallelism();
        });
    } catch (Exception exception) {
        return FutureUtils.completedExceptionally(exception);
    }
    return createExecutionGraphAndRestoreStateAsync(adjustedParallelismStore).thenApply(executionGraph -> CreatingExecutionGraph.ExecutionGraphWithVertexParallelism.create(executionGraph, vertexParallelism));
}