Examples with ExecutionGraph org.apache.flink.runtime.executiongraph.ExecutionGraph used on opensource projects

Example 56 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class SchedulerBase method createAndRestoreExecutionGraph.

private ExecutionGraph createAndRestoreExecutionGraph(CompletedCheckpointStore completedCheckpointStore, CheckpointsCleaner checkpointsCleaner, CheckpointIDCounter checkpointIdCounter, long initializationTimestamp, ComponentMainThreadExecutor mainThreadExecutor, JobStatusListener jobStatusListener, VertexParallelismStore vertexParallelismStore) throws Exception {
    final ExecutionGraph newExecutionGraph = executionGraphFactory.createAndRestoreExecutionGraph(jobGraph, completedCheckpointStore, checkpointsCleaner, checkpointIdCounter, TaskDeploymentDescriptorFactory.PartitionLocationConstraint.fromJobType(jobGraph.getJobType()), initializationTimestamp, new DefaultVertexAttemptNumberStore(), vertexParallelismStore, deploymentStateTimeMetrics, log);
    newExecutionGraph.setInternalTaskFailuresListener(new UpdateSchedulerNgOnInternalFailuresListener(this));
    newExecutionGraph.registerJobStatusListener(jobStatusListener);
    newExecutionGraph.start(mainThreadExecutor);
    return newExecutionGraph;
}

Example 57 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class Executing method stopWithSavepoint.

CompletableFuture<String> stopWithSavepoint(@Nullable final String targetDirectory, boolean terminate, SavepointFormatType formatType) {
    final ExecutionGraph executionGraph = getExecutionGraph();
    StopWithSavepointTerminationManager.checkSavepointActionPreconditions(executionGraph.getCheckpointCoordinator(), targetDirectory, executionGraph.getJobID(), getLogger());
    getLogger().info("Triggering stop-with-savepoint for job {}.", executionGraph.getJobID());
    CheckpointScheduling schedulingProvider = new CheckpointSchedulingProvider(executionGraph);
    schedulingProvider.stopCheckpointScheduler();
    final CompletableFuture<String> savepointFuture = executionGraph.getCheckpointCoordinator().triggerSynchronousSavepoint(terminate, targetDirectory, formatType).thenApply(CompletedCheckpoint::getExternalPointer);
    return context.goToStopWithSavepoint(executionGraph, getExecutionGraphHandler(), getOperatorCoordinatorHandler(), schedulingProvider, savepointFuture, getFailures());
}

Example 58 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class PipelinedRegionSchedulingStrategyTest method testSchedulingTopologyWithCrossRegionConsumedPartitionGroups.

@Test
public void testSchedulingTopologyWithCrossRegionConsumedPartitionGroups() throws Exception {
    final JobVertex v1 = createJobVertex("v1", 4);
    final JobVertex v2 = createJobVertex("v2", 3);
    final JobVertex v3 = createJobVertex("v3", 2);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    v3.connectNewDataSetAsInput(v2, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
    v3.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    final List<JobVertex> ordered = new ArrayList<>(Arrays.asList(v1, v2, v3));
    final JobGraph jobGraph = JobGraphBuilder.newBatchJobGraphBuilder().addJobVertices(ordered).build();
    final ExecutionGraph executionGraph = TestingDefaultExecutionGraphBuilder.newBuilder().setJobGraph(jobGraph).build();
    final SchedulingTopology schedulingTopology = executionGraph.getSchedulingTopology();
    // Test whether the topology is built correctly
    final List<SchedulingPipelinedRegion> regions = new ArrayList<>();
    schedulingTopology.getAllPipelinedRegions().forEach(regions::add);
    assertEquals(2, regions.size());
    final ExecutionVertex v31 = executionGraph.getJobVertex(v3.getID()).getTaskVertices()[0];
    final Set<ExecutionVertexID> region1 = new HashSet<>();
    schedulingTopology.getPipelinedRegionOfVertex(v31.getID()).getVertices().forEach(vertex -> region1.add(vertex.getId()));
    assertEquals(5, region1.size());
    final ExecutionVertex v32 = executionGraph.getJobVertex(v3.getID()).getTaskVertices()[1];
    final Set<ExecutionVertexID> region2 = new HashSet<>();
    schedulingTopology.getPipelinedRegionOfVertex(v32.getID()).getVertices().forEach(vertex -> region2.add(vertex.getId()));
    assertEquals(4, region2.size());
    // Test whether region 1 is scheduled correctly
    PipelinedRegionSchedulingStrategy schedulingStrategy = startScheduling(schedulingTopology);
    assertEquals(1, testingSchedulerOperation.getScheduledVertices().size());
    final List<ExecutionVertexDeploymentOption> deploymentOptions1 = testingSchedulerOperation.getScheduledVertices().get(0);
    assertEquals(5, deploymentOptions1.size());
    for (ExecutionVertexDeploymentOption deploymentOption : deploymentOptions1) {
        assertTrue(region1.contains(deploymentOption.getExecutionVertexId()));
    }
    // Test whether the region 2 is scheduled correctly when region 1 is finished
    final ExecutionVertex v22 = executionGraph.getJobVertex(v2.getID()).getTaskVertices()[1];
    v22.finishAllBlockingPartitions();
    schedulingStrategy.onExecutionStateChange(v22.getID(), ExecutionState.FINISHED);
    assertEquals(2, testingSchedulerOperation.getScheduledVertices().size());
    final List<ExecutionVertexDeploymentOption> deploymentOptions2 = testingSchedulerOperation.getScheduledVertices().get(1);
    assertEquals(4, deploymentOptions2.size());
    for (ExecutionVertexDeploymentOption deploymentOption : deploymentOptions2) {
        assertTrue(region2.contains(deploymentOption.getExecutionVertexId()));
    }
}

Example 59 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class DefaultOperatorCoordinatorHandlerTest method testRegisterAndStartNewCoordinators.

@Test
public void testRegisterAndStartNewCoordinators() throws Exception {
    final JobVertex[] jobVertices = createJobVertices(BLOCKING);
    OperatorID operatorId1 = OperatorID.fromJobVertexID(jobVertices[0].getID());
    OperatorID operatorId2 = OperatorID.fromJobVertexID(jobVertices[1].getID());
    ExecutionGraph executionGraph = createDynamicGraph(jobVertices);
    ExecutionJobVertex ejv1 = executionGraph.getJobVertex(jobVertices[0].getID());
    ExecutionJobVertex ejv2 = executionGraph.getJobVertex(jobVertices[1].getID());
    executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
    executionGraph.initializeJobVertex(ejv1, 0L);
    DefaultOperatorCoordinatorHandler handler = new DefaultOperatorCoordinatorHandler(executionGraph, throwable -> {
    });
    assertThat(handler.getCoordinatorMap().keySet(), containsInAnyOrder(operatorId1));
    executionGraph.initializeJobVertex(ejv2, 0L);
    handler.registerAndStartNewCoordinators(ejv2.getOperatorCoordinators(), executionGraph.getJobMasterMainThreadExecutor());
    assertThat(handler.getCoordinatorMap().keySet(), containsInAnyOrder(operatorId1, operatorId2));
}

Example 60 with ExecutionGraph

use of org.apache.flink.runtime.executiongraph.ExecutionGraph in project flink by apache.

the class ExecutionGraphToInputsLocationsRetrieverAdapterTest method testGetTaskManagerLocationWhenScheduled.

/**
 * Tests that it can get the task manager location in an Execution.
 */
@Test
public void testGetTaskManagerLocationWhenScheduled() throws Exception {
    final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);
    final TestingLogicalSlot testingLogicalSlot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
    final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(jobVertex);
    final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever = new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);
    final ExecutionVertex onlyExecutionVertex = eg.getAllExecutionVertices().iterator().next();
    onlyExecutionVertex.getCurrentExecutionAttempt().transitionState(ExecutionState.SCHEDULED);
    onlyExecutionVertex.deployToSlot(testingLogicalSlot);
    ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
    Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocationOptional = inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);
    assertTrue(taskManagerLocationOptional.isPresent());
    final CompletableFuture<TaskManagerLocation> taskManagerLocationFuture = taskManagerLocationOptional.get();
    assertThat(taskManagerLocationFuture.get(), is(testingLogicalSlot.getTaskManagerLocation()));
}