Examples with SchedulingTopology org.apache.flink.runtime.scheduler.strategy.SchedulingTopology used on opensource projects

Example 1 with SchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.SchedulingTopology in project flink by apache.

the class DefaultSchedulerTest method vertexIsResetBeforeRestarted.

@Test
public void vertexIsResetBeforeRestarted() throws Exception {
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory);
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    final SchedulingTopology topology = schedulingStrategy.getSchedulingTopology();
    scheduler.startScheduling();
    final SchedulingExecutionVertex onlySchedulingVertex = Iterables.getOnlyElement(topology.getVertices());
    schedulingStrategy.schedule(Collections.singletonList(onlySchedulingVertex.getId()));
    final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices());
    final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
    scheduler.updateTaskExecutionState(createFailedTaskExecutionState(attemptId));
    taskRestartExecutor.triggerScheduledTasks();
    assertThat(schedulingStrategy.getReceivedVerticesToRestart(), hasSize(1));
    assertThat(onlySchedulingVertex.getState(), is(equalTo(ExecutionState.CREATED)));
}

Example 2 with SchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.SchedulingTopology in project flink by apache.

the class LocalInputPreferredSlotSharingStrategyTest method testInputLocalityIsRespectedWithTwoEdgesBetweenTwoVertices.

/**
 * In this test case, there are two JobEdges between two JobVertices. There will be no
 * ExecutionSlotSharingGroup that contains two vertices with the same JobVertexID.
 */
@Test
public void testInputLocalityIsRespectedWithTwoEdgesBetweenTwoVertices() throws Exception {
    int parallelism = 4;
    JobVertex v1 = createJobVertex("v1", JOB_VERTEX_ID_1, parallelism);
    JobVertex v2 = createJobVertex("v2", JOB_VERTEX_ID_2, parallelism);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
    assertEquals(2, v1.getProducedDataSets().size());
    assertEquals(2, v2.getInputs().size());
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(v1, v2);
    final ExecutionGraph executionGraph = TestingDefaultExecutionGraphBuilder.newBuilder().setJobGraph(jobGraph).build();
    final SchedulingTopology topology = executionGraph.getSchedulingTopology();
    final SlotSharingStrategy strategy = new LocalInputPreferredSlotSharingStrategy(topology, slotSharingGroups, Collections.emptySet());
    assertThat(strategy.getExecutionSlotSharingGroups(), hasSize(4));
    ExecutionVertex[] ev1 = Objects.requireNonNull(executionGraph.getJobVertex(JOB_VERTEX_ID_1)).getTaskVertices();
    ExecutionVertex[] ev2 = Objects.requireNonNull(executionGraph.getJobVertex(JOB_VERTEX_ID_2)).getTaskVertices();
    for (int i = 0; i < parallelism; i++) {
        assertThat(strategy.getExecutionSlotSharingGroup(ev1[i].getID()).getExecutionVertexIds(), containsInAnyOrder(ev1[i].getID(), ev2[i].getID()));
    }
}

Example 3 with SchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.SchedulingTopology in project flink by apache.

the class DefaultSchedulerTest method cancelWhileRestartingShouldWaitForRunningTasks.

@Test
public void cancelWhileRestartingShouldWaitForRunningTasks() {
    final JobGraph jobGraph = singleJobVertexJobGraph(2);
    final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);
    final SchedulingTopology topology = scheduler.getSchedulingTopology();
    final Iterator<ArchivedExecutionVertex> vertexIterator = scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices().iterator();
    final ExecutionAttemptID attemptId1 = vertexIterator.next().getCurrentExecutionAttempt().getAttemptId();
    final ExecutionAttemptID attemptId2 = vertexIterator.next().getCurrentExecutionAttempt().getAttemptId();
    final ExecutionVertexID executionVertex2 = scheduler.getExecutionVertexIdOrThrow(attemptId2);
    scheduler.updateTaskExecutionState(new TaskExecutionState(attemptId1, ExecutionState.FAILED, new RuntimeException("expected")));
    scheduler.cancel();
    final ExecutionState vertex2StateAfterCancel = topology.getVertex(executionVertex2).getState();
    final JobStatus statusAfterCancelWhileRestarting = scheduler.requestJobStatus();
    scheduler.updateTaskExecutionState(new TaskExecutionState(attemptId2, ExecutionState.CANCELED, new RuntimeException("expected")));
    assertThat(vertex2StateAfterCancel, is(equalTo(ExecutionState.CANCELING)));
    assertThat(statusAfterCancelWhileRestarting, is(equalTo(JobStatus.CANCELLING)));
    assertThat(scheduler.requestJobStatus(), is(equalTo(JobStatus.CANCELED)));
}

Example 4 with SchedulingTopology

use of org.apache.flink.runtime.scheduler.strategy.SchedulingTopology in project flink by apache.

the class DefaultSchedulerTest method scheduleOnlyIfVertexIsCreated.

@Test
public void scheduleOnlyIfVertexIsCreated() throws Exception {
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory);
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    final SchedulingTopology topology = schedulingStrategy.getSchedulingTopology();
    scheduler.startScheduling();
    final ExecutionVertexID onlySchedulingVertexId = Iterables.getOnlyElement(topology.getVertices()).getId();
    // Schedule the vertex to get it to a non-CREATED state
    schedulingStrategy.schedule(Collections.singletonList(onlySchedulingVertexId));
    // The scheduling of a non-CREATED vertex will result in IllegalStateException
    try {
        schedulingStrategy.schedule(Collections.singletonList(onlySchedulingVertexId));
        fail("IllegalStateException should happen");
    } catch (IllegalStateException e) {
    // expected exception
    }
}