Examples with SchedulingStrategyFactory org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory used on opensource projects

Example 1 with SchedulingStrategyFactory

use of org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory in project flink by apache.

the class DefaultSchedulerTest method vertexIsResetBeforeRestarted.

@Test
public void vertexIsResetBeforeRestarted() throws Exception {
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory);
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    final SchedulingTopology topology = schedulingStrategy.getSchedulingTopology();
    scheduler.startScheduling();
    final SchedulingExecutionVertex onlySchedulingVertex = Iterables.getOnlyElement(topology.getVertices());
    schedulingStrategy.schedule(Collections.singletonList(onlySchedulingVertex.getId()));
    final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices());
    final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
    scheduler.updateTaskExecutionState(createFailedTaskExecutionState(attemptId));
    taskRestartExecutor.triggerScheduledTasks();
    assertThat(schedulingStrategy.getReceivedVerticesToRestart(), hasSize(1));
    assertThat(onlySchedulingVertex.getState(), is(equalTo(ExecutionState.CREATED)));
}

Example 2 with SchedulingStrategyFactory

use of org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory in project flink by apache.

the class DefaultSchedulerTest method deployTasksOnlyWhenAllSlotRequestsAreFulfilled.

@Test
public void deployTasksOnlyWhenAllSlotRequestsAreFulfilled() throws Exception {
    final JobGraph jobGraph = singleJobVertexJobGraph(4);
    final JobVertexID onlyJobVertexId = getOnlyJobVertex(jobGraph).getID();
    testExecutionSlotAllocator.disableAutoCompletePendingRequests();
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory);
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    scheduler.startScheduling();
    final List<ExecutionVertexID> verticesToSchedule = Arrays.asList(new ExecutionVertexID(onlyJobVertexId, 0), new ExecutionVertexID(onlyJobVertexId, 1), new ExecutionVertexID(onlyJobVertexId, 2), new ExecutionVertexID(onlyJobVertexId, 3));
    schedulingStrategy.schedule(verticesToSchedule);
    assertThat(testExecutionVertexOperations.getDeployedVertices(), hasSize(0));
    testExecutionSlotAllocator.completePendingRequest(verticesToSchedule.get(0));
    assertThat(testExecutionVertexOperations.getDeployedVertices(), hasSize(0));
    testExecutionSlotAllocator.completePendingRequests();
    assertThat(testExecutionVertexOperations.getDeployedVertices(), hasSize(4));
}

Example 3 with SchedulingStrategyFactory

use of org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory in project flink by apache.

the class DefaultSchedulerTest method scheduledVertexOrderFromSchedulingStrategyIsRespected.

@Test
public void scheduledVertexOrderFromSchedulingStrategyIsRespected() throws Exception {
    final JobGraph jobGraph = singleJobVertexJobGraph(10);
    final JobVertexID onlyJobVertexId = getOnlyJobVertex(jobGraph).getID();
    final List<ExecutionVertexID> desiredScheduleOrder = Arrays.asList(new ExecutionVertexID(onlyJobVertexId, 4), new ExecutionVertexID(onlyJobVertexId, 0), new ExecutionVertexID(onlyJobVertexId, 3), new ExecutionVertexID(onlyJobVertexId, 1), new ExecutionVertexID(onlyJobVertexId, 2));
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory);
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    schedulingStrategy.schedule(desiredScheduleOrder);
    final List<ExecutionVertexID> deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices();
    assertEquals(desiredScheduleOrder, deployedExecutionVertices);
}

Example 4 with SchedulingStrategyFactory

use of org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory in project flink by apache.

the class DefaultSchedulerTest method testRestartVerticesOnFailuresInScheduling.

private void testRestartVerticesOnFailuresInScheduling(Consumer<ExecutionVertexID> actionsToTriggerTaskFailure) throws Exception {
    final int parallelism = 2;
    final JobVertex v1 = createVertex("vertex1", parallelism);
    final JobVertex v2 = createVertex("vertex2", parallelism);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(v1, v2);
    testExecutionSlotAllocator.disableAutoCompletePendingRequests();
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory, new RestartPipelinedRegionFailoverStrategy.Factory());
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    scheduler.startScheduling();
    final ExecutionVertexID vid11 = new ExecutionVertexID(v1.getID(), 0);
    final ExecutionVertexID vid12 = new ExecutionVertexID(v1.getID(), 1);
    final ExecutionVertexID vid21 = new ExecutionVertexID(v2.getID(), 0);
    final ExecutionVertexID vid22 = new ExecutionVertexID(v2.getID(), 1);
    schedulingStrategy.schedule(Arrays.asList(vid11, vid12, vid21, vid22));
    assertThat(testExecutionSlotAllocator.getPendingRequests().keySet(), hasSize(4));
    actionsToTriggerTaskFailure.accept(vid11);
    final Iterator<ArchivedExecutionVertex> vertexIterator = scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices().iterator();
    final ArchivedExecutionVertex ev11 = vertexIterator.next();
    final ArchivedExecutionVertex ev12 = vertexIterator.next();
    final ArchivedExecutionVertex ev21 = vertexIterator.next();
    final ArchivedExecutionVertex ev22 = vertexIterator.next();
    // ev11 and ev21 needs to be restarted because it is pipelined region failover and
    // they are in the same region. ev12 and ev22 will not be affected
    assertThat(testExecutionSlotAllocator.getPendingRequests().keySet(), hasSize(2));
    assertThat(ev11.getExecutionState(), is(ExecutionState.FAILED));
    assertThat(ev21.getExecutionState(), is(ExecutionState.CANCELED));
    assertThat(ev12.getExecutionState(), is(ExecutionState.SCHEDULED));
    assertThat(ev22.getExecutionState(), is(ExecutionState.SCHEDULED));
    taskRestartExecutor.triggerScheduledTasks();
    assertThat(schedulingStrategy.getReceivedVerticesToRestart(), containsInAnyOrder(vid11, vid21));
}

Example 5 with SchedulingStrategyFactory

use of org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory in project flink by apache.

the class DefaultSchedulerTest method scheduleOnlyIfVertexIsCreated.

@Test
public void scheduleOnlyIfVertexIsCreated() throws Exception {
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory);
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    final SchedulingTopology topology = schedulingStrategy.getSchedulingTopology();
    scheduler.startScheduling();
    final ExecutionVertexID onlySchedulingVertexId = Iterables.getOnlyElement(topology.getVertices()).getId();
    // Schedule the vertex to get it to a non-CREATED state
    schedulingStrategy.schedule(Collections.singletonList(onlySchedulingVertexId));
    // The scheduling of a non-CREATED vertex will result in IllegalStateException
    try {
        schedulingStrategy.schedule(Collections.singletonList(onlySchedulingVertexId));
        fail("IllegalStateException should happen");
    } catch (IllegalStateException e) {
    // expected exception
    }
}