Examples with Scheduler org.apache.flink.runtime.jobmanager.scheduler.Scheduler used on opensource projects

Example 6 with Scheduler

use of org.apache.flink.runtime.jobmanager.scheduler.Scheduler in project flink by apache.

the class ExecutionGraphRestartTest method testFailExecutionGraphAfterCancel.

/**
	 * Tests that it is possible to fail a graph via a call to
	 * {@link ExecutionGraph#fail(Throwable)} after cancellation.
	 */
@Test
public void testFailExecutionGraphAfterCancel() throws Exception {
    Instance instance = ExecutionGraphTestUtils.getInstance(new ActorTaskManagerGateway(new SimpleActorGateway(TestingUtils.directExecutionContext())), 2);
    Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
    scheduler.newInstanceAvailable(instance);
    JobVertex vertex = newJobVertex("Test Vertex", 1, NoOpInvokable.class);
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, Integer.MAX_VALUE));
    JobGraph jobGraph = new JobGraph("Test Job", vertex);
    jobGraph.setExecutionConfig(executionConfig);
    ExecutionGraph eg = newExecutionGraph(new InfiniteDelayRestartStrategy(), scheduler);
    eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
    assertEquals(JobStatus.CREATED, eg.getState());
    eg.scheduleForExecution();
    assertEquals(JobStatus.RUNNING, eg.getState());
    // Fail right after cancel (for example with concurrent slot release)
    eg.cancel();
    assertEquals(JobStatus.CANCELLING, eg.getState());
    eg.fail(new Exception("Test Exception"));
    assertEquals(JobStatus.FAILING, eg.getState());
    Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();
    execution.cancelingComplete();
    assertEquals(JobStatus.RESTARTING, eg.getState());
}

Example 7 with Scheduler

use of org.apache.flink.runtime.jobmanager.scheduler.Scheduler in project flink by apache.

the class ExecutionGraphRestartTest method testFailingExecutionAfterRestart.

/**
	 * Tests that a failing execution does not affect a restarted job. This is important if a
	 * callback handler fails an execution after it has already reached a final state and the job
	 * has been restarted.
	 */
@Test
public void testFailingExecutionAfterRestart() throws Exception {
    Instance instance = ExecutionGraphTestUtils.getInstance(new ActorTaskManagerGateway(new SimpleActorGateway(TestingUtils.directExecutionContext())), 2);
    Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
    scheduler.newInstanceAvailable(instance);
    JobVertex sender = newJobVertex("Task1", 1, NoOpInvokable.class);
    JobVertex receiver = newJobVertex("Task2", 1, NoOpInvokable.class);
    JobGraph jobGraph = new JobGraph("Pointwise job", sender, receiver);
    ExecutionGraph eg = newExecutionGraph(new FixedDelayRestartStrategy(1, 1000), scheduler);
    eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());
    assertEquals(JobStatus.CREATED, eg.getState());
    eg.scheduleForExecution();
    assertEquals(JobStatus.RUNNING, eg.getState());
    Iterator<ExecutionVertex> executionVertices = eg.getAllExecutionVertices().iterator();
    Execution finishedExecution = executionVertices.next().getCurrentExecutionAttempt();
    Execution failedExecution = executionVertices.next().getCurrentExecutionAttempt();
    finishedExecution.markFinished();
    failedExecution.fail(new Exception("Test Exception"));
    failedExecution.cancelingComplete();
    FiniteDuration timeout = new FiniteDuration(2, TimeUnit.MINUTES);
    waitForAsyncRestart(eg, timeout);
    assertEquals(JobStatus.RUNNING, eg.getState());
    // Wait for all resources to be assigned after async restart
    waitForAllResourcesToBeAssignedAfterAsyncRestart(eg, timeout.fromNow());
    // At this point all resources have been assigned
    for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
        assertNotNull("No assigned resource (test instability).", vertex.getCurrentAssignedResource());
        vertex.getCurrentExecutionAttempt().switchToRunning();
    }
    // fail old finished execution, this should not affect the execution
    finishedExecution.fail(new Exception("This should have no effect"));
    for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
        vertex.getCurrentExecutionAttempt().markFinished();
    }
    // the state of the finished execution should have not changed since it is terminal
    assertEquals(ExecutionState.FINISHED, finishedExecution.getState());
    assertEquals(JobStatus.FINISHED, eg.getState());
}

Example 8 with Scheduler

use of org.apache.flink.runtime.jobmanager.scheduler.Scheduler in project flink by apache.

the class ExecutionVertexSchedulingTest method testSlotReleasedWhenScheduledQueued.

@Test
public void testSlotReleasedWhenScheduledQueued() {
    try {
        final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
        final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0], AkkaUtils.getDefaultTimeout());
        // a slot than cannot be deployed to
        final Instance instance = getInstance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE));
        final SimpleSlot slot = instance.allocateSimpleSlot(ejv.getJobId());
        slot.releaseSlot();
        assertTrue(slot.isReleased());
        final FlinkCompletableFuture<SimpleSlot> future = new FlinkCompletableFuture<>();
        Scheduler scheduler = mock(Scheduler.class);
        when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean())).thenReturn(future);
        assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
        // try to deploy to the slot
        vertex.scheduleForExecution(scheduler, true);
        // future has not yet a slot
        assertEquals(ExecutionState.SCHEDULED, vertex.getExecutionState());
        future.complete(slot);
        // will have failed
        assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 9 with Scheduler

use of org.apache.flink.runtime.jobmanager.scheduler.Scheduler in project flink by apache.

the class ExecutionVertexSchedulingTest method testSlotReleasedWhenScheduledImmediately.

@Test
public void testSlotReleasedWhenScheduledImmediately() {
    try {
        final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
        final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0], AkkaUtils.getDefaultTimeout());
        // a slot than cannot be deployed to
        final Instance instance = getInstance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE));
        final SimpleSlot slot = instance.allocateSimpleSlot(ejv.getJobId());
        slot.releaseSlot();
        assertTrue(slot.isReleased());
        Scheduler scheduler = mock(Scheduler.class);
        FlinkCompletableFuture<SimpleSlot> future = new FlinkCompletableFuture<>();
        future.complete(slot);
        when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean())).thenReturn(future);
        assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
        // try to deploy to the slot
        vertex.scheduleForExecution(scheduler, false);
        // will have failed
        assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with Scheduler

use of org.apache.flink.runtime.jobmanager.scheduler.Scheduler in project flink by apache.

the class ExecutionVertexSchedulingTest method testScheduleToDeploying.

@Test
public void testScheduleToDeploying() {
    try {
        final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
        final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0], AkkaUtils.getDefaultTimeout());
        final Instance instance = getInstance(new ActorTaskManagerGateway(new ExecutionGraphTestUtils.SimpleActorGateway(TestingUtils.defaultExecutionContext())));
        final SimpleSlot slot = instance.allocateSimpleSlot(ejv.getJobId());
        Scheduler scheduler = mock(Scheduler.class);
        FlinkCompletableFuture<SimpleSlot> future = new FlinkCompletableFuture<>();
        future.complete(slot);
        when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean())).thenReturn(future);
        assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
        // try to deploy to the slot
        vertex.scheduleForExecution(scheduler, false);
        assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}