Examples with SchedulerBase org.apache.flink.runtime.scheduler.SchedulerBase used on opensource projects

Example 31 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method testExecutionGraphIsDeployedInTopologicalOrder.

/**
 * Tests that the {@link ExecutionGraph} is deployed in topological order.
 */
@Test
public void testExecutionGraphIsDeployedInTopologicalOrder() throws Exception {
    final int sourceParallelism = 2;
    final int sinkParallelism = 1;
    final JobVertex sourceVertex = new JobVertex("source");
    sourceVertex.setInvokableClass(NoOpInvokable.class);
    sourceVertex.setParallelism(sourceParallelism);
    final JobVertex sinkVertex = new JobVertex("sink");
    sinkVertex.setInvokableClass(NoOpInvokable.class);
    sinkVertex.setParallelism(sinkParallelism);
    sinkVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    final int numberTasks = sourceParallelism + sinkParallelism;
    final ArrayBlockingQueue<ExecutionAttemptID> submittedTasksQueue = new ArrayBlockingQueue<>(numberTasks);
    TestingTaskExecutorGatewayBuilder testingTaskExecutorGatewayBuilder = new TestingTaskExecutorGatewayBuilder();
    testingTaskExecutorGatewayBuilder.setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> {
        submittedTasksQueue.offer(taskDeploymentDescriptor.getExecutionAttemptId());
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingTaskExecutorGateway taskExecutorGateway = testingTaskExecutorGatewayBuilder.createTestingTaskExecutorGateway();
    final RpcTaskManagerGateway taskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, JobMasterId.generate());
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sourceVertex, sinkVertex);
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithoutImmediatePhysicalSlotCreation();
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).setFutureExecutor(new DirectScheduledExecutorService()).build();
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    scheduler.startScheduling();
    // change the order in which the futures are completed
    final List<CompletableFuture<TestingPhysicalSlot>> shuffledFutures = new ArrayList<>(physicalSlotProvider.getResponses().values());
    Collections.shuffle(shuffledFutures);
    for (CompletableFuture<TestingPhysicalSlot> slotFuture : shuffledFutures) {
        slotFuture.complete(TestingPhysicalSlot.builder().withTaskManagerLocation(taskManagerLocation).withTaskManagerGateway(taskManagerGateway).build());
    }
    final List<ExecutionAttemptID> submittedTasks = new ArrayList<>(numberTasks);
    for (int i = 0; i < numberTasks; i++) {
        submittedTasks.add(submittedTasksQueue.take());
    }
    final Collection<ExecutionAttemptID> firstStage = new ArrayList<>(sourceParallelism);
    for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sourceVertex.getID()).getTaskVertices()) {
        firstStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
    }
    final Collection<ExecutionAttemptID> secondStage = new ArrayList<>(sinkParallelism);
    for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sinkVertex.getID()).getTaskVertices()) {
        secondStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
    }
    assertThat(submittedTasks, new ExecutionStageMatcher(Arrays.asList(firstStage, secondStage)));
}

Example 32 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method testRegistrationOfExecutionsFailing.

@Test
public void testRegistrationOfExecutionsFailing() {
    try {
        final JobVertexID jid1 = new JobVertexID();
        final JobVertexID jid2 = new JobVertexID();
        JobVertex v1 = new JobVertex("v1", jid1);
        JobVertex v2 = new JobVertex("v2", jid2);
        SchedulerBase scheduler = setupScheduler(v1, 7, v2, 6);
        Collection<Execution> executions = new ArrayList<>(scheduler.getExecutionGraph().getRegisteredExecutions().values());
        for (Execution e : executions) {
            e.markFailed(null);
        }
        assertEquals(0, scheduler.getExecutionGraph().getRegisteredExecutions().size());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 33 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphRestartTest method testCancelAllPendingRequestWhileCanceling.

@Test
public void testCancelAllPendingRequestWhileCanceling() throws Exception {
    try (DeclarativeSlotPoolBridge slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        final int numTasksExceedSlotPool = 50;
        // create a graph with task count larger than slot pool
        JobVertex sender = ExecutionGraphTestUtils.createJobVertex("Task", NUM_TASKS + numTasksExceedSlotPool, NoOpInvokable.class);
        JobGraph graph = JobGraphTestUtils.streamingJobGraph(sender);
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(graph, mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).build();
        ExecutionGraph executionGraph = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        offerSlots(slotPool, NUM_TASKS);
        assertEquals(numTasksExceedSlotPool, slotPool.getNumPendingRequests());
        scheduler.cancel();
        assertEquals(JobStatus.CANCELLING, executionGraph.getState());
        assertEquals(0, slotPool.getNumPendingRequests());
    }
}

Example 34 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphRestartTest method testCancelAllPendingRequestWhileFailing.

@Test
public void testCancelAllPendingRequestWhileFailing() throws Exception {
    try (DeclarativeSlotPoolBridge slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        final int numTasksExceedSlotPool = 50;
        // create a graph with task count larger than slot pool
        JobVertex sender = ExecutionGraphTestUtils.createJobVertex("Task", NUM_TASKS + numTasksExceedSlotPool, NoOpInvokable.class);
        JobGraph graph = JobGraphTestUtils.streamingJobGraph(sender);
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(graph, mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).build();
        ExecutionGraph executionGraph = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        offerSlots(slotPool, NUM_TASKS);
        assertEquals(numTasksExceedSlotPool, slotPool.getNumPendingRequests());
        scheduler.handleGlobalFailure(new Exception("test"));
        assertEquals(JobStatus.FAILING, executionGraph.getState());
        assertEquals(0, slotPool.getNumPendingRequests());
    }
}

Example 35 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphRestartTest method testFailExecutionAfterCancel.

/**
 * Tests that a graph is not restarted after cancellation via a call to {@link
 * Execution#fail(Throwable)}. This can happen when a slot is released concurrently with
 * cancellation.
 */
@Test
public void testFailExecutionAfterCancel() throws Exception {
    try (SlotPool slotPool = SlotPoolUtils.createDeclarativeSlotPoolBridge()) {
        SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(createJobGraphToCancel(), mainThreadExecutor).setExecutionSlotAllocatorFactory(createExecutionSlotAllocatorFactory(slotPool)).setRestartBackoffTimeStrategy(new TestRestartBackoffTimeStrategy(false, Long.MAX_VALUE)).setDelayExecutor(taskRestartExecutor).build();
        ExecutionGraph eg = scheduler.getExecutionGraph();
        startScheduling(scheduler);
        offerSlots(slotPool, 1);
        // Fail right after cancel (for example with concurrent slot release)
        scheduler.cancel();
        for (ExecutionVertex v : eg.getAllExecutionVertices()) {
            v.getCurrentExecutionAttempt().fail(new Exception("Test Exception"));
        }
        assertEquals(JobStatus.CANCELED, eg.getTerminationFuture().get());
        Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();
        execution.completeCancelling();
        assertEquals(JobStatus.CANCELED, eg.getState());
    }
}