Examples with SchedulerBase org.apache.flink.runtime.scheduler.SchedulerBase used on opensource projects

Example 26 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class FinalizeOnMasterTest method testFinalizeIsCalledUponSuccess.

@Test
public void testFinalizeIsCalledUponSuccess() throws Exception {
    final JobVertex vertex1 = spy(new JobVertex("test vertex 1"));
    vertex1.setInvokableClass(NoOpInvokable.class);
    vertex1.setParallelism(3);
    final JobVertex vertex2 = spy(new JobVertex("test vertex 2"));
    vertex2.setInvokableClass(NoOpInvokable.class);
    vertex2.setParallelism(2);
    final SchedulerBase scheduler = createScheduler(JobGraphTestUtils.streamingJobGraph(vertex1, vertex2), ComponentMainThreadExecutorServiceAdapter.forMainThread());
    scheduler.startScheduling();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    assertEquals(JobStatus.RUNNING, eg.getState());
    ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
    // move all vertices to finished state
    ExecutionGraphTestUtils.finishAllVertices(eg);
    assertEquals(JobStatus.FINISHED, eg.waitUntilTerminal());
    verify(vertex1, times(1)).finalizeOnMaster(any(ClassLoader.class));
    verify(vertex2, times(1)).finalizeOnMaster(any(ClassLoader.class));
    assertEquals(0, eg.getRegisteredExecutions().size());
}

Example 27 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphCoLocationRestartTest method testConstraintsAfterRestart.

@Test
public void testConstraintsAfterRestart() throws Exception {
    final long timeout = 5000L;
    JobVertex groupVertex = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
    JobVertex groupVertex2 = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
    groupVertex2.connectNewDataSetAsInput(groupVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    SlotSharingGroup sharingGroup = new SlotSharingGroup();
    groupVertex.setSlotSharingGroup(sharingGroup);
    groupVertex2.setSlotSharingGroup(sharingGroup);
    groupVertex.setStrictlyCoLocatedWith(groupVertex2);
    // initiate and schedule job
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(groupVertex, groupVertex2);
    final ManuallyTriggeredScheduledExecutorService delayExecutor = new ManuallyTriggeredScheduledExecutorService();
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(TestingPhysicalSlotProvider.create((ignored) -> CompletableFuture.completedFuture(TestingPhysicalSlot.builder().build())))).setDelayExecutor(delayExecutor).setRestartBackoffTimeStrategy(new FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory(1, 0).create()).build();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    // enable the queued scheduling for the slot pool
    assertEquals(JobStatus.CREATED, eg.getState());
    scheduler.startScheduling();
    Predicate<AccessExecution> isDeploying = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING);
    ExecutionGraphTestUtils.waitForAllExecutionsPredicate(eg, isDeploying, timeout);
    assertEquals(JobStatus.RUNNING, eg.getState());
    // sanity checks
    validateConstraints(eg);
    eg.getAllExecutionVertices().iterator().next().fail(new FlinkException("Test exception"));
    assertEquals(JobStatus.RESTARTING, eg.getState());
    // trigger registration of restartTasks(...) callback to cancelFuture before completing the
    // cancellation. This ensures the restarting actions to be performed in main thread.
    delayExecutor.triggerNonPeriodicScheduledTask();
    for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
        if (vertex.getExecutionState() == ExecutionState.CANCELING) {
            vertex.getCurrentExecutionAttempt().completeCancelling();
        }
    }
    // wait until we have restarted
    ExecutionGraphTestUtils.waitUntilJobStatus(eg, JobStatus.RUNNING, timeout);
    ExecutionGraphTestUtils.waitForAllExecutionsPredicate(eg, isDeploying, timeout);
    // checking execution vertex properties
    validateConstraints(eg);
    ExecutionGraphTestUtils.finishAllVertices(eg);
    assertThat(eg.getState(), is(FINISHED));
}

Example 28 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphPartitionReleaseTest method testStrategyNotifiedOfFinishedVerticesAndResultsRespected.

@Test
public void testStrategyNotifiedOfFinishedVerticesAndResultsRespected() throws Exception {
    // setup a simple pipeline of 3 operators with blocking partitions
    final JobVertex sourceVertex = ExecutionGraphTestUtils.createNoOpVertex(1);
    final JobVertex operatorVertex = ExecutionGraphTestUtils.createNoOpVertex(1);
    final JobVertex sinkVertex = ExecutionGraphTestUtils.createNoOpVertex(1);
    operatorVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
    sinkVertex.connectNewDataSetAsInput(operatorVertex, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
    // setup partition tracker to intercept partition release calls
    final TestingJobMasterPartitionTracker partitionTracker = new TestingJobMasterPartitionTracker();
    final Queue<ResultPartitionID> releasedPartitions = new ArrayDeque<>();
    partitionTracker.setStopTrackingAndReleasePartitionsConsumer(partitionIds -> releasedPartitions.add(partitionIds.iterator().next()));
    final SchedulerBase scheduler = createScheduler(partitionTracker, sourceVertex, operatorVertex, sinkVertex);
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    // finish vertices one after another, and verify that the appropriate partitions are
    // released
    mainThreadExecutor.execute(() -> {
        final Execution sourceExecution = getCurrentExecution(sourceVertex, executionGraph);
        scheduler.updateTaskExecutionState(new TaskExecutionState(sourceExecution.getAttemptId(), ExecutionState.FINISHED));
        assertThat(releasedPartitions, empty());
    });
    mainThreadExecutor.execute(() -> {
        final Execution sourceExecution = getCurrentExecution(sourceVertex, executionGraph);
        final Execution operatorExecution = getCurrentExecution(operatorVertex, executionGraph);
        scheduler.updateTaskExecutionState(new TaskExecutionState(operatorExecution.getAttemptId(), ExecutionState.FINISHED));
        assertThat(releasedPartitions, hasSize(1));
        assertThat(releasedPartitions.remove(), equalTo(new ResultPartitionID(sourceExecution.getVertex().getProducedPartitions().keySet().iterator().next(), sourceExecution.getAttemptId())));
    });
    mainThreadExecutor.execute(() -> {
        final Execution operatorExecution = getCurrentExecution(operatorVertex, executionGraph);
        final Execution sinkExecution = getCurrentExecution(sinkVertex, executionGraph);
        scheduler.updateTaskExecutionState(new TaskExecutionState(sinkExecution.getAttemptId(), ExecutionState.FINISHED));
        assertThat(releasedPartitions, hasSize(1));
        assertThat(releasedPartitions.remove(), equalTo(new ResultPartitionID(operatorExecution.getVertex().getProducedPartitions().keySet().iterator().next(), operatorExecution.getAttemptId())));
    });
}

Example 29 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphPartitionReleaseTest method createScheduler.

private SchedulerBase createScheduler(final JobMasterPartitionTracker partitionTracker, final JobVertex... vertices) throws Exception {
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(vertices);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, mainThreadExecutor.getMainThreadExecutor()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory()).setPartitionTracker(partitionTracker).build();
    mainThreadExecutor.execute(scheduler::startScheduling);
    return scheduler;
}

Example 30 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method setupScheduler.

private SchedulerBase setupScheduler(JobVertex v1, int dop1, JobVertex v2, int dop2) throws Exception {
    v1.setParallelism(dop1);
    v2.setParallelism(dop2);
    v1.setInvokableClass(BatchTask.class);
    v2.setInvokableClass(BatchTask.class);
    DirectScheduledExecutorService executorService = new DirectScheduledExecutorService();
    // execution graph that executes actions synchronously
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.streamingJobGraph(v1, v2), ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory()).setFutureExecutor(executorService).setBlobWriter(blobWriter).build();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    checkJobOffloaded((DefaultExecutionGraph) eg);
    // schedule, this triggers mock deployment
    scheduler.startScheduling();
    Map<ExecutionAttemptID, Execution> executions = eg.getRegisteredExecutions();
    assertEquals(dop1 + dop2, executions.size());
    return scheduler;
}