Examples with DefaultScheduler org.apache.flink.runtime.scheduler.DefaultScheduler used on opensource projects

Example 1 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class RemoveCachedShuffleDescriptorTest method createSchedulerAndDeploy.

private DefaultScheduler createSchedulerAndDeploy(JobID jobId, JobVertex v1, JobVertex v2, DistributionPattern distributionPattern, BlobWriter blobWriter) throws Exception {
    v2.connectNewDataSetAsInput(v1, distributionPattern, ResultPartitionType.BLOCKING);
    final List<JobVertex> ordered = new ArrayList<>(Arrays.asList(v1, v2));
    final DefaultScheduler scheduler = createScheduler(jobId, ordered, blobWriter, mainThreadExecutor, ioExecutor);
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    final TestingLogicalSlotBuilder slotBuilder = new TestingLogicalSlotBuilder();
    CompletableFuture.runAsync(() -> {
        try {
            // Deploy upstream source vertices
            deployTasks(executionGraph, v1.getID(), slotBuilder);
            // Transition upstream vertices into FINISHED
            transitionTasksToFinished(executionGraph, v1.getID());
            // Deploy downstream sink vertices
            deployTasks(executionGraph, v2.getID(), slotBuilder);
        } catch (Exception e) {
            throw new RuntimeException("Exceptions shouldn't happen here.", e);
        }
    }, mainThreadExecutor).join();
    return scheduler;
}

Example 2 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class RemoveCachedShuffleDescriptorTest method testRemoveCacheForAllToAllEdgeAfterFinished.

private void testRemoveCacheForAllToAllEdgeAfterFinished(TestingBlobWriter blobWriter, int expectedBefore, int expectedAfter) throws Exception {
    final JobID jobId = new JobID();
    final JobVertex v1 = ExecutionGraphTestUtils.createNoOpVertex("v1", PARALLELISM);
    final JobVertex v2 = ExecutionGraphTestUtils.createNoOpVertex("v2", PARALLELISM);
    final DefaultScheduler scheduler = createSchedulerAndDeploy(jobId, v1, v2, DistributionPattern.ALL_TO_ALL, blobWriter);
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    // ShuffleDescriptors should be cached during the deployment
    final ShuffleDescriptor[] shuffleDescriptors = deserializeShuffleDescriptors(getConsumedCachedShuffleDescriptor(executionGraph, v2), jobId, blobWriter);
    assertEquals(PARALLELISM, shuffleDescriptors.length);
    assertEquals(expectedBefore, blobWriter.numberOfBlobs());
    // For the all-to-all edge, we transition all downstream tasks to finished
    CompletableFuture.runAsync(() -> transitionTasksToFinished(executionGraph, v2.getID()), mainThreadExecutor).join();
    ioExecutor.triggerAll();
    // Cache should be removed since partitions are released
    assertNull(getConsumedCachedShuffleDescriptor(executionGraph, v2));
    assertEquals(expectedAfter, blobWriter.numberOfBlobs());
}

Example 3 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class SchedulerBenchmarkUtils method createAndInitExecutionGraph.

public static ExecutionGraph createAndInitExecutionGraph(List<JobVertex> jobVertices, JobConfiguration jobConfiguration, ScheduledExecutorService scheduledExecutorService) throws Exception {
    final JobGraph jobGraph = createJobGraph(jobVertices, jobConfiguration);
    final ComponentMainThreadExecutor mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
    final DefaultScheduler scheduler = SchedulerTestingUtils.createSchedulerBuilder(jobGraph, mainThreadExecutor).setIoExecutor(scheduledExecutorService).setFutureExecutor(scheduledExecutorService).setDelayExecutor(new ScheduledExecutorServiceAdapter(scheduledExecutorService)).build();
    return scheduler.getExecutionGraph();
}

Example 4 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class OperatorCoordinatorSchedulerTest method testFailureToStartPropagatesExceptions.

@Test
public void testFailureToStartPropagatesExceptions() throws Exception {
    final OperatorCoordinator.Provider failingCoordinatorProvider = new TestingOperatorCoordinator.Provider(testOperatorId, CoordinatorThatFailsInStart::new);
    final DefaultScheduler scheduler = createScheduler(failingCoordinatorProvider);
    try {
        scheduler.startScheduling();
        fail("expected an exception");
    } catch (Exception ignored) {
    // expected
    }
}

Example 5 with DefaultScheduler

use of org.apache.flink.runtime.scheduler.DefaultScheduler in project flink by apache.

the class OperatorCoordinatorSchedulerTest method testBatchGlobalFailureResetsToEmptyState.

// ------------------------------------------------------------------------
// tests for failover notifications in a batch setup (no checkpoints)
// ------------------------------------------------------------------------
@Test
public void testBatchGlobalFailureResetsToEmptyState() throws Exception {
    final DefaultScheduler scheduler = createSchedulerWithoutCheckpointingAndDeployTasks();
    final TestingOperatorCoordinator coordinator = getCoordinator(scheduler);
    failGlobalAndRestart(scheduler, new TestException());
    assertSame("coordinator should have null restored state", TestingOperatorCoordinator.NULL_RESTORE_VALUE, coordinator.getLastRestoredCheckpointState());
    assertEquals(OperatorCoordinator.NO_CHECKPOINT, coordinator.getLastRestoredCheckpointId());
}