Examples with SchedulerBase org.apache.flink.runtime.scheduler.SchedulerBase used on opensource projects

Example 6 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphTestUtils method getExecutionJobVertex.

public static ExecutionJobVertex getExecutionJobVertex(JobVertex jobVertex, ScheduledExecutorService executor) throws Exception {
    JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(jobVertex);
    SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setIoExecutor(executor).setFutureExecutor(executor).build();
    return scheduler.getExecutionJobVertex(jobVertex.getID());
}

Example 7 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionPartitionLifecycleTest method setupExecutionGraphAndStartRunningJob.

private void setupExecutionGraphAndStartRunningJob(ResultPartitionType resultPartitionType, JobMasterPartitionTracker partitionTracker, TaskManagerGateway taskManagerGateway, ShuffleMaster<?> shuffleMaster) throws Exception {
    final JobVertex producerVertex = createNoOpJobVertex();
    final JobVertex consumerVertex = createNoOpJobVertex();
    consumerVertex.connectNewDataSetAsInput(producerVertex, DistributionPattern.ALL_TO_ALL, resultPartitionType);
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.create((resourceProfile) -> CompletableFuture.completedFuture(TestingPhysicalSlot.builder().withTaskManagerGateway(taskManagerGateway).withTaskManagerLocation(taskManagerLocation).build()));
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(producerVertex, consumerVertex);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).setShuffleMaster(shuffleMaster).setPartitionTracker(partitionTracker).build();
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(producerVertex.getID());
    final ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];
    execution = executionVertex.getCurrentExecutionAttempt();
    scheduler.startScheduling();
    execution.switchToRecovering();
    execution.switchToRunning();
    final IntermediateResultPartitionID expectedIntermediateResultPartitionId = executionJobVertex.getProducedDataSets()[0].getPartitions()[0].getPartitionId();
    descriptor = execution.getResultPartitionDeploymentDescriptor(expectedIntermediateResultPartitionId).get();
    taskExecutorResourceId = taskManagerLocation.getResourceID();
    jobId = executionGraph.getJobID();
}

Example 8 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionTest method testTerminationFutureIsCompletedAfterSlotRelease.

/**
 * Checks that the {@link Execution} termination future is only completed after the assigned
 * slot has been released.
 *
 * <p>NOTE: This test only fails spuriously without the fix of this commit. Thus, one has to
 * execute this test multiple times to see the failure.
 */
@Test
public void testTerminationFutureIsCompletedAfterSlotRelease() throws Exception {
    final JobVertex jobVertex = createNoOpJobVertex();
    final JobVertexID jobVertexId = jobVertex.getID();
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithLimitedAmountOfPhysicalSlots(1);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.streamingJobGraph(jobVertex), ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).build();
    ExecutionJobVertex executionJobVertex = scheduler.getExecutionJobVertex(jobVertexId);
    ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];
    scheduler.startScheduling();
    Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
    CompletableFuture<? extends PhysicalSlot> returnedSlotFuture = physicalSlotProvider.getFirstResponseOrFail();
    CompletableFuture<?> terminationFuture = executionVertex.cancel();
    currentExecutionAttempt.completeCancelling();
    CompletableFuture<Boolean> restartFuture = terminationFuture.thenApply(ignored -> {
        assertTrue(returnedSlotFuture.isDone());
        return true;
    });
    // check if the returned slot future was completed first
    restartFuture.get();
}

Example 9 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class FinalizeOnMasterTest method testFinalizeIsNotCalledUponFailure.

@Test
public void testFinalizeIsNotCalledUponFailure() throws Exception {
    final JobVertex vertex = spy(new JobVertex("test vertex 1"));
    vertex.setInvokableClass(NoOpInvokable.class);
    vertex.setParallelism(1);
    final SchedulerBase scheduler = createScheduler(JobGraphTestUtils.streamingJobGraph(vertex), ComponentMainThreadExecutorServiceAdapter.forMainThread());
    scheduler.startScheduling();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    assertEquals(JobStatus.RUNNING, eg.getState());
    ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
    // fail the execution
    final Execution exec = eg.getJobVertex(vertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();
    exec.fail(new Exception("test"));
    assertEquals(JobStatus.FAILED, eg.waitUntilTerminal());
    verify(vertex, times(0)).finalizeOnMaster(any(ClassLoader.class));
    assertEquals(0, eg.getRegisteredExecutions().size());
}

Example 10 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionVertexTest method testFindLatestAllocationIgnoresFailedAttempts.

@Test
public void testFindLatestAllocationIgnoresFailedAttempts() throws Exception {
    final JobVertex source = ExecutionGraphTestUtils.createNoOpVertex(1);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(source);
    final TestingPhysicalSlotProvider withLimitedAmountOfPhysicalSlots = TestingPhysicalSlotProvider.createWithLimitedAmountOfPhysicalSlots(1);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(withLimitedAmountOfPhysicalSlots)).build();
    scheduler.startScheduling();
    final ExecutionJobVertex sourceExecutionJobVertex = scheduler.getExecutionJobVertex(source.getID());
    final ExecutionVertex sourceExecutionVertex = sourceExecutionJobVertex.getTaskVertices()[0];
    final Execution firstExecution = sourceExecutionVertex.getCurrentExecutionAttempt();
    final TestingPhysicalSlot physicalSlot = withLimitedAmountOfPhysicalSlots.getFirstResponseOrFail().join();
    final AllocationID allocationId = physicalSlot.getAllocationId();
    final TaskManagerLocation taskManagerLocation = physicalSlot.getTaskManagerLocation();
    cancelExecution(firstExecution);
    sourceExecutionVertex.resetForNewExecution();
    assertThat(sourceExecutionVertex.findLatestPriorAllocation()).hasValue(allocationId);
    assertThat(sourceExecutionVertex.findLatestPriorLocation()).hasValue(taskManagerLocation);
    final Execution secondExecution = sourceExecutionVertex.getCurrentExecutionAttempt();
    cancelExecution(secondExecution);
    sourceExecutionVertex.resetForNewExecution();
    assertThat(sourceExecutionVertex.findLatestPriorAllocation()).hasValue(allocationId);
    assertThat(sourceExecutionVertex.findLatestPriorLocation()).hasValue(taskManagerLocation);
}