Examples with TestingPhysicalSlotProvider org.apache.flink.runtime.scheduler.TestingPhysicalSlotProvider used on opensource projects

Example 1 with TestingPhysicalSlotProvider

use of org.apache.flink.runtime.scheduler.TestingPhysicalSlotProvider in project flink by apache.

the class ExecutionPartitionLifecycleTest method setupExecutionGraphAndStartRunningJob.

private void setupExecutionGraphAndStartRunningJob(ResultPartitionType resultPartitionType, JobMasterPartitionTracker partitionTracker, TaskManagerGateway taskManagerGateway, ShuffleMaster<?> shuffleMaster) throws Exception {
    final JobVertex producerVertex = createNoOpJobVertex();
    final JobVertex consumerVertex = createNoOpJobVertex();
    consumerVertex.connectNewDataSetAsInput(producerVertex, DistributionPattern.ALL_TO_ALL, resultPartitionType);
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.create((resourceProfile) -> CompletableFuture.completedFuture(TestingPhysicalSlot.builder().withTaskManagerGateway(taskManagerGateway).withTaskManagerLocation(taskManagerLocation).build()));
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(producerVertex, consumerVertex);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).setShuffleMaster(shuffleMaster).setPartitionTracker(partitionTracker).build();
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(producerVertex.getID());
    final ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];
    execution = executionVertex.getCurrentExecutionAttempt();
    scheduler.startScheduling();
    execution.switchToRecovering();
    execution.switchToRunning();
    final IntermediateResultPartitionID expectedIntermediateResultPartitionId = executionJobVertex.getProducedDataSets()[0].getPartitions()[0].getPartitionId();
    descriptor = execution.getResultPartitionDeploymentDescriptor(expectedIntermediateResultPartitionId).get();
    taskExecutorResourceId = taskManagerLocation.getResourceID();
    jobId = executionGraph.getJobID();
}

Example 2 with TestingPhysicalSlotProvider

use of org.apache.flink.runtime.scheduler.TestingPhysicalSlotProvider in project flink by apache.

the class ExecutionTest method testTerminationFutureIsCompletedAfterSlotRelease.

/**
 * Checks that the {@link Execution} termination future is only completed after the assigned
 * slot has been released.
 *
 * <p>NOTE: This test only fails spuriously without the fix of this commit. Thus, one has to
 * execute this test multiple times to see the failure.
 */
@Test
public void testTerminationFutureIsCompletedAfterSlotRelease() throws Exception {
    final JobVertex jobVertex = createNoOpJobVertex();
    final JobVertexID jobVertexId = jobVertex.getID();
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithLimitedAmountOfPhysicalSlots(1);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.streamingJobGraph(jobVertex), ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).build();
    ExecutionJobVertex executionJobVertex = scheduler.getExecutionJobVertex(jobVertexId);
    ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];
    scheduler.startScheduling();
    Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
    CompletableFuture<? extends PhysicalSlot> returnedSlotFuture = physicalSlotProvider.getFirstResponseOrFail();
    CompletableFuture<?> terminationFuture = executionVertex.cancel();
    currentExecutionAttempt.completeCancelling();
    CompletableFuture<Boolean> restartFuture = terminationFuture.thenApply(ignored -> {
        assertTrue(returnedSlotFuture.isDone());
        return true;
    });
    // check if the returned slot future was completed first
    restartFuture.get();
}

Example 3 with TestingPhysicalSlotProvider

use of org.apache.flink.runtime.scheduler.TestingPhysicalSlotProvider in project flink by apache.

the class ExecutionVertexTest method testFindLatestAllocationIgnoresFailedAttempts.

@Test
public void testFindLatestAllocationIgnoresFailedAttempts() throws Exception {
    final JobVertex source = ExecutionGraphTestUtils.createNoOpVertex(1);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(source);
    final TestingPhysicalSlotProvider withLimitedAmountOfPhysicalSlots = TestingPhysicalSlotProvider.createWithLimitedAmountOfPhysicalSlots(1);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(withLimitedAmountOfPhysicalSlots)).build();
    scheduler.startScheduling();
    final ExecutionJobVertex sourceExecutionJobVertex = scheduler.getExecutionJobVertex(source.getID());
    final ExecutionVertex sourceExecutionVertex = sourceExecutionJobVertex.getTaskVertices()[0];
    final Execution firstExecution = sourceExecutionVertex.getCurrentExecutionAttempt();
    final TestingPhysicalSlot physicalSlot = withLimitedAmountOfPhysicalSlots.getFirstResponseOrFail().join();
    final AllocationID allocationId = physicalSlot.getAllocationId();
    final TaskManagerLocation taskManagerLocation = physicalSlot.getTaskManagerLocation();
    cancelExecution(firstExecution);
    sourceExecutionVertex.resetForNewExecution();
    assertThat(sourceExecutionVertex.findLatestPriorAllocation()).hasValue(allocationId);
    assertThat(sourceExecutionVertex.findLatestPriorLocation()).hasValue(taskManagerLocation);
    final Execution secondExecution = sourceExecutionVertex.getCurrentExecutionAttempt();
    cancelExecution(secondExecution);
    sourceExecutionVertex.resetForNewExecution();
    assertThat(sourceExecutionVertex.findLatestPriorAllocation()).hasValue(allocationId);
    assertThat(sourceExecutionVertex.findLatestPriorLocation()).hasValue(taskManagerLocation);
}

Example 4 with TestingPhysicalSlotProvider

use of org.apache.flink.runtime.scheduler.TestingPhysicalSlotProvider in project flink by apache.

the class ExecutionTest method testCanceledExecutionReturnsSlot.

@Test
public void testCanceledExecutionReturnsSlot() throws Exception {
    final JobVertex jobVertex = createNoOpJobVertex();
    final JobVertexID jobVertexId = jobVertex.getID();
    final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.create((resourceProfile) -> CompletableFuture.completedFuture(TestingPhysicalSlot.builder().withTaskManagerGateway(taskManagerGateway).build()));
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.streamingJobGraph(jobVertex), testMainThreadUtil.getMainThreadExecutor()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).build();
    ExecutionJobVertex executionJobVertex = scheduler.getExecutionJobVertex(jobVertexId);
    ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];
    final Execution execution = executionVertex.getCurrentExecutionAttempt();
    taskManagerGateway.setCancelConsumer(executionAttemptID -> {
        if (execution.getAttemptId().equals(executionAttemptID)) {
            execution.completeCancelling();
        }
    });
    testMainThreadUtil.execute(scheduler::startScheduling);
    // cancel the execution in case we could schedule the execution
    testMainThreadUtil.execute(execution::cancel);
    assertThat(physicalSlotProvider.getRequests().keySet(), is(physicalSlotProvider.getCancellations().keySet()));
}

Example 5 with TestingPhysicalSlotProvider

use of org.apache.flink.runtime.scheduler.TestingPhysicalSlotProvider in project flink by apache.

the class ExecutionTest method testSlotReleaseAtomicallyReleasesExecution.

/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
    final JobVertex jobVertex = createNoOpJobVertex();
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithLimitedAmountOfPhysicalSlots(1);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.streamingJobGraph(jobVertex), testMainThreadUtil.getMainThreadExecutor()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).build();
    final Execution execution = scheduler.getExecutionJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();
    testMainThreadUtil.execute(scheduler::startScheduling);
    // wait until the slot has been requested
    physicalSlotProvider.awaitAllSlotRequests();
    TestingPhysicalSlot physicalSlot = physicalSlotProvider.getFirstResponseOrFail().get();
    testMainThreadUtil.execute(() -> {
        assertThat(execution.getAssignedAllocationID(), is(physicalSlot.getAllocationId()));
        physicalSlot.releasePayload(new FlinkException("Test exception"));
        assertThat(execution.getReleaseFuture().isDone(), is(true));
    });
}