use of org.apache.flink.runtime.scheduler.TestingPhysicalSlotProvider in project flink by apache.
the class DefaultExecutionGraphDeploymentTest method testExecutionGraphIsDeployedInTopologicalOrder.
/**
* Tests that the {@link ExecutionGraph} is deployed in topological order.
*/
@Test
public void testExecutionGraphIsDeployedInTopologicalOrder() throws Exception {
final int sourceParallelism = 2;
final int sinkParallelism = 1;
final JobVertex sourceVertex = new JobVertex("source");
sourceVertex.setInvokableClass(NoOpInvokable.class);
sourceVertex.setParallelism(sourceParallelism);
final JobVertex sinkVertex = new JobVertex("sink");
sinkVertex.setInvokableClass(NoOpInvokable.class);
sinkVertex.setParallelism(sinkParallelism);
sinkVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
final int numberTasks = sourceParallelism + sinkParallelism;
final ArrayBlockingQueue<ExecutionAttemptID> submittedTasksQueue = new ArrayBlockingQueue<>(numberTasks);
TestingTaskExecutorGatewayBuilder testingTaskExecutorGatewayBuilder = new TestingTaskExecutorGatewayBuilder();
testingTaskExecutorGatewayBuilder.setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> {
submittedTasksQueue.offer(taskDeploymentDescriptor.getExecutionAttemptId());
return CompletableFuture.completedFuture(Acknowledge.get());
});
final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
final TestingTaskExecutorGateway taskExecutorGateway = testingTaskExecutorGatewayBuilder.createTestingTaskExecutorGateway();
final RpcTaskManagerGateway taskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, JobMasterId.generate());
final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sourceVertex, sinkVertex);
final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithoutImmediatePhysicalSlotCreation();
final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).setFutureExecutor(new DirectScheduledExecutorService()).build();
final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
scheduler.startScheduling();
// change the order in which the futures are completed
final List<CompletableFuture<TestingPhysicalSlot>> shuffledFutures = new ArrayList<>(physicalSlotProvider.getResponses().values());
Collections.shuffle(shuffledFutures);
for (CompletableFuture<TestingPhysicalSlot> slotFuture : shuffledFutures) {
slotFuture.complete(TestingPhysicalSlot.builder().withTaskManagerLocation(taskManagerLocation).withTaskManagerGateway(taskManagerGateway).build());
}
final List<ExecutionAttemptID> submittedTasks = new ArrayList<>(numberTasks);
for (int i = 0; i < numberTasks; i++) {
submittedTasks.add(submittedTasksQueue.take());
}
final Collection<ExecutionAttemptID> firstStage = new ArrayList<>(sourceParallelism);
for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sourceVertex.getID()).getTaskVertices()) {
firstStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
}
final Collection<ExecutionAttemptID> secondStage = new ArrayList<>(sinkParallelism);
for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sinkVertex.getID()).getTaskVertices()) {
secondStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
}
assertThat(submittedTasks, new ExecutionStageMatcher(Arrays.asList(firstStage, secondStage)));
}
Aggregations