Examples with DirectScheduledExecutorService org.apache.flink.runtime.testutils.DirectScheduledExecutorService used on opensource projects

Example 41 with DirectScheduledExecutorService

use of org.apache.flink.runtime.testutils.DirectScheduledExecutorService in project flink-mirror by flink-ci.

the class CheckpointCoordinatorTest method testMinCheckpointPause.

@Test
public void testMinCheckpointPause() throws Exception {
    // will use a different thread to allow checkpoint triggering before exiting from
    // receiveAcknowledgeMessage
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    CheckpointCoordinator coordinator = null;
    try {
        int pause = 1000;
        JobVertexID jobVertexId = new JobVertexID();
        ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().addJobVertex(jobVertexId).setMainThreadExecutor(ComponentMainThreadExecutorServiceAdapter.forSingleThreadExecutor(new DirectScheduledExecutorService())).build();
        ExecutionVertex vertex = graph.getJobVertex(jobVertexId).getTaskVertices()[0];
        ExecutionAttemptID attemptId = vertex.getCurrentExecutionAttempt().getAttemptId();
        coordinator = new CheckpointCoordinatorBuilder().setTimer(new ScheduledExecutorServiceAdapter(executorService)).setCheckpointCoordinatorConfiguration(CheckpointCoordinatorConfiguration.builder().setCheckpointInterval(pause).setCheckpointTimeout(Long.MAX_VALUE).setMaxConcurrentCheckpoints(1).setMinPauseBetweenCheckpoints(pause).build()).setExecutionGraph(graph).build();
        coordinator.startCheckpointScheduler();
        coordinator.triggerCheckpoint(// trigger, execute, and later complete by receiveAcknowledgeMessage
        true);
        coordinator.triggerCheckpoint(// enqueue and later see if it gets executed in the middle of
        true);
        // receiveAcknowledgeMessage
        while (coordinator.getNumberOfPendingCheckpoints() == 0) {
            // wait for at least 1 request to be fully processed
            Thread.sleep(10);
        }
        coordinator.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(graph.getJobID(), attemptId, 1L), TASK_MANAGER_LOCATION_INFO);
        Thread.sleep(pause / 2);
        assertEquals(0, coordinator.getNumberOfPendingCheckpoints());
        // make sure that the 2nd request is eventually processed
        while (coordinator.getNumberOfPendingCheckpoints() == 0) {
            Thread.sleep(1);
        }
    } finally {
        if (coordinator != null) {
            coordinator.shutdown();
        }
        executorService.shutdownNow();
    }
}

Example 42 with DirectScheduledExecutorService

use of org.apache.flink.runtime.testutils.DirectScheduledExecutorService in project flink-mirror by flink-ci.

the class DefaultExecutionGraphDeploymentTest method testExecutionGraphIsDeployedInTopologicalOrder.

/**
 * Tests that the {@link ExecutionGraph} is deployed in topological order.
 */
@Test
public void testExecutionGraphIsDeployedInTopologicalOrder() throws Exception {
    final int sourceParallelism = 2;
    final int sinkParallelism = 1;
    final JobVertex sourceVertex = new JobVertex("source");
    sourceVertex.setInvokableClass(NoOpInvokable.class);
    sourceVertex.setParallelism(sourceParallelism);
    final JobVertex sinkVertex = new JobVertex("sink");
    sinkVertex.setInvokableClass(NoOpInvokable.class);
    sinkVertex.setParallelism(sinkParallelism);
    sinkVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    final int numberTasks = sourceParallelism + sinkParallelism;
    final ArrayBlockingQueue<ExecutionAttemptID> submittedTasksQueue = new ArrayBlockingQueue<>(numberTasks);
    TestingTaskExecutorGatewayBuilder testingTaskExecutorGatewayBuilder = new TestingTaskExecutorGatewayBuilder();
    testingTaskExecutorGatewayBuilder.setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> {
        submittedTasksQueue.offer(taskDeploymentDescriptor.getExecutionAttemptId());
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingTaskExecutorGateway taskExecutorGateway = testingTaskExecutorGatewayBuilder.createTestingTaskExecutorGateway();
    final RpcTaskManagerGateway taskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, JobMasterId.generate());
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sourceVertex, sinkVertex);
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithoutImmediatePhysicalSlotCreation();
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).setFutureExecutor(new DirectScheduledExecutorService()).build();
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    scheduler.startScheduling();
    // change the order in which the futures are completed
    final List<CompletableFuture<TestingPhysicalSlot>> shuffledFutures = new ArrayList<>(physicalSlotProvider.getResponses().values());
    Collections.shuffle(shuffledFutures);
    for (CompletableFuture<TestingPhysicalSlot> slotFuture : shuffledFutures) {
        slotFuture.complete(TestingPhysicalSlot.builder().withTaskManagerLocation(taskManagerLocation).withTaskManagerGateway(taskManagerGateway).build());
    }
    final List<ExecutionAttemptID> submittedTasks = new ArrayList<>(numberTasks);
    for (int i = 0; i < numberTasks; i++) {
        submittedTasks.add(submittedTasksQueue.take());
    }
    final Collection<ExecutionAttemptID> firstStage = new ArrayList<>(sourceParallelism);
    for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sourceVertex.getID()).getTaskVertices()) {
        firstStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
    }
    final Collection<ExecutionAttemptID> secondStage = new ArrayList<>(sinkParallelism);
    for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sinkVertex.getID()).getTaskVertices()) {
        secondStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
    }
    assertThat(submittedTasks, new ExecutionStageMatcher(Arrays.asList(firstStage, secondStage)));
}

Example 43 with DirectScheduledExecutorService

use of org.apache.flink.runtime.testutils.DirectScheduledExecutorService in project flink-mirror by flink-ci.

the class DefaultExecutionGraphDeploymentTest method testNoResourceAvailableFailure.

/**
 * Tests that a blocking batch job fails if there are not enough resources left to schedule the
 * succeeding tasks. This test case is related to [FLINK-4296] where finished producing tasks
 * swallow the fail exception when scheduling a consumer task.
 */
@Test
public void testNoResourceAvailableFailure() throws Exception {
    JobVertex v1 = new JobVertex("source");
    JobVertex v2 = new JobVertex("sink");
    int dop1 = 2;
    int dop2 = 2;
    v1.setParallelism(dop1);
    v2.setParallelism(dop2);
    v1.setInvokableClass(BatchTask.class);
    v2.setInvokableClass(BatchTask.class);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
    final JobGraph graph = JobGraphTestUtils.batchJobGraph(v1, v2);
    DirectScheduledExecutorService directExecutor = new DirectScheduledExecutorService();
    // execution graph that executes actions synchronously
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(graph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(TestingPhysicalSlotProvider.createWithLimitedAmountOfPhysicalSlots(1))).setFutureExecutor(directExecutor).setBlobWriter(blobWriter).build();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    checkJobOffloaded((DefaultExecutionGraph) eg);
    // schedule, this triggers mock deployment
    scheduler.startScheduling();
    ExecutionAttemptID attemptID = eg.getJobVertex(v1.getID()).getTaskVertices()[0].getCurrentExecutionAttempt().getAttemptId();
    scheduler.updateTaskExecutionState(new TaskExecutionState(attemptID, ExecutionState.RUNNING));
    scheduler.updateTaskExecutionState(new TaskExecutionState(attemptID, ExecutionState.FINISHED, null));
    assertEquals(JobStatus.FAILED, eg.getState());
}

Example 44 with DirectScheduledExecutorService

use of org.apache.flink.runtime.testutils.DirectScheduledExecutorService in project flink-mirror by flink-ci.

the class DefaultExecutionGraphDeploymentTest method testBuildDeploymentDescriptor.

@Test
public void testBuildDeploymentDescriptor() throws Exception {
    final JobVertexID jid1 = new JobVertexID();
    final JobVertexID jid2 = new JobVertexID();
    final JobVertexID jid3 = new JobVertexID();
    final JobVertexID jid4 = new JobVertexID();
    JobVertex v1 = new JobVertex("v1", jid1);
    JobVertex v2 = new JobVertex("v2", jid2);
    JobVertex v3 = new JobVertex("v3", jid3);
    JobVertex v4 = new JobVertex("v4", jid4);
    v1.setParallelism(10);
    v2.setParallelism(10);
    v3.setParallelism(10);
    v4.setParallelism(10);
    v1.setInvokableClass(BatchTask.class);
    v2.setInvokableClass(BatchTask.class);
    v3.setInvokableClass(BatchTask.class);
    v4.setInvokableClass(BatchTask.class);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(v1, v2, v3, v4);
    final JobID jobId = jobGraph.getJobID();
    DirectScheduledExecutorService executor = new DirectScheduledExecutorService();
    DefaultExecutionGraph eg = TestingDefaultExecutionGraphBuilder.newBuilder().setJobGraph(jobGraph).setFutureExecutor(executor).setIoExecutor(executor).setBlobWriter(blobWriter).build();
    eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
    checkJobOffloaded(eg);
    ExecutionJobVertex ejv = eg.getAllVertices().get(jid2);
    ExecutionVertex vertex = ejv.getTaskVertices()[3];
    final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    final CompletableFuture<TaskDeploymentDescriptor> tdd = new CompletableFuture<>();
    taskManagerGateway.setSubmitConsumer(FunctionUtils.uncheckedConsumer(taskDeploymentDescriptor -> {
        taskDeploymentDescriptor.loadBigData(blobCache);
        tdd.complete(taskDeploymentDescriptor);
    }));
    final LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway).createTestingLogicalSlot();
    assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
    vertex.getCurrentExecutionAttempt().transitionState(ExecutionState.SCHEDULED);
    vertex.getCurrentExecutionAttempt().registerProducedPartitions(slot.getTaskManagerLocation(), true).get();
    vertex.deployToSlot(slot);
    assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
    checkTaskOffloaded(eg, vertex.getJobvertexId());
    TaskDeploymentDescriptor descr = tdd.get();
    assertNotNull(descr);
    JobInformation jobInformation = descr.getSerializedJobInformation().deserializeValue(getClass().getClassLoader());
    TaskInformation taskInformation = descr.getSerializedTaskInformation().deserializeValue(getClass().getClassLoader());
    assertEquals(jobId, descr.getJobId());
    assertEquals(jobId, jobInformation.getJobId());
    assertEquals(jid2, taskInformation.getJobVertexId());
    assertEquals(3, descr.getSubtaskIndex());
    assertEquals(10, taskInformation.getNumberOfSubtasks());
    assertEquals(BatchTask.class.getName(), taskInformation.getInvokableClassName());
    assertEquals("v2", taskInformation.getTaskName());
    Collection<ResultPartitionDeploymentDescriptor> producedPartitions = descr.getProducedPartitions();
    Collection<InputGateDeploymentDescriptor> consumedPartitions = descr.getInputGates();
    assertEquals(2, producedPartitions.size());
    assertEquals(1, consumedPartitions.size());
    Iterator<ResultPartitionDeploymentDescriptor> iteratorProducedPartitions = producedPartitions.iterator();
    Iterator<InputGateDeploymentDescriptor> iteratorConsumedPartitions = consumedPartitions.iterator();
    assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
    assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
    ShuffleDescriptor[] shuffleDescriptors = iteratorConsumedPartitions.next().getShuffleDescriptors();
    assertEquals(10, shuffleDescriptors.length);
    Iterator<ConsumedPartitionGroup> iteratorConsumedPartitionGroup = vertex.getAllConsumedPartitionGroups().iterator();
    int idx = 0;
    for (IntermediateResultPartitionID partitionId : iteratorConsumedPartitionGroup.next()) {
        assertEquals(partitionId, shuffleDescriptors[idx++].getResultPartitionID().getPartitionId());
    }
}

Example 45 with DirectScheduledExecutorService

use of org.apache.flink.runtime.testutils.DirectScheduledExecutorService in project flink-mirror by flink-ci.

the class DefaultExecutionGraphDeploymentTest method setupScheduler.

private SchedulerBase setupScheduler(JobVertex v1, int dop1, JobVertex v2, int dop2) throws Exception {
    v1.setParallelism(dop1);
    v2.setParallelism(dop2);
    v1.setInvokableClass(BatchTask.class);
    v2.setInvokableClass(BatchTask.class);
    DirectScheduledExecutorService executorService = new DirectScheduledExecutorService();
    // execution graph that executes actions synchronously
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.streamingJobGraph(v1, v2), ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory()).setFutureExecutor(executorService).setBlobWriter(blobWriter).build();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    checkJobOffloaded((DefaultExecutionGraph) eg);
    // schedule, this triggers mock deployment
    scheduler.startScheduling();
    Map<ExecutionAttemptID, Execution> executions = eg.getRegisteredExecutions();
    assertEquals(dop1 + dop2, executions.size());
    return scheduler;
}