Examples with SchedulerBase org.apache.flink.runtime.scheduler.SchedulerBase used on opensource projects

Example 41 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionGraphVariousFailuesTest method testFailingNotifyPartitionDataAvailable.

/**
 * Tests that a failing notifyPartitionDataAvailable call with a non-existing execution attempt
 * id, will not fail the execution graph.
 */
@Test
public void testFailingNotifyPartitionDataAvailable() throws Exception {
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.emptyJobGraph(), ComponentMainThreadExecutorServiceAdapter.forMainThread()).build();
    scheduler.startScheduling();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    assertEquals(JobStatus.RUNNING, eg.getState());
    ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
    IntermediateResultPartitionID intermediateResultPartitionId = new IntermediateResultPartitionID();
    ExecutionAttemptID producerId = new ExecutionAttemptID();
    ResultPartitionID resultPartitionId = new ResultPartitionID(intermediateResultPartitionId, producerId);
    try {
        scheduler.notifyPartitionDataAvailable(resultPartitionId);
        fail("Error expected.");
    } catch (IllegalStateException e) {
        // we've expected this exception to occur
        assertThat(e.getMessage(), containsString("Cannot find execution for execution Id"));
    }
    assertEquals(JobStatus.RUNNING, eg.getState());
}

Example 42 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionVertexTest method testResetForNewExecutionReleasesPartitions.

@Test
public void testResetForNewExecutionReleasesPartitions() throws Exception {
    final JobVertex producerJobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);
    final JobVertex consumerJobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);
    consumerJobVertex.connectNewDataSetAsInput(producerJobVertex, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
    final CompletableFuture<Collection<ResultPartitionID>> releasePartitionsFuture = new CompletableFuture<>();
    final TestingJobMasterPartitionTracker partitionTracker = new TestingJobMasterPartitionTracker();
    partitionTracker.setStopTrackingAndReleasePartitionsConsumer(releasePartitionsFuture::complete);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(producerJobVertex, consumerJobVertex);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setPartitionTracker(partitionTracker).build();
    scheduler.startScheduling();
    final ExecutionJobVertex producerExecutionJobVertex = scheduler.getExecutionJobVertex(producerJobVertex.getID());
    Execution execution = producerExecutionJobVertex.getTaskVertices()[0].getCurrentExecutionAttempt();
    assertFalse(releasePartitionsFuture.isDone());
    execution.markFinished();
    assertFalse(releasePartitionsFuture.isDone());
    for (ExecutionVertex executionVertex : producerExecutionJobVertex.getTaskVertices()) {
        executionVertex.resetForNewExecution();
    }
    final IntermediateResultPartitionID intermediateResultPartitionID = producerExecutionJobVertex.getProducedDataSets()[0].getPartitions()[0].getPartitionId();
    final ResultPartitionID resultPartitionID = execution.getResultPartitionDeploymentDescriptor(intermediateResultPartitionID).get().getShuffleDescriptor().getResultPartitionID();
    assertThat(releasePartitionsFuture.get()).contains(resultPartitionID);
}

Example 43 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class IntermediateResultPartitionTest method createResult.

private static IntermediateResult createResult(ResultPartitionType resultPartitionType, int parallelism) throws Exception {
    JobVertex source = new JobVertex("v1");
    source.setInvokableClass(NoOpInvokable.class);
    source.setParallelism(parallelism);
    JobVertex sink = new JobVertex("v2");
    sink.setInvokableClass(NoOpInvokable.class);
    sink.setParallelism(parallelism);
    sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, resultPartitionType);
    ScheduledExecutorService executorService = new DirectScheduledExecutorService();
    JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(source, sink);
    SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setIoExecutor(executorService).setFutureExecutor(executorService).build();
    ExecutionJobVertex ejv = scheduler.getExecutionJobVertex(source.getID());
    return ejv.getProducedDataSets()[0];
}

Example 44 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class ExecutionVertexCancelTest method testSendCancelAndReceiveFail.

@Test
public void testSendCancelAndReceiveFail() throws Exception {
    final SchedulerBase scheduler = SchedulerTestingUtils.createScheduler(JobGraphTestUtils.streamingJobGraph(createNoOpVertex(10)), ComponentMainThreadExecutorServiceAdapter.forMainThread());
    final ExecutionGraph graph = scheduler.getExecutionGraph();
    scheduler.startScheduling();
    ExecutionGraphTestUtils.switchAllVerticesToRunning(graph);
    assertEquals(JobStatus.RUNNING, graph.getState());
    final ExecutionVertex[] vertices = graph.getVerticesTopologically().iterator().next().getTaskVertices();
    assertEquals(vertices.length, graph.getRegisteredExecutions().size());
    final Execution exec = vertices[3].getCurrentExecutionAttempt();
    exec.cancel();
    assertEquals(ExecutionState.CANCELING, exec.getState());
    exec.markFailed(new Exception("test"));
    assertTrue(exec.getState() == ExecutionState.FAILED || exec.getState() == ExecutionState.CANCELED);
    assertFalse(exec.getAssignedResource().isAlive());
    assertEquals(vertices.length - 1, graph.getRegisteredExecutions().size());
}

Example 45 with SchedulerBase

use of org.apache.flink.runtime.scheduler.SchedulerBase in project flink by apache.

the class UpdatePartitionConsumersTest method testUpdatePartitionConsumers.

/**
 * Test BLOCKING partition information are properly updated to consumers when its producer
 * finishes.
 */
@Test
public void testUpdatePartitionConsumers() throws Exception {
    final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(new TestExecutionSlotAllocatorFactory(taskManagerGateway)).build();
    final ExecutionVertex ev1 = scheduler.getExecutionVertex(new ExecutionVertexID(v1.getID(), 0));
    final ExecutionVertex ev2 = scheduler.getExecutionVertex(new ExecutionVertexID(v2.getID(), 0));
    final ExecutionVertex ev3 = scheduler.getExecutionVertex(new ExecutionVertexID(v3.getID(), 0));
    final ExecutionVertex ev4 = scheduler.getExecutionVertex(new ExecutionVertexID(v4.getID(), 0));
    final CompletableFuture<TaskDeploymentDescriptor> ev4TddFuture = new CompletableFuture<>();
    taskManagerGateway.setSubmitConsumer(tdd -> {
        if (tdd.getExecutionAttemptId().equals(ev4.getCurrentExecutionAttempt().getAttemptId())) {
            ev4TddFuture.complete(tdd);
        }
    });
    scheduler.startScheduling();
    assertThat(ev1.getExecutionState(), is(ExecutionState.DEPLOYING));
    assertThat(ev2.getExecutionState(), is(ExecutionState.DEPLOYING));
    assertThat(ev3.getExecutionState(), is(ExecutionState.DEPLOYING));
    assertThat(ev4.getExecutionState(), is(ExecutionState.DEPLOYING));
    updateState(scheduler, ev1, ExecutionState.INITIALIZING);
    updateState(scheduler, ev1, ExecutionState.RUNNING);
    updateState(scheduler, ev2, ExecutionState.INITIALIZING);
    updateState(scheduler, ev2, ExecutionState.RUNNING);
    updateState(scheduler, ev3, ExecutionState.INITIALIZING);
    updateState(scheduler, ev3, ExecutionState.RUNNING);
    updateState(scheduler, ev4, ExecutionState.INITIALIZING);
    updateState(scheduler, ev4, ExecutionState.RUNNING);
    final InputGateDeploymentDescriptor ev4Igdd2 = ev4TddFuture.get(TIMEOUT, TimeUnit.MILLISECONDS).getInputGates().get(1);
    assertThat(ev4Igdd2.getShuffleDescriptors()[0], instanceOf(UnknownShuffleDescriptor.class));
    final CompletableFuture<Void> updatePartitionFuture = new CompletableFuture<>();
    taskManagerGateway.setUpdatePartitionsConsumer((attemptId, partitionInfos, time) -> {
        assertThat(attemptId, equalTo(ev4.getCurrentExecutionAttempt().getAttemptId()));
        final List<PartitionInfo> partitionInfoList = IterableUtils.toStream(partitionInfos).collect(Collectors.toList());
        assertThat(partitionInfoList, hasSize(1));
        final PartitionInfo partitionInfo = partitionInfoList.get(0);
        assertThat(partitionInfo.getIntermediateDataSetID(), equalTo(v3.getProducedDataSets().get(0).getId()));
        assertThat(partitionInfo.getShuffleDescriptor(), instanceOf(NettyShuffleDescriptor.class));
        updatePartitionFuture.complete(null);
    });
    updateState(scheduler, ev1, ExecutionState.FINISHED);
    updateState(scheduler, ev3, ExecutionState.FINISHED);
    updatePartitionFuture.get(TIMEOUT, TimeUnit.MILLISECONDS);
}