Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 81 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class OperatorIDGeneratorTest method getOperatorID.

private static OperatorID getOperatorID() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.fromElements(1, 2, 3).uid(UID).name(OPERATOR_NAME).disableChaining().addSink(new DiscardingSink<>());
    JobGraph graph = env.getStreamGraph().getJobGraph(new JobID());
    JobVertex vertex = StreamSupport.stream(graph.getVertices().spliterator(), false).filter(node -> node.getName().contains(OPERATOR_NAME)).findFirst().orElseThrow(() -> new IllegalStateException("Unable to find vertex"));
    return vertex.getOperatorIDs().get(0).getGeneratedOperatorID();
}

Example 82 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionPartitionLifecycleTest method setupExecutionGraphAndStartRunningJob.

private void setupExecutionGraphAndStartRunningJob(ResultPartitionType resultPartitionType, JobMasterPartitionTracker partitionTracker, TaskManagerGateway taskManagerGateway, ShuffleMaster<?> shuffleMaster) throws Exception {
    final JobVertex producerVertex = createNoOpJobVertex();
    final JobVertex consumerVertex = createNoOpJobVertex();
    consumerVertex.connectNewDataSetAsInput(producerVertex, DistributionPattern.ALL_TO_ALL, resultPartitionType);
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.create((resourceProfile) -> CompletableFuture.completedFuture(TestingPhysicalSlot.builder().withTaskManagerGateway(taskManagerGateway).withTaskManagerLocation(taskManagerLocation).build()));
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(producerVertex, consumerVertex);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).setShuffleMaster(shuffleMaster).setPartitionTracker(partitionTracker).build();
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(producerVertex.getID());
    final ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];
    execution = executionVertex.getCurrentExecutionAttempt();
    scheduler.startScheduling();
    execution.switchToRecovering();
    execution.switchToRunning();
    final IntermediateResultPartitionID expectedIntermediateResultPartitionId = executionJobVertex.getProducedDataSets()[0].getPartitions()[0].getPartitionId();
    descriptor = execution.getResultPartitionDeploymentDescriptor(expectedIntermediateResultPartitionId).get();
    taskExecutorResourceId = taskManagerLocation.getResourceID();
    jobId = executionGraph.getJobID();
}

Example 83 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionTest method createNoOpJobVertex.

@Nonnull
private JobVertex createNoOpJobVertex() {
    final JobVertex jobVertex = new JobVertex("Test vertex", new JobVertexID());
    jobVertex.setInvokableClass(NoOpInvokable.class);
    return jobVertex;
}

Example 84 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionTest method testTerminationFutureIsCompletedAfterSlotRelease.

/**
 * Checks that the {@link Execution} termination future is only completed after the assigned
 * slot has been released.
 *
 * <p>NOTE: This test only fails spuriously without the fix of this commit. Thus, one has to
 * execute this test multiple times to see the failure.
 */
@Test
public void testTerminationFutureIsCompletedAfterSlotRelease() throws Exception {
    final JobVertex jobVertex = createNoOpJobVertex();
    final JobVertexID jobVertexId = jobVertex.getID();
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithLimitedAmountOfPhysicalSlots(1);
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(JobGraphTestUtils.streamingJobGraph(jobVertex), ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).build();
    ExecutionJobVertex executionJobVertex = scheduler.getExecutionJobVertex(jobVertexId);
    ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];
    scheduler.startScheduling();
    Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
    CompletableFuture<? extends PhysicalSlot> returnedSlotFuture = physicalSlotProvider.getFirstResponseOrFail();
    CompletableFuture<?> terminationFuture = executionVertex.cancel();
    currentExecutionAttempt.completeCancelling();
    CompletableFuture<Boolean> restartFuture = terminationFuture.thenApply(ignored -> {
        assertTrue(returnedSlotFuture.isDone());
        return true;
    });
    // check if the returned slot future was completed first
    restartFuture.get();
}

Example 85 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class FinalizeOnMasterTest method testFinalizeIsNotCalledUponFailure.

@Test
public void testFinalizeIsNotCalledUponFailure() throws Exception {
    final JobVertex vertex = spy(new JobVertex("test vertex 1"));
    vertex.setInvokableClass(NoOpInvokable.class);
    vertex.setParallelism(1);
    final SchedulerBase scheduler = createScheduler(JobGraphTestUtils.streamingJobGraph(vertex), ComponentMainThreadExecutorServiceAdapter.forMainThread());
    scheduler.startScheduling();
    final ExecutionGraph eg = scheduler.getExecutionGraph();
    assertEquals(JobStatus.RUNNING, eg.getState());
    ExecutionGraphTestUtils.switchAllVerticesToRunning(eg);
    // fail the execution
    final Execution exec = eg.getJobVertex(vertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();
    exec.fail(new Exception("test"));
    assertEquals(JobStatus.FAILED, eg.waitUntilTerminal());
    verify(vertex, times(0)).finalizeOnMaster(any(ClassLoader.class));
    assertEquals(0, eg.getRegisteredExecutions().size());
}