Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 6 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionVertexLocalityTest method createTestGraph.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
/**
	 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
	 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {
    JobVertex source = new JobVertex("source", sourceVertexId);
    source.setParallelism(parallelism);
    source.setInvokableClass(NoOpInvokable.class);
    JobVertex target = new JobVertex("source", targetVertexId);
    target.setParallelism(parallelism);
    target.setInvokableClass(NoOpInvokable.class);
    DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
    target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);
    JobGraph testJob = new JobGraph(jobId, "test job", source, target);
    return ExecutionGraphBuilder.buildGraph(null, testJob, new Configuration(), TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), mock(SlotProvider.class), getClass().getClassLoader(), new StandaloneCheckpointRecoveryFactory(), Time.of(10, TimeUnit.SECONDS), new FixedDelayRestartStrategy(10, 0L), new UnregisteredMetricsGroup(), 1, log);
}

Example 7 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class LegacyJobVertexIdTest method testIntroduceLegacyJobVertexIds.

@Test
public void testIntroduceLegacyJobVertexIds() throws Exception {
    JobVertexID defaultId = new JobVertexID();
    JobVertexID legacyId1 = new JobVertexID();
    JobVertexID legacyId2 = new JobVertexID();
    JobVertex jobVertex = new JobVertex("test", defaultId, Arrays.asList(legacyId1, legacyId2));
    jobVertex.setInvokableClass(AbstractInvokable.class);
    ExecutionGraph executionGraph = new ExecutionGraph(mock(ScheduledExecutorService.class), mock(Executor.class), new JobID(), "test", mock(Configuration.class), mock(SerializedValue.class), Time.seconds(1), mock(RestartStrategy.class), mock(SlotProvider.class));
    ExecutionJobVertex executionJobVertex = new ExecutionJobVertex(executionGraph, jobVertex, 1, Time.seconds(1));
    Map<JobVertexID, ExecutionJobVertex> idToVertex = new HashMap<>();
    idToVertex.put(executionJobVertex.getJobVertexId(), executionJobVertex);
    Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
    Assert.assertNull(idToVertex.get(legacyId1));
    Assert.assertNull(idToVertex.get(legacyId2));
    idToVertex = ExecutionJobVertex.includeLegacyJobVertexIDs(idToVertex);
    Assert.assertEquals(3, idToVertex.size());
    Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
    Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId1));
    Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId2));
}

Example 8 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class TerminalStateDeadlockTest method testProvokeDeadlock.

// ------------------------------------------------------------------------
@Test
public void testProvokeDeadlock() {
    try {
        final JobID jobId = resource.getJobID();
        final JobVertexID vid1 = new JobVertexID();
        final JobVertexID vid2 = new JobVertexID();
        final List<JobVertex> vertices;
        {
            JobVertex v1 = new JobVertex("v1", vid1);
            JobVertex v2 = new JobVertex("v2", vid2);
            v1.setParallelism(1);
            v2.setParallelism(1);
            v1.setInvokableClass(DummyInvokable.class);
            v2.setInvokableClass(DummyInvokable.class);
            vertices = Arrays.asList(v1, v2);
        }
        final Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
        final Executor executor = Executors.newFixedThreadPool(4);
        // try a lot!
        for (int i = 0; i < 20000; i++) {
            final TestExecGraph eg = new TestExecGraph(jobId);
            eg.attachJobGraph(vertices);
            final Execution e1 = eg.getJobVertex(vid1).getTaskVertices()[0].getCurrentExecutionAttempt();
            final Execution e2 = eg.getJobVertex(vid2).getTaskVertices()[0].getCurrentExecutionAttempt();
            initializeExecution(e1);
            initializeExecution(e2);
            execGraphStateField.set(eg, JobStatus.FAILING);
            execGraphSlotProviderField.set(eg, scheduler);
            Runnable r1 = new Runnable() {

                @Override
                public void run() {
                    e1.cancelingComplete();
                }
            };
            Runnable r2 = new Runnable() {

                @Override
                public void run() {
                    e2.cancelingComplete();
                }
            };
            executor.execute(r1);
            executor.execute(r2);
            eg.waitTillDone();
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 9 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionStateProgressTest method testAccumulatedStateFinished.

@Test
public void testAccumulatedStateFinished() {
    try {
        final JobID jid = new JobID();
        final JobVertexID vid = new JobVertexID();
        JobVertex ajv = new JobVertex("TestVertex", vid);
        ajv.setParallelism(3);
        ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());
        ExecutionGraph graph = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jid, "test job", new Configuration(), new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
        graph.attachJobGraph(Collections.singletonList(ajv));
        setGraphStatus(graph, JobStatus.RUNNING);
        ExecutionJobVertex ejv = graph.getJobVertex(vid);
        // mock resources and mock taskmanager
        for (ExecutionVertex ee : ejv.getTaskVertices()) {
            SimpleSlot slot = getInstance(new ActorTaskManagerGateway(new SimpleActorGateway(TestingUtils.defaultExecutionContext()))).allocateSimpleSlot(jid);
            ee.deployToSlot(slot);
        }
        // finish all
        for (ExecutionVertex ee : ejv.getTaskVertices()) {
            ee.executionFinished();
        }
        assertTrue(ejv.isInFinalState());
        assertEquals(JobStatus.FINISHED, graph.getState());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphSchedulingTest method testScheduleSourceBeforeTarget.

// ------------------------------------------------------------------------
//  Tests
// ------------------------------------------------------------------------
/**
	 * Tests that with scheduling futures and pipelined deployment, the target vertex will
	 * not deploy its task before the source vertex does.
	 */
@Test
public void testScheduleSourceBeforeTarget() throws Exception {
    //                                            [pipelined]
    //  we construct a simple graph    (source) ----------------> (target)
    final int parallelism = 1;
    final JobVertex sourceVertex = new JobVertex("source");
    sourceVertex.setParallelism(parallelism);
    sourceVertex.setInvokableClass(NoOpInvokable.class);
    final JobVertex targetVertex = new JobVertex("target");
    targetVertex.setParallelism(parallelism);
    targetVertex.setInvokableClass(NoOpInvokable.class);
    targetVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    final JobID jobId = new JobID();
    final JobGraph jobGraph = new JobGraph(jobId, "test", sourceVertex, targetVertex);
    final FlinkCompletableFuture<SimpleSlot> sourceFuture = new FlinkCompletableFuture<>();
    final FlinkCompletableFuture<SimpleSlot> targetFuture = new FlinkCompletableFuture<>();
    ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);
    slotProvider.addSlot(sourceVertex.getID(), 0, sourceFuture);
    slotProvider.addSlot(targetVertex.getID(), 0, targetFuture);
    final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
    //  set up two TaskManager gateways and slots
    final TaskManagerGateway gatewaySource = createTaskManager();
    final TaskManagerGateway gatewayTarget = createTaskManager();
    final SimpleSlot sourceSlot = createSlot(gatewaySource, jobId);
    final SimpleSlot targetSlot = createSlot(gatewayTarget, jobId);
    eg.setScheduleMode(ScheduleMode.EAGER);
    eg.setQueuedSchedulingAllowed(true);
    eg.scheduleForExecution();
    // job should be running
    assertEquals(JobStatus.RUNNING, eg.getState());
    // we fulfill the target slot before the source slot
    // that should not cause a deployment or deployment related failure
    targetFuture.complete(targetSlot);
    verify(gatewayTarget, new Timeout(50, times(0))).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
    assertEquals(JobStatus.RUNNING, eg.getState());
    // now supply the source slot
    sourceFuture.complete(sourceSlot);
    // by now, all deployments should have happened
    verify(gatewaySource, timeout(1000)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
    verify(gatewayTarget, timeout(1000)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class));
    assertEquals(JobStatus.RUNNING, eg.getState());
}