Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 21 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphConstructionTest method testCoLocationConstraintCreation.

@Test
public void testCoLocationConstraintCreation() {
    try {
        final JobID jobId = new JobID();
        final String jobName = "Co-Location Constraint Sample Job";
        final Configuration cfg = new Configuration();
        // simple group of two, cyclic
        JobVertex v1 = new JobVertex("vertex1");
        JobVertex v2 = new JobVertex("vertex2");
        v1.setParallelism(6);
        v2.setParallelism(4);
        v1.setInvokableClass(AbstractInvokable.class);
        v2.setInvokableClass(AbstractInvokable.class);
        SlotSharingGroup sl1 = new SlotSharingGroup();
        v1.setSlotSharingGroup(sl1);
        v2.setSlotSharingGroup(sl1);
        v2.setStrictlyCoLocatedWith(v1);
        v1.setStrictlyCoLocatedWith(v2);
        // complex forked dependency pattern
        JobVertex v3 = new JobVertex("vertex3");
        JobVertex v4 = new JobVertex("vertex4");
        JobVertex v5 = new JobVertex("vertex5");
        JobVertex v6 = new JobVertex("vertex6");
        JobVertex v7 = new JobVertex("vertex7");
        v3.setParallelism(3);
        v4.setParallelism(3);
        v5.setParallelism(3);
        v6.setParallelism(3);
        v7.setParallelism(3);
        v3.setInvokableClass(AbstractInvokable.class);
        v4.setInvokableClass(AbstractInvokable.class);
        v5.setInvokableClass(AbstractInvokable.class);
        v6.setInvokableClass(AbstractInvokable.class);
        v7.setInvokableClass(AbstractInvokable.class);
        SlotSharingGroup sl2 = new SlotSharingGroup();
        v3.setSlotSharingGroup(sl2);
        v4.setSlotSharingGroup(sl2);
        v5.setSlotSharingGroup(sl2);
        v6.setSlotSharingGroup(sl2);
        v7.setSlotSharingGroup(sl2);
        v4.setStrictlyCoLocatedWith(v3);
        v5.setStrictlyCoLocatedWith(v4);
        v6.setStrictlyCoLocatedWith(v3);
        v3.setStrictlyCoLocatedWith(v7);
        // isolated vertex
        JobVertex v8 = new JobVertex("vertex8");
        v8.setParallelism(2);
        v8.setInvokableClass(AbstractInvokable.class);
        JobGraph jg = new JobGraph(jobId, jobName, v1, v2, v3, v4, v5, v6, v7, v8);
        ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
        eg.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());
        // check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
        {
            ExecutionVertex[] v1s = eg.getJobVertex(v1.getID()).getTaskVertices();
            ExecutionVertex[] v2s = eg.getJobVertex(v2.getID()).getTaskVertices();
            Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
            for (int i = 0; i < v2.getParallelism(); i++) {
                assertNotNull(v1s[i].getLocationConstraint());
                assertNotNull(v2s[i].getLocationConstraint());
                assertTrue(v1s[i].getLocationConstraint() == v2s[i].getLocationConstraint());
                all.add(v1s[i].getLocationConstraint());
            }
            for (int i = v2.getParallelism(); i < v1.getParallelism(); i++) {
                assertNotNull(v1s[i].getLocationConstraint());
                all.add(v1s[i].getLocationConstraint());
            }
            assertEquals("not all co location constraints are distinct", v1.getParallelism(), all.size());
        }
        // check the v1 / v2 co location hints ( assumes parallelism(v1) >= parallelism(v2) )
        {
            ExecutionVertex[] v3s = eg.getJobVertex(v3.getID()).getTaskVertices();
            ExecutionVertex[] v4s = eg.getJobVertex(v4.getID()).getTaskVertices();
            ExecutionVertex[] v5s = eg.getJobVertex(v5.getID()).getTaskVertices();
            ExecutionVertex[] v6s = eg.getJobVertex(v6.getID()).getTaskVertices();
            ExecutionVertex[] v7s = eg.getJobVertex(v7.getID()).getTaskVertices();
            Set<CoLocationConstraint> all = new HashSet<CoLocationConstraint>();
            for (int i = 0; i < v3.getParallelism(); i++) {
                assertNotNull(v3s[i].getLocationConstraint());
                assertTrue(v3s[i].getLocationConstraint() == v4s[i].getLocationConstraint());
                assertTrue(v4s[i].getLocationConstraint() == v5s[i].getLocationConstraint());
                assertTrue(v5s[i].getLocationConstraint() == v6s[i].getLocationConstraint());
                assertTrue(v6s[i].getLocationConstraint() == v7s[i].getLocationConstraint());
                all.add(v3s[i].getLocationConstraint());
            }
            assertEquals("not all co location constraints are distinct", v3.getParallelism(), all.size());
        }
        // check the v8 has no co location hints
        {
            ExecutionVertex[] v8s = eg.getJobVertex(v8.getID()).getTaskVertices();
            for (int i = 0; i < v8.getParallelism(); i++) {
                assertNull(v8s[i].getLocationConstraint());
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 22 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphConstructionTest method testSetupInputSplits.

@Test
public void testSetupInputSplits() {
    try {
        final InputSplit[] emptySplits = new InputSplit[0];
        InputSplitAssigner assigner1 = mock(InputSplitAssigner.class);
        InputSplitAssigner assigner2 = mock(InputSplitAssigner.class);
        @SuppressWarnings("unchecked") InputSplitSource<InputSplit> source1 = mock(InputSplitSource.class);
        @SuppressWarnings("unchecked") InputSplitSource<InputSplit> source2 = mock(InputSplitSource.class);
        when(source1.createInputSplits(Matchers.anyInt())).thenReturn(emptySplits);
        when(source2.createInputSplits(Matchers.anyInt())).thenReturn(emptySplits);
        when(source1.getInputSplitAssigner(emptySplits)).thenReturn(assigner1);
        when(source2.getInputSplitAssigner(emptySplits)).thenReturn(assigner2);
        final JobID jobId = new JobID();
        final String jobName = "Test Job Sample Name";
        final Configuration cfg = new Configuration();
        JobVertex v1 = new JobVertex("vertex1");
        JobVertex v2 = new JobVertex("vertex2");
        JobVertex v3 = new JobVertex("vertex3");
        JobVertex v4 = new JobVertex("vertex4");
        JobVertex v5 = new JobVertex("vertex5");
        v1.setParallelism(5);
        v2.setParallelism(7);
        v3.setParallelism(2);
        v4.setParallelism(11);
        v5.setParallelism(4);
        v1.setInvokableClass(AbstractInvokable.class);
        v2.setInvokableClass(AbstractInvokable.class);
        v3.setInvokableClass(AbstractInvokable.class);
        v4.setInvokableClass(AbstractInvokable.class);
        v5.setInvokableClass(AbstractInvokable.class);
        v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        v3.setInputSplitSource(source1);
        v5.setInputSplitSource(source2);
        List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
        ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
        try {
            eg.attachJobGraph(ordered);
        } catch (JobException e) {
            e.printStackTrace();
            fail("Job failed with exception: " + e.getMessage());
        }
        assertEquals(assigner1, eg.getAllVertices().get(v3.getID()).getSplitAssigner());
        assertEquals(assigner2, eg.getAllVertices().get(v5.getID()).getSplitAssigner());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 23 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphConstructionTest method testAttachViaIds.

@Test
public void testAttachViaIds() throws Exception {
    final JobID jobId = new JobID();
    final String jobName = "Test Job Sample Name";
    final Configuration cfg = new Configuration();
    // construct part one of the execution graph
    JobVertex v1 = new JobVertex("vertex1");
    JobVertex v2 = new JobVertex("vertex2");
    JobVertex v3 = new JobVertex("vertex3");
    v1.setParallelism(5);
    v2.setParallelism(7);
    v3.setParallelism(2);
    v1.setInvokableClass(AbstractInvokable.class);
    v2.setInvokableClass(AbstractInvokable.class);
    v3.setInvokableClass(AbstractInvokable.class);
    // this creates an intermediate result for v1
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    // create results for v2 and v3
    IntermediateDataSet v2result = v2.createAndAddResultDataSet(ResultPartitionType.PIPELINED);
    IntermediateDataSet v3result_1 = v3.createAndAddResultDataSet(ResultPartitionType.PIPELINED);
    IntermediateDataSet v3result_2 = v3.createAndAddResultDataSet(ResultPartitionType.PIPELINED);
    List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3));
    ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
    try {
        eg.attachJobGraph(ordered);
    } catch (JobException e) {
        e.printStackTrace();
        fail("Job failed with exception: " + e.getMessage());
    }
    // attach the second part of the graph
    JobVertex v4 = new JobVertex("vertex4");
    JobVertex v5 = new JobVertex("vertex5");
    v4.setParallelism(11);
    v5.setParallelism(4);
    v4.setInvokableClass(AbstractInvokable.class);
    v5.setInvokableClass(AbstractInvokable.class);
    v4.connectIdInput(v2result.getId(), DistributionPattern.ALL_TO_ALL);
    v4.connectIdInput(v3result_1.getId(), DistributionPattern.ALL_TO_ALL);
    v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v5.connectIdInput(v3result_2.getId(), DistributionPattern.ALL_TO_ALL);
    List<JobVertex> ordered2 = new ArrayList<JobVertex>(Arrays.asList(v4, v5));
    try {
        eg.attachJobGraph(ordered2);
    } catch (JobException e) {
        e.printStackTrace();
        fail("Job failed with exception: " + e.getMessage());
    }
    // verify
    verifyTestGraph(eg, jobId, v1, v2, v3, v4, v5);
}

Example 24 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphDeploymentTest method testAccumulatorsAndMetricsForwarding.

/**
	 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
	 * execution that failed or was canceled.
	 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
    final JobVertexID jid1 = new JobVertexID();
    final JobVertexID jid2 = new JobVertexID();
    JobVertex v1 = new JobVertex("v1", jid1);
    JobVertex v2 = new JobVertex("v2", jid2);
    Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
    ExecutionGraph graph = graphAndExecutions.f0;
    // verify behavior for canceled executions
    Execution execution1 = graphAndExecutions.f1.values().iterator().next();
    IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
    Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
    accumulators.put("acc", new IntCounter(4));
    AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);
    TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);
    graph.updateState(state);
    assertEquals(ioMetrics, execution1.getIOMetrics());
    assertNotNull(execution1.getUserAccumulators());
    assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());
    // verify behavior for failed executions
    Execution execution2 = graphAndExecutions.f1.values().iterator().next();
    IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
    Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
    accumulators2.put("acc", new IntCounter(8));
    AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);
    TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);
    graph.updateState(state2);
    assertEquals(ioMetrics2, execution2.getIOMetrics());
    assertNotNull(execution2.getUserAccumulators());
    assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}

Example 25 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class ExecutionGraphDeploymentTest method testBuildDeploymentDescriptor.

@Test
public void testBuildDeploymentDescriptor() {
    try {
        final JobID jobId = new JobID();
        final JobVertexID jid1 = new JobVertexID();
        final JobVertexID jid2 = new JobVertexID();
        final JobVertexID jid3 = new JobVertexID();
        final JobVertexID jid4 = new JobVertexID();
        JobVertex v1 = new JobVertex("v1", jid1);
        JobVertex v2 = new JobVertex("v2", jid2);
        JobVertex v3 = new JobVertex("v3", jid3);
        JobVertex v4 = new JobVertex("v4", jid4);
        v1.setParallelism(10);
        v2.setParallelism(10);
        v3.setParallelism(10);
        v4.setParallelism(10);
        v1.setInvokableClass(BatchTask.class);
        v2.setInvokableClass(BatchTask.class);
        v3.setInvokableClass(BatchTask.class);
        v4.setInvokableClass(BatchTask.class);
        v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
        ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, "some job", new Configuration(), new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
        List<JobVertex> ordered = Arrays.asList(v1, v2, v3, v4);
        eg.attachJobGraph(ordered);
        ExecutionJobVertex ejv = eg.getAllVertices().get(jid2);
        ExecutionVertex vertex = ejv.getTaskVertices()[3];
        ExecutionGraphTestUtils.SimpleActorGateway instanceGateway = new ExecutionGraphTestUtils.SimpleActorGateway(TestingUtils.directExecutionContext());
        final Instance instance = getInstance(new ActorTaskManagerGateway(instanceGateway));
        final SimpleSlot slot = instance.allocateSimpleSlot(jobId);
        assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
        vertex.deployToSlot(slot);
        assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
        TaskDeploymentDescriptor descr = instanceGateway.lastTDD;
        assertNotNull(descr);
        JobInformation jobInformation = descr.getSerializedJobInformation().deserializeValue(getClass().getClassLoader());
        TaskInformation taskInformation = descr.getSerializedTaskInformation().deserializeValue(getClass().getClassLoader());
        assertEquals(jobId, jobInformation.getJobId());
        assertEquals(jid2, taskInformation.getJobVertexId());
        assertEquals(3, descr.getSubtaskIndex());
        assertEquals(10, taskInformation.getNumberOfSubtasks());
        assertEquals(BatchTask.class.getName(), taskInformation.getInvokableClassName());
        assertEquals("v2", taskInformation.getTaskName());
        Collection<ResultPartitionDeploymentDescriptor> producedPartitions = descr.getProducedPartitions();
        Collection<InputGateDeploymentDescriptor> consumedPartitions = descr.getInputGates();
        assertEquals(2, producedPartitions.size());
        assertEquals(1, consumedPartitions.size());
        Iterator<ResultPartitionDeploymentDescriptor> iteratorProducedPartitions = producedPartitions.iterator();
        Iterator<InputGateDeploymentDescriptor> iteratorConsumedPartitions = consumedPartitions.iterator();
        assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
        assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
        assertEquals(10, iteratorConsumedPartitions.next().getInputChannelDeploymentDescriptors().length);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}