Examples with JobVertex org.apache.flink.runtime.jobgraph.JobVertex used on opensource projects

Example 46 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class UnionClosedBranchingTest method testUnionClosedBranchingTest.

@Test
public void testUnionClosedBranchingTest() throws Exception {
    // -----------------------------------------------------------------------------------------
    // Build test program
    // -----------------------------------------------------------------------------------------
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setExecutionMode(executionMode);
    env.setParallelism(4);
    DataSet<Tuple1<Integer>> src1 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
    DataSet<Tuple1<Integer>> src2 = env.fromElements(new Tuple1<>(0), new Tuple1<>(1));
    DataSet<Tuple1<Integer>> union = src1.union(src2);
    DataSet<Tuple2<Integer, Integer>> join = union.join(union).where(0).equalTo(0).projectFirst(0).projectSecond(0);
    join.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
    // -----------------------------------------------------------------------------------------
    // Verify optimized plan
    // -----------------------------------------------------------------------------------------
    OptimizedPlan optimizedPlan = compileNoStats(env.createProgramPlan());
    SinkPlanNode sinkNode = optimizedPlan.getDataSinks().iterator().next();
    DualInputPlanNode joinNode = (DualInputPlanNode) sinkNode.getPredecessor();
    // Verify that the compiler correctly sets the expected data exchange modes.
    for (Channel channel : joinNode.getInputs()) {
        assertEquals("Unexpected data exchange mode between union and join node.", unionToJoin, channel.getDataExchangeMode());
        assertEquals("Unexpected ship strategy between union and join node.", unionToJoinStrategy, channel.getShipStrategy());
    }
    for (SourcePlanNode src : optimizedPlan.getDataSources()) {
        for (Channel channel : src.getOutgoingChannels()) {
            assertEquals("Unexpected data exchange mode between source and union node.", sourceToUnion, channel.getDataExchangeMode());
            assertEquals("Unexpected ship strategy between source and union node.", sourceToUnionStrategy, channel.getShipStrategy());
        }
    }
    // -----------------------------------------------------------------------------------------
    // Verify generated JobGraph
    // -----------------------------------------------------------------------------------------
    JobGraphGenerator jgg = new JobGraphGenerator();
    JobGraph jobGraph = jgg.compileJobGraph(optimizedPlan);
    List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    // Sanity check for the test setup
    assertEquals("Unexpected number of vertices created.", 4, vertices.size());
    // Verify all sources
    JobVertex[] sources = new JobVertex[] { vertices.get(0), vertices.get(1) };
    for (JobVertex src : sources) {
        // Sanity check
        assertTrue("Unexpected vertex type. Test setup is broken.", src.isInputVertex());
        // The union is not translated to an extra union task, but the join uses a union
        // input gate to read multiple inputs. The source create a single result per consumer.
        assertEquals("Unexpected number of created results.", 2, src.getNumberOfProducedIntermediateDataSets());
        for (IntermediateDataSet dataSet : src.getProducedDataSets()) {
            ResultPartitionType dsType = dataSet.getResultType();
            // Ensure batch exchange unless PIPELINED_FORCE is enabled.
            if (!executionMode.equals(ExecutionMode.PIPELINED_FORCED)) {
                assertTrue("Expected batch exchange, but result type is " + dsType + ".", dsType.isBlocking());
            } else {
                assertFalse("Expected non-batch exchange, but result type is " + dsType + ".", dsType.isBlocking());
            }
        }
    }
}

Example 47 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class JobGraphGeneratorTest method testResourcesForChainedOperators.

/**
 * Verifies that the resources are merged correctly for chained operators when generating job
 * graph
 */
@Test
public void testResourcesForChainedOperators() throws Exception {
    ResourceSpec resource1 = ResourceSpec.newBuilder(0.1, 100).build();
    ResourceSpec resource2 = ResourceSpec.newBuilder(0.2, 200).build();
    ResourceSpec resource3 = ResourceSpec.newBuilder(0.3, 300).build();
    ResourceSpec resource4 = ResourceSpec.newBuilder(0.4, 400).build();
    ResourceSpec resource5 = ResourceSpec.newBuilder(0.5, 500).build();
    ResourceSpec resource6 = ResourceSpec.newBuilder(0.6, 600).build();
    ResourceSpec resource7 = ResourceSpec.newBuilder(0.7, 700).build();
    Method opMethod = Operator.class.getDeclaredMethod("setResources", ResourceSpec.class);
    opMethod.setAccessible(true);
    Method sinkMethod = DataSink.class.getDeclaredMethod("setResources", ResourceSpec.class);
    sinkMethod.setAccessible(true);
    MapFunction<Long, Long> mapFunction = new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return value;
        }
    };
    FilterFunction<Long> filterFunction = new FilterFunction<Long>() {

        @Override
        public boolean filter(Long value) throws Exception {
            return false;
        }
    };
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Long> input = env.fromElements(1L, 2L, 3L);
    opMethod.invoke(input, resource1);
    DataSet<Long> map1 = input.map(mapFunction);
    opMethod.invoke(map1, resource2);
    // CHAIN(Source -> Map -> Filter)
    DataSet<Long> filter1 = map1.filter(filterFunction);
    opMethod.invoke(filter1, resource3);
    IterativeDataSet<Long> startOfIteration = filter1.iterate(10);
    opMethod.invoke(startOfIteration, resource4);
    DataSet<Long> map2 = startOfIteration.map(mapFunction);
    opMethod.invoke(map2, resource5);
    // CHAIN(Map -> Filter)
    DataSet<Long> feedback = map2.filter(filterFunction);
    opMethod.invoke(feedback, resource6);
    DataSink<Long> sink = startOfIteration.closeWith(feedback).output(new DiscardingOutputFormat<Long>());
    sinkMethod.invoke(sink, resource7);
    JobGraph jobGraph = compileJob(env);
    JobVertex sourceMapFilterVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
    JobVertex iterationHeadVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    JobVertex feedbackVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(2);
    JobVertex sinkVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(3);
    JobVertex iterationSyncVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(4);
    assertTrue(sourceMapFilterVertex.getMinResources().equals(resource1.merge(resource2).merge(resource3)));
    assertTrue(iterationHeadVertex.getPreferredResources().equals(resource4));
    assertTrue(feedbackVertex.getMinResources().equals(resource5.merge(resource6)));
    assertTrue(sinkVertex.getPreferredResources().equals(resource7));
    assertTrue(iterationSyncVertex.getMinResources().equals(resource4));
}

Example 48 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class TempInIterationsTest method testTempInIterationTest.

/*
     * Tests whether temps barriers are correctly set in within iterations
     */
@Test
public void testTempInIterationTest() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class);
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 1, 0);
    DataSet<Tuple2<Long, Long>> update = iteration.getWorkset().join(iteration.getSolutionSet()).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
    iteration.closeWith(update, update).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan);
    JobGraphGenerator jgg = new JobGraphGenerator();
    JobGraph jg = jgg.compileJobGraph(oPlan);
    boolean solutionSetUpdateChecked = false;
    for (JobVertex v : jg.getVertices()) {
        if (v.getName().equals("SolutionSet Delta")) {
            // check if input of solution set delta is temped
            TaskConfig tc = new TaskConfig(v.getConfiguration());
            assertTrue(tc.isInputAsynchronouslyMaterialized(0));
            solutionSetUpdateChecked = true;
        }
    }
    assertTrue(solutionSetUpdateChecked);
}

Example 49 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class WebFrontendITCase method testCancelYarn.

@Test
public void testCancelYarn() throws Exception {
    // this only works if there is no active job at this point
    assertTrue(getRunningJobs(CLUSTER.getClusterClient()).isEmpty());
    // Create a task
    final JobVertex sender = new JobVertex("Sender");
    sender.setParallelism(2);
    sender.setInvokableClass(BlockingInvokable.class);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sender);
    final JobID jid = jobGraph.getJobID();
    ClusterClient<?> clusterClient = CLUSTER.getClusterClient();
    clusterClient.submitJob(jobGraph).get();
    // wait for job to show up
    while (getRunningJobs(CLUSTER.getClusterClient()).isEmpty()) {
        Thread.sleep(10);
    }
    // wait for tasks to be properly running
    BlockingInvokable.latch.await();
    final Duration testTimeout = Duration.ofMinutes(2);
    final Deadline deadline = Deadline.fromNow(testTimeout);
    try (HttpTestClient client = new HttpTestClient("localhost", getRestPort())) {
        // Request the file from the web server
        client.sendGetRequest("/jobs/" + jid + "/yarn-cancel", deadline.timeLeft());
        HttpTestClient.SimpleHttpResponse response = client.getNextResponse(deadline.timeLeft());
        assertEquals(HttpResponseStatus.ACCEPTED, response.getStatus());
        assertEquals("application/json; charset=UTF-8", response.getType());
        assertEquals("{}", response.getContent());
    }
    // wait for cancellation to finish
    while (!getRunningJobs(CLUSTER.getClusterClient()).isEmpty()) {
        Thread.sleep(20);
    }
    BlockingInvokable.reset();
}

Example 50 with JobVertex

use of org.apache.flink.runtime.jobgraph.JobVertex in project flink by apache.

the class DispatcherTest method testJobSubmissionWithPartialResourceConfigured.

/**
 * Tests that we can submit a job to the Dispatcher which then spawns a new JobManagerRunner.
 */
@Test
public void testJobSubmissionWithPartialResourceConfigured() throws Exception {
    ResourceSpec resourceSpec = ResourceSpec.newBuilder(2.0, 10).build();
    final JobVertex firstVertex = new JobVertex("firstVertex");
    firstVertex.setInvokableClass(NoOpInvokable.class);
    firstVertex.setResources(resourceSpec, resourceSpec);
    final JobVertex secondVertex = new JobVertex("secondVertex");
    secondVertex.setInvokableClass(NoOpInvokable.class);
    JobGraph jobGraphWithTwoVertices = JobGraphTestUtils.streamingJobGraph(firstVertex, secondVertex);
    dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(jobId, createdJobManagerRunnerLatch));
    DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);
    CompletableFuture<Acknowledge> acknowledgeFuture = dispatcherGateway.submitJob(jobGraphWithTwoVertices, TIMEOUT);
    try {
        acknowledgeFuture.get();
        fail("job submission should have failed");
    } catch (ExecutionException e) {
        assertTrue(ExceptionUtils.findThrowable(e, JobSubmissionException.class).isPresent());
    }
}