Examples with JobGraph org.apache.flink.runtime.jobgraph.JobGraph used on opensource projects

Example 66 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class StackTraceSampleCoordinatorITCase method testTaskClearedWhileSampling.

/**
	 * Tests that a cleared task is answered with a partial success response.
	 */
@Test
public void testTaskClearedWhileSampling() throws Exception {
    new JavaTestKit(testActorSystem) {

        {
            final FiniteDuration deadline = new FiniteDuration(60, TimeUnit.SECONDS);
            // The JobGraph
            final JobGraph jobGraph = new JobGraph();
            final int parallelism = 1;
            final JobVertex task = new JobVertex("Task");
            task.setInvokableClass(BlockingNoOpInvokable.class);
            task.setParallelism(parallelism);
            jobGraph.addVertex(task);
            ActorGateway jobManger = null;
            ActorGateway taskManager = null;
            try {
                jobManger = TestingUtils.createJobManager(testActorSystem, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), new Configuration());
                final Configuration config = new Configuration();
                config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, parallelism);
                taskManager = TestingUtils.createTaskManager(testActorSystem, jobManger, config, true, true);
                final ActorGateway jm = jobManger;
                new Within(deadline) {

                    @Override
                    protected void run() {
                        try {
                            ActorGateway testActor = new AkkaActorGateway(getTestActor(), null);
                            int maxAttempts = 10;
                            int sleepTime = 100;
                            for (int i = 0; i < maxAttempts; i++, sleepTime *= 2) {
                                // Submit the job and wait until it is running
                                JobClient.submitJobDetached(jm, config, jobGraph, deadline, ClassLoader.getSystemClassLoader());
                                jm.tell(new WaitForAllVerticesToBeRunning(jobGraph.getJobID()), testActor);
                                expectMsgEquals(new AllVerticesRunning(jobGraph.getJobID()));
                                // Get the ExecutionGraph
                                jm.tell(new RequestExecutionGraph(jobGraph.getJobID()), testActor);
                                ExecutionGraphFound executionGraphResponse = expectMsgClass(ExecutionGraphFound.class);
                                ExecutionGraph executionGraph = (ExecutionGraph) executionGraphResponse.executionGraph();
                                ExecutionJobVertex vertex = executionGraph.getJobVertex(task.getID());
                                StackTraceSampleCoordinator coordinator = new StackTraceSampleCoordinator(testActorSystem.dispatcher(), 60000);
                                Future<StackTraceSample> sampleFuture = coordinator.triggerStackTraceSample(vertex.getTaskVertices(), // sampling.
                                21474700 * 100, Time.milliseconds(10L), 0);
                                // Wait before cancelling so that some samples
                                // are actually taken.
                                Thread.sleep(sleepTime);
                                // Cancel job
                                scala.concurrent.Future<?> removeFuture = jm.ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobGraph.getJobID()), remaining());
                                jm.tell(new JobManagerMessages.CancelJob(jobGraph.getJobID()));
                                try {
                                    // Throws Exception on failure
                                    sampleFuture.get(remaining().toMillis(), TimeUnit.MILLISECONDS);
                                    // partial result.
                                    break;
                                } catch (Throwable t) {
                                // We were too fast in cancelling the job.
                                // Fall through and retry.
                                } finally {
                                    Await.ready(removeFuture, remaining());
                                }
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                            fail(e.getMessage());
                        }
                    }
                };
            } finally {
                TestingUtils.stopActor(jobManger);
                TestingUtils.stopActor(taskManager);
            }
        }
    };
}

Example 67 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JarActionHandler method getJobGraphAndClassLoader.

protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception {
    // generate the graph
    JobGraph graph = null;
    PackagedProgram program = new PackagedProgram(new File(jarDir, config.getJarFile()), config.getEntryClass(), config.getProgramArgs());
    ClassLoader classLoader = program.getUserCodeClassLoader();
    Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration());
    FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism());
    if (plan instanceof StreamingPlan) {
        graph = ((StreamingPlan) plan).getJobGraph();
    } else if (plan instanceof OptimizedPlan) {
        graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan);
    }
    if (graph == null) {
        throw new CompilerException("A valid job graph couldn't be generated for the jar.");
    }
    // Set the savepoint settings
    graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings());
    for (URL jar : program.getAllLibraries()) {
        try {
            graph.addJar(new Path(jar.toURI()));
        } catch (URISyntaxException e) {
            throw new ProgramInvocationException("Invalid jar path. Unexpected error. :(");
        }
    }
    return Tuple2.of(graph, classLoader);
}

Example 68 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JarRunHandler method handleJsonRequest.

@Override
public String handleJsonRequest(Map<String, String> pathParams, Map<String, String> queryParams, ActorGateway jobManager) throws Exception {
    try {
        JarActionHandlerConfig config = JarActionHandlerConfig.fromParams(pathParams, queryParams);
        Tuple2<JobGraph, ClassLoader> graph = getJobGraphAndClassLoader(config);
        try {
            graph.f0.uploadUserJars(jobManager, timeout, clientConfig);
        } catch (IOException e) {
            throw new ProgramInvocationException("Failed to upload jar files to the job manager", e);
        }
        try {
            JobClient.submitJobDetached(jobManager, clientConfig, graph.f0, timeout, graph.f1);
        } catch (JobExecutionException e) {
            throw new ProgramInvocationException("Failed to submit the job to the job manager", e);
        }
        StringWriter writer = new StringWriter();
        JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
        gen.writeStartObject();
        gen.writeStringField("jobid", graph.f0.getJobID().toString());
        gen.writeEndObject();
        gen.close();
        return writer.toString();
    } catch (Exception e) {
        return sendError(e);
    }
}

Example 69 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class JobGraphGeneratorTest method testResourcesForDeltaIteration.

/**
	 * Verifies that the resources are set onto each job vertex correctly when generating job graph
	 * which covers the delta iteration case
	 */
@Test
public void testResourcesForDeltaIteration() throws Exception {
    ResourceSpec resource1 = new ResourceSpec(0.1, 100);
    ResourceSpec resource2 = new ResourceSpec(0.2, 200);
    ResourceSpec resource3 = new ResourceSpec(0.3, 300);
    ResourceSpec resource4 = new ResourceSpec(0.4, 400);
    ResourceSpec resource5 = new ResourceSpec(0.5, 500);
    ResourceSpec resource6 = new ResourceSpec(0.6, 600);
    Method opMethod = Operator.class.getDeclaredMethod("setResources", ResourceSpec.class);
    opMethod.setAccessible(true);
    Method deltaMethod = DeltaIteration.class.getDeclaredMethod("setResources", ResourceSpec.class);
    deltaMethod.setAccessible(true);
    Method sinkMethod = DataSink.class.getDeclaredMethod("setResources", ResourceSpec.class);
    sinkMethod.setAccessible(true);
    MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> mapFunction = new MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {

        @Override
        public Tuple2<Long, Long> map(Tuple2<Long, Long> value) throws Exception {
            return value;
        }
    };
    FilterFunction<Tuple2<Long, Long>> filterFunction = new FilterFunction<Tuple2<Long, Long>>() {

        @Override
        public boolean filter(Tuple2<Long, Long> value) throws Exception {
            return false;
        }
    };
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<>(1L, 2L));
    opMethod.invoke(input, resource1);
    // CHAIN(Map -> Filter)
    DataSet<Tuple2<Long, Long>> map = input.map(mapFunction);
    opMethod.invoke(map, resource2);
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = map.iterateDelta(map, 100, 0).registerAggregator("test", new LongSumAggregator());
    deltaMethod.invoke(iteration, resource3);
    DataSet<Tuple2<Long, Long>> delta = iteration.getWorkset().map(mapFunction);
    opMethod.invoke(delta, resource4);
    DataSet<Tuple2<Long, Long>> feedback = delta.filter(filterFunction);
    opMethod.invoke(feedback, resource5);
    DataSink<Tuple2<Long, Long>> sink = iteration.closeWith(delta, feedback).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    sinkMethod.invoke(sink, resource6);
    Plan plan = env.createProgramPlan();
    Optimizer pc = new Optimizer(new Configuration());
    OptimizedPlan op = pc.compile(plan);
    JobGraphGenerator jgg = new JobGraphGenerator();
    JobGraph jobGraph = jgg.compileJobGraph(op);
    JobVertex sourceMapVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
    JobVertex iterationHeadVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    JobVertex deltaVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(2);
    JobVertex iterationTailVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(3);
    JobVertex feedbackVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(4);
    JobVertex sinkVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(5);
    JobVertex iterationSyncVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(6);
    assertTrue(sourceMapVertex.getMinResources().equals(resource1.merge(resource2)));
    assertTrue(iterationHeadVertex.getPreferredResources().equals(resource3));
    assertTrue(deltaVertex.getMinResources().equals(resource4));
    // the iteration tail task will be scheduled in the same instance with iteration head, and currently not set resources.
    assertTrue(iterationTailVertex.getPreferredResources().equals(ResourceSpec.DEFAULT));
    assertTrue(feedbackVertex.getMinResources().equals(resource5));
    assertTrue(sinkVertex.getPreferredResources().equals(resource6));
    assertTrue(iterationSyncVertex.getMinResources().equals(resource3));
}

Example 70 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class ClusterClient method getJobGraph.

private JobGraph getJobGraph(FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) {
    JobGraph job;
    if (optPlan instanceof StreamingPlan) {
        job = ((StreamingPlan) optPlan).getJobGraph();
        job.setSavepointRestoreSettings(savepointSettings);
    } else {
        JobGraphGenerator gen = new JobGraphGenerator(this.flinkConfig);
        job = gen.compileJobGraph((OptimizedPlan) optPlan);
    }
    for (URL jar : jarFiles) {
        try {
            job.addJar(new Path(jar.toURI()));
        } catch (URISyntaxException e) {
            throw new RuntimeException("URL is invalid. This should not happen.", e);
        }
    }
    job.setClasspaths(classpaths);
    return job;
}