Examples with JobGraph org.apache.flink.runtime.jobgraph.JobGraph used on opensource projects

Example 41 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class WebFrontendITCase method testStopYarn.

@Test
public void testStopYarn() throws Exception {
    // this only works if there is no active job at this point
    assertTrue(cluster.getCurrentlyRunningJobsJava().isEmpty());
    // Create a task
    final JobVertex sender = new JobVertex("Sender");
    sender.setParallelism(2);
    sender.setInvokableClass(StoppableInvokable.class);
    final JobGraph jobGraph = new JobGraph("Stoppable streaming test job", sender);
    final JobID jid = jobGraph.getJobID();
    cluster.submitJobDetached(jobGraph);
    // wait for job to show up
    while (cluster.getCurrentlyRunningJobsJava().isEmpty()) {
        Thread.sleep(10);
    }
    final FiniteDuration testTimeout = new FiniteDuration(2, TimeUnit.MINUTES);
    final Deadline deadline = testTimeout.fromNow();
    while (!cluster.getCurrentlyRunningJobsJava().isEmpty()) {
        try (HttpTestClient client = new HttpTestClient("localhost", port)) {
            // Request the file from the web server
            client.sendGetRequest("/jobs/" + jid + "/yarn-stop", deadline.timeLeft());
            HttpTestClient.SimpleHttpResponse response = client.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("json"));
            assertEquals("{}", response.getContent());
        }
        Thread.sleep(20);
    }
}

Example 42 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class RestartStrategyTest method testAutomaticRestartingWhenCheckpointing.

/**
	 * Tests that in a streaming use case where checkpointing is enabled, a
	 * fixed delay with Integer.MAX_VALUE retries is instantiated if no other restart
	 * strategy has been specified
	 */
@Test
public void testAutomaticRestartingWhenCheckpointing() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(500);
    env.fromElements(1).print();
    StreamGraph graph = env.getStreamGraph();
    JobGraph jobGraph = graph.getJobGraph();
    RestartStrategies.RestartStrategyConfiguration restartStrategy = jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();
    Assert.assertNotNull(restartStrategy);
    Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
    Assert.assertEquals(Integer.MAX_VALUE, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
}

Example 43 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class AsyncWaitOperatorTest method createChainedVertex.

private JobVertex createChainedVertex(boolean withLazyFunction) {
    StreamExecutionEnvironment chainEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    // the input is only used to construct a chained operator, and they will not be used in the real tests.
    DataStream<Integer> input = chainEnv.fromElements(1, 2, 3);
    if (withLazyFunction) {
        input = AsyncDataStream.orderedWait(input, new LazyAsyncFunction(), TIMEOUT, TimeUnit.MILLISECONDS, 6);
    } else {
        input = AsyncDataStream.orderedWait(input, new MyAsyncFunction(), TIMEOUT, TimeUnit.MILLISECONDS, 6);
    }
    // the map function is designed to chain after async function. we place an Integer object in it and
    // it is initialized in the open() method.
    // it is used to verify that operators in the operator chain should be opened from the tail to the head,
    // so the result from AsyncWaitOperator can pass down successfully and correctly.
    // if not, the test can not be passed.
    input = input.map(new RichMapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        private Integer initialValue = null;

        @Override
        public void open(Configuration parameters) throws Exception {
            initialValue = 1;
        }

        @Override
        public Integer map(Integer value) throws Exception {
            return initialValue + value;
        }
    });
    input = AsyncDataStream.unorderedWait(input, new MyAsyncFunction(), TIMEOUT, TimeUnit.MILLISECONDS, 3);
    input.map(new MapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 5162085254238405527L;

        @Override
        public Integer map(Integer value) throws Exception {
            return value;
        }
    }).startNewChain().addSink(new DiscardingSink<Integer>());
    // be build our own OperatorChain
    final JobGraph jobGraph = chainEnv.getStreamGraph().getJobGraph();
    Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 3);
    return jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
}

Example 44 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class KafkaConsumerTestBase method runMetricsTest.

/**
 * Test metrics reporting for consumer.
 *
 * @throws Exception
 */
public void runMetricsTest() throws Throwable {
    // create a stream with 5 topics
    final String topic = "metricsStream";
    createTestTopic(topic, 5, 1);
    final Tuple1<Throwable> error = new Tuple1<>(null);
    // start job writing & reading data.
    final StreamExecutionEnvironment env1 = StreamExecutionEnvironment.getExecutionEnvironment();
    env1.setParallelism(1);
    env1.getConfig().setRestartStrategy(RestartStrategies.noRestart());
    // let the source read everything into the network buffers
    env1.disableOperatorChaining();
    TypeInformationSerializationSchema<Tuple2<Integer, Integer>> schema = new TypeInformationSerializationSchema<>(TypeInformation.of(new TypeHint<Tuple2<Integer, Integer>>() {
    }), env1.getConfig());
    DataStream<Tuple2<Integer, Integer>> fromKafka = getStream(env1, topic, schema, standardProps);
    fromKafka.flatMap(new FlatMapFunction<Tuple2<Integer, Integer>, Void>() {

        @Override
        public void flatMap(Tuple2<Integer, Integer> value, Collector<Void> out) throws Exception {
        // no op
        }
    });
    DataStream<Tuple2<Integer, Integer>> fromGen = env1.addSource(new RichSourceFunction<Tuple2<Integer, Integer>>() {

        boolean running = true;

        @Override
        public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
            int i = 0;
            while (running) {
                ctx.collect(Tuple2.of(i++, getRuntimeContext().getIndexOfThisSubtask()));
                Thread.sleep(1);
            }
        }

        @Override
        public void cancel() {
            running = false;
        }
    });
    kafkaServer.produceIntoKafka(fromGen, topic, schema, standardProps, null);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env1.getStreamGraph());
    final JobID jobId = jobGraph.getJobID();
    Thread jobThread = new Thread(() -> {
        try {
            submitJobAndWaitForResult(client, jobGraph, getClass().getClassLoader());
        } catch (Throwable t) {
            if (!ExceptionUtils.findThrowable(t, JobCancellationException.class).isPresent()) {
                LOG.warn("Got exception during execution", t);
                error.f0 = t;
            }
        }
    });
    jobThread.start();
    try {
        // connect to JMX
        MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
        // wait until we've found all 5 offset metrics
        Set<ObjectName> offsetMetrics = mBeanServer.queryNames(new ObjectName("*current-offsets*:*"), null);
        while (offsetMetrics.size() < 5) {
            // test will time out if metrics are not properly working
            if (error.f0 != null) {
                // fail test early
                throw error.f0;
            }
            offsetMetrics = mBeanServer.queryNames(new ObjectName("*current-offsets*:*"), null);
            Thread.sleep(50);
        }
        Assert.assertEquals(5, offsetMetrics.size());
        // The test will fail if we never meet the condition
        while (true) {
            int numPosOffsets = 0;
            // check that offsets are correctly reported
            for (ObjectName object : offsetMetrics) {
                Object offset = mBeanServer.getAttribute(object, "Value");
                if ((long) offset >= 0) {
                    numPosOffsets++;
                }
            }
            if (numPosOffsets == 5) {
                break;
            }
            // wait for the consumer to consume on all partitions
            Thread.sleep(50);
        }
        // check if producer metrics are also available.
        Set<ObjectName> producerMetrics = mBeanServer.queryNames(new ObjectName("*KafkaProducer*:*"), null);
        Assert.assertTrue("No producer metrics found", producerMetrics.size() > 30);
        LOG.info("Found all JMX metrics. Cancelling job.");
    } finally {
        // cancel
        client.cancel(jobId).get();
        // wait for the job to finish (it should due to the cancel command above)
        jobThread.join();
    }
    if (error.f0 != null) {
        throw error.f0;
    }
    deleteTestTopic(topic);
}

Example 45 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class KafkaConsumerTestBase method runCancelingOnEmptyInputTest.

/**
 * Tests that the source can be properly canceled when reading empty partitions.
 */
public void runCancelingOnEmptyInputTest() throws Exception {
    final String topic = "cancelingOnEmptyInputTopic";
    final int parallelism = 3;
    createTestTopic(topic, parallelism, 1);
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(parallelism);
    env.enableCheckpointing(100);
    Properties props = new Properties();
    props.putAll(standardProps);
    props.putAll(secureProps);
    getStream(env, topic, new SimpleStringSchema(), props).addSink(new DiscardingSink<String>());
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    final JobID jobId = jobGraph.getJobID();
    final Runnable jobRunner = () -> {
        try {
            submitJobAndWaitForResult(client, jobGraph, getClass().getClassLoader());
        } catch (Throwable t) {
            LOG.error("Job Runner failed with exception", t);
            error.set(t);
        }
    };
    Thread runnerThread = new Thread(jobRunner, "program runner thread");
    runnerThread.start();
    // wait a bit before canceling
    Thread.sleep(2000);
    Throwable failueCause = error.get();
    if (failueCause != null) {
        failueCause.printStackTrace();
        Assert.fail("Test failed prematurely with: " + failueCause.getMessage());
    }
    // cancel
    client.cancel(jobId).get();
    // wait for the program to be done and validate that we failed with the right exception
    runnerThread.join();
    assertEquals(JobStatus.CANCELED, client.getJobStatus(jobId).get());
    deleteTestTopic(topic);
}