Examples with StreamExecutionEnvironment org.apache.flink.streaming.api.environment.StreamExecutionEnvironment used on opensource projects

Example 96 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StreamingJobGraphGeneratorNodeHashTest method testNodeHashAfterIntermediateUnchaining.

/**
	 * Tests that (un)chaining affects the node hash (for intermediate nodes).
	 *
	 * <pre>
	 * A (chained): [ (src0) -> (map) -> (filter) -> (sink) ]
	 * B (unchained): [ (src0) ] -> [ (map) -> (filter) -> (sink) ]
	 * </pre>
	 *
	 * The hashes for the single vertex in A and the source vertex in B need to be different.
	 */
@Test
public void testNodeHashAfterIntermediateUnchaining() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    env.addSource(new NoOpSourceFunction()).map(new NoOpMapFunction()).name("map").startNewChain().filter(new NoOpFilterFunction()).addSink(new NoOpSinkFunction());
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    JobVertex chainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    assertTrue(chainedMap.getName().startsWith("map"));
    JobVertexID chainedMapId = chainedMap.getID();
    env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    env.addSource(new NoOpSourceFunction()).map(new NoOpMapFunction()).name("map").startNewChain().filter(new NoOpFilterFunction()).startNewChain().addSink(new NoOpSinkFunction());
    jobGraph = env.getStreamGraph().getJobGraph();
    JobVertex unchainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    assertEquals("map", unchainedMap.getName());
    JobVertexID unchainedMapId = unchainedMap.getID();
    assertNotEquals(chainedMapId, unchainedMapId);
}

Example 97 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StreamingJobGraphGeneratorNodeHashTest method testNodeHashIsDeterministic.

// ------------------------------------------------------------------------
// Deterministic hash assignment
// ------------------------------------------------------------------------
/**
	 * Creates the same flow twice and checks that all IDs are the same.
	 *
	 * <pre>
	 * [ (src) -> (map) -> (filter) -> (reduce) -> (map) -> (sink) ]
	 *                                                       //
	 * [ (src) -> (filter) ] -------------------------------//
	 *                                                      /
	 * [ (src) -> (filter) ] ------------------------------/
	 * </pre>
	 */
@Test
public void testNodeHashIsDeterministic() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    DataStream<String> src0 = env.addSource(new NoOpSourceFunction(), "src0").map(new NoOpMapFunction()).filter(new NoOpFilterFunction()).keyBy(new NoOpKeySelector()).reduce(new NoOpReduceFunction()).name("reduce");
    DataStream<String> src1 = env.addSource(new NoOpSourceFunction(), "src1").filter(new NoOpFilterFunction());
    DataStream<String> src2 = env.addSource(new NoOpSourceFunction(), "src2").filter(new NoOpFilterFunction());
    src0.map(new NoOpMapFunction()).union(src1, src2).addSink(new NoOpSinkFunction()).name("sink");
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    final Map<JobVertexID, String> ids = rememberIds(jobGraph);
    // Do it again and verify
    env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    src0 = env.addSource(new NoOpSourceFunction(), "src0").map(new NoOpMapFunction()).filter(new NoOpFilterFunction()).keyBy(new NoOpKeySelector()).reduce(new NoOpReduceFunction()).name("reduce");
    src1 = env.addSource(new NoOpSourceFunction(), "src1").filter(new NoOpFilterFunction());
    src2 = env.addSource(new NoOpSourceFunction(), "src2").filter(new NoOpFilterFunction());
    src0.map(new NoOpMapFunction()).union(src1, src2).addSink(new NoOpSinkFunction()).name("sink");
    jobGraph = env.getStreamGraph().getJobGraph();
    verifyIdsEqual(jobGraph, ids);
}

Example 98 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StreamingJobGraphGeneratorNodeHashTest method testNodeHashAfterSourceUnchaining.

/**
	 * Tests that (un)chaining affects the node hash (for sources).
	 *
	 * <pre>
	 * A (chained): [ (src0) -> (map) -> (filter) -> (sink) ]
	 * B (unchained): [ (src0) ] -> [ (map) -> (filter) -> (sink) ]
	 * </pre>
	 *
	 * The hashes for the single vertex in A and the source vertex in B need to be different.
	 */
@Test
public void testNodeHashAfterSourceUnchaining() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    env.addSource(new NoOpSourceFunction()).map(new NoOpMapFunction()).filter(new NoOpFilterFunction()).addSink(new NoOpSinkFunction());
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    JobVertexID sourceId = jobGraph.getVerticesSortedTopologicallyFromSources().get(0).getID();
    env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    env.addSource(new NoOpSourceFunction()).map(new NoOpMapFunction()).startNewChain().filter(new NoOpFilterFunction()).addSink(new NoOpSinkFunction());
    jobGraph = env.getStreamGraph().getJobGraph();
    JobVertexID unchainedSourceId = jobGraph.getVerticesSortedTopologicallyFromSources().get(0).getID();
    assertNotEquals(sourceId, unchainedSourceId);
}

Example 99 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StreamingJobGraphGeneratorNodeHashTest method testManualHashAssignmentForStartNodeInInChain.

/**
	 * Tests that a manual hash at the beginning of a chain is accepted.
	 *
	 * <p>This should work, because the ID is used at the beginning of a chain. This is currently
	 * not allowed for intermediate nodes (see {@link #testManualHashAssignmentForIntermediateNodeInChainThrowsException()}).
	 */
@Test
public void testManualHashAssignmentForStartNodeInInChain() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    env.addSource(new NoOpSourceFunction()).uid("source").map(new NoOpMapFunction()).addSink(new NoOpSinkFunction());
    env.getStreamGraph().getJobGraph();
}

Example 100 with StreamExecutionEnvironment

use of org.apache.flink.streaming.api.environment.StreamExecutionEnvironment in project flink by apache.

the class StreamingJobGraphGeneratorNodeHashTest method testManualHashAssignmentCollisionThrowsException.

/**
	 * Tests that a collision on the manual hash throws an Exception.
	 */
@Test(expected = IllegalArgumentException.class)
public void testManualHashAssignmentCollisionThrowsException() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(4);
    env.disableOperatorChaining();
    env.addSource(new NoOpSourceFunction()).uid("source").map(new NoOpMapFunction()).uid(// Collision
    "source").addSink(new NoOpSinkFunction());
    // This call is necessary to generate the job graph
    env.getStreamGraph().getJobGraph();
}