Examples with NoOpIntMap org.apache.flink.streaming.util.NoOpIntMap used on opensource projects

Example 1 with NoOpIntMap

use of org.apache.flink.streaming.util.NoOpIntMap in project flink by apache.

the class StreamGraphGeneratorTest method testVirtualTransformations2.

/**
	 * This tests whether virtual Transformations behave correctly.
	 *
	 * Checks whether output selector, partitioning works correctly when applied on a union.
	 */
@Test
public void testVirtualTransformations2() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source = env.fromElements(1, 10);
    DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());
    DataStream<Integer> map1 = rebalanceMap.map(new NoOpIntMap());
    DataStream<Integer> map2 = rebalanceMap.map(new NoOpIntMap());
    DataStream<Integer> map3 = rebalanceMap.map(new NoOpIntMap());
    EvenOddOutputSelector selector = new EvenOddOutputSelector();
    SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3).broadcast().split(selector).select("foo").map(new NoOpIntMap());
    unionedMap.addSink(new DiscardingSink<Integer>());
    StreamGraph graph = env.getStreamGraph();
    // verify that the properties are correctly set on all input operators
    assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
    assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));
    assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
    assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));
    assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
    assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));
}

Example 2 with NoOpIntMap

use of org.apache.flink.streaming.util.NoOpIntMap in project flink by apache.

the class StreamGraphGeneratorTest method testVirtualTransformations.

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source = env.fromElements(1, 10);
    DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());
    // verify that only the partitioning that was set last is used
    DataStream<Integer> broadcastMap = rebalanceMap.forward().global().broadcast().map(new NoOpIntMap());
    broadcastMap.addSink(new DiscardingSink<>());
    DataStream<Integer> broadcastOperator = rebalanceMap.map(new NoOpIntMap()).name("broadcast");
    DataStream<Integer> map1 = broadcastOperator.broadcast();
    DataStream<Integer> globalOperator = rebalanceMap.map(new NoOpIntMap()).name("global");
    DataStream<Integer> map2 = globalOperator.global();
    DataStream<Integer> shuffleOperator = rebalanceMap.map(new NoOpIntMap()).name("shuffle");
    DataStream<Integer> map3 = shuffleOperator.shuffle();
    SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3).map(new NoOpIntMap()).name("union");
    unionedMap.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    // rebalanceMap
    assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);
    // verify that only last partitioning takes precedence
    assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());
    // verify that partitioning in unions is preserved
    assertTrue(graph.getStreamNode(broadcastOperator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
    assertTrue(graph.getStreamNode(globalOperator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
    assertTrue(graph.getStreamNode(shuffleOperator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
}

Example 3 with NoOpIntMap

use of org.apache.flink.streaming.util.NoOpIntMap in project flink by apache.

the class StreamGraphGeneratorTest method testAutoMaxParallelism.

/**
 * Tests that the max parallelism is automatically set to the parallelism if it has not been
 * specified.
 */
@Test
public void testAutoMaxParallelism() {
    int globalParallelism = 42;
    int mapParallelism = 17;
    int maxParallelism = 21;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(globalParallelism);
    DataStream<Integer> source = env.fromElements(1, 2, 3);
    DataStream<Integer> keyedResult1 = source.keyBy(value -> value).map(new NoOpIntMap());
    DataStream<Integer> keyedResult2 = keyedResult1.keyBy(value -> value).map(new NoOpIntMap()).setParallelism(mapParallelism);
    DataStream<Integer> keyedResult3 = keyedResult2.keyBy(value -> value).map(new NoOpIntMap()).setMaxParallelism(maxParallelism);
    DataStream<Integer> keyedResult4 = keyedResult3.keyBy(value -> value).map(new NoOpIntMap()).setMaxParallelism(maxParallelism).setParallelism(mapParallelism);
    keyedResult4.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    StreamNode keyedResult3Node = graph.getStreamNode(keyedResult3.getId());
    StreamNode keyedResult4Node = graph.getStreamNode(keyedResult4.getId());
    assertEquals(maxParallelism, keyedResult3Node.getMaxParallelism());
    assertEquals(maxParallelism, keyedResult4Node.getMaxParallelism());
}

Example 4 with NoOpIntMap

use of org.apache.flink.streaming.util.NoOpIntMap in project flink by apache.

the class StreamGraphGeneratorTest method testSetupOfKeyGroupPartitioner.

/**
 * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of
 * maximum parallelism.
 */
@Test
public void testSetupOfKeyGroupPartitioner() {
    int maxParallelism = 42;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setMaxParallelism(maxParallelism);
    DataStream<Integer> source = env.fromElements(1, 2, 3);
    DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap());
    keyedResult.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId());
    StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner();
}

Example 5 with NoOpIntMap

use of org.apache.flink.streaming.util.NoOpIntMap in project flink by apache.

the class StreamGraphGeneratorTest method testMaxParallelismForwarding.

/**
 * Tests that the global and operator-wide max parallelism setting is respected.
 */
@Test
public void testMaxParallelismForwarding() {
    int globalMaxParallelism = 42;
    int keyedResult2MaxParallelism = 17;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setMaxParallelism(globalMaxParallelism);
    DataStream<Integer> source = env.fromElements(1, 2, 3);
    DataStream<Integer> keyedResult1 = source.keyBy(value -> value).map(new NoOpIntMap());
    DataStream<Integer> keyedResult2 = keyedResult1.keyBy(value -> value).map(new NoOpIntMap()).setMaxParallelism(keyedResult2MaxParallelism);
    keyedResult2.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    StreamNode keyedResult1Node = graph.getStreamNode(keyedResult1.getId());
    StreamNode keyedResult2Node = graph.getStreamNode(keyedResult2.getId());
    assertEquals(globalMaxParallelism, keyedResult1Node.getMaxParallelism());
    assertEquals(keyedResult2MaxParallelism, keyedResult2Node.getMaxParallelism());
}