Examples with StreamEdge org.apache.flink.streaming.api.graph.StreamEdge used on opensource projects

Example 21 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class StreamTaskMailboxTestHarnessBuilder method initializeNetworkInput.

private void initializeNetworkInput(NetworkInputConfig networkInput, StreamNode targetVertexDummy, StreamMockEnvironment streamMockEnvironment, List<StreamEdge> inPhysicalEdges) {
    int gateIndex = networkInput.getInputGateIndex();
    TypeSerializer<?> inputSerializer = networkInput.getTypeSerializer();
    inputGates[gateIndex] = new StreamTestSingleInputGate<>(inputChannelsPerGate.get(gateIndex), gateIndex, inputSerializer, bufferSize, modifyGateBuilder.apply(new SingleInputGateBuilder().setBufferDebloatConfiguration(BufferDebloatConfiguration.fromConfiguration(streamMockEnvironment.getTaskManagerInfo().getConfiguration()))));
    StreamNode sourceVertexDummy = new StreamNode(0, null, null, (StreamOperator<?>) null, null, SourceStreamTask.class);
    StreamEdge streamEdge = new StreamEdge(sourceVertexDummy, targetVertexDummy, gateIndex + 1, new BroadcastPartitioner<>(), null);
    inPhysicalEdges.add(streamEdge);
    streamMockEnvironment.addInputGate(inputGates[gateIndex].getInputGate());
}

Example 22 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class StreamConfigChainer method chain.

public <IN, OUT> StreamConfigChainer<OWNER> chain(OperatorID operatorID, StreamOperatorFactory<OUT> operatorFactory, TypeSerializer<IN> inputSerializer, TypeSerializer<OUT> outputSerializer, boolean createKeyedStateBackend) {
    chainIndex++;
    StreamEdge streamEdge = new StreamEdge(new StreamNode(tailConfig.getChainIndex(), null, null, (StreamOperator<?>) null, null, null), new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null), 0, null, null);
    streamEdge.setBufferTimeout(bufferTimeout);
    tailConfig.setChainedOutputs(Collections.singletonList(streamEdge));
    tailConfig = new StreamConfig(new Configuration());
    tailConfig.setStreamOperatorFactory(checkNotNull(operatorFactory));
    tailConfig.setOperatorID(checkNotNull(operatorID));
    tailConfig.setupNetworkInputs(inputSerializer);
    tailConfig.setTypeSerializerOut(outputSerializer);
    if (createKeyedStateBackend) {
        // used to test multiple stateful operators chained in a single task.
        tailConfig.setStateKeySerializer(inputSerializer);
        tailConfig.setStateBackendUsesManagedMemory(true);
        tailConfig.setManagedMemoryFractionOperatorOfUseCase(ManagedMemoryUseCase.STATE_BACKEND, 1.0);
    }
    tailConfig.setChainIndex(chainIndex);
    chainedConfigs.put(chainIndex, tailConfig);
    return this;
}

Example 23 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class DataStreamTest method testUnion.

/**
 * Tests union functionality. This ensures that self-unions and unions of streams with differing
 * parallelism work.
 *
 * @throws Exception
 */
@Test
public void testUnion() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    DataStream<Long> input1 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> selfUnion = input1.union(input1).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> input6 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> selfUnionDifferentPartition = input6.broadcast().union(input6).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    });
    DataStream<Long> input2 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> input3 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(2);
    DataStream<Long> unionDifferingParallelism = input2.union(input3).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> input4 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(2);
    DataStream<Long> input5 = env.generateSequence(0, 0).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    DataStream<Long> unionDifferingPartitioning = input4.broadcast().union(input5).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return null;
        }
    }).setParallelism(4);
    StreamGraph streamGraph = getStreamGraph(env);
    // verify self union
    assertTrue(streamGraph.getStreamNode(selfUnion.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(selfUnion.getId()).getInEdges()) {
        assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
    }
    // verify self union with different partitioners
    assertTrue(streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges().size() == 2);
    boolean hasForward = false;
    boolean hasBroadcast = false;
    for (StreamEdge edge : streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) {
        if (edge.getPartitioner() instanceof ForwardPartitioner) {
            hasForward = true;
        }
        if (edge.getPartitioner() instanceof BroadcastPartitioner) {
            hasBroadcast = true;
        }
    }
    assertTrue(hasForward && hasBroadcast);
    // verify union of streams with differing parallelism
    assertTrue(streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges()) {
        if (edge.getSourceId() == input2.getId()) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        } else if (edge.getSourceId() == input3.getId()) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
        } else {
            fail("Wrong input edge.");
        }
    }
    // verify union of streams with differing partitionings
    assertTrue(streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges().size() == 2);
    for (StreamEdge edge : streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges()) {
        if (edge.getSourceId() == input4.getId()) {
            assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
        } else if (edge.getSourceId() == input5.getId()) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
        } else {
            fail("Wrong input edge.");
        }
    }
}

Example 24 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class OneInputStreamTaskTest method configureChainedTestingStreamOperator.

// ==============================================================================================
// Utility functions and classes
// ==============================================================================================
private void configureChainedTestingStreamOperator(StreamConfig streamConfig, int numberChainedTasks) {
    Preconditions.checkArgument(numberChainedTasks >= 1, "The operator chain must at least " + "contain one operator.");
    TestingStreamOperator<Integer, Integer> previousOperator = new TestingStreamOperator<>();
    streamConfig.setStreamOperator(previousOperator);
    streamConfig.setOperatorID(new OperatorID(0L, 0L));
    // create the chain of operators
    Map<Integer, StreamConfig> chainedTaskConfigs = new HashMap<>(numberChainedTasks - 1);
    List<StreamEdge> outputEdges = new ArrayList<>(numberChainedTasks - 1);
    for (int chainedIndex = 1; chainedIndex < numberChainedTasks; chainedIndex++) {
        TestingStreamOperator<Integer, Integer> chainedOperator = new TestingStreamOperator<>();
        StreamConfig chainedConfig = new StreamConfig(new Configuration());
        chainedConfig.setupNetworkInputs(StringSerializer.INSTANCE);
        chainedConfig.setStreamOperator(chainedOperator);
        chainedConfig.setOperatorID(new OperatorID(0L, chainedIndex));
        chainedTaskConfigs.put(chainedIndex, chainedConfig);
        StreamEdge outputEdge = new StreamEdge(new StreamNode(chainedIndex - 1, null, null, (StreamOperator<?>) null, null, null), new StreamNode(chainedIndex, null, null, (StreamOperator<?>) null, null, null), 0, null, null);
        outputEdges.add(outputEdge);
    }
    streamConfig.setChainedOutputs(outputEdges);
    streamConfig.setTransitiveChainedTaskConfigs(chainedTaskConfigs);
}

Example 25 with StreamEdge

use of org.apache.flink.streaming.api.graph.StreamEdge in project flink by apache.

the class ForwardForUnspecifiedPartitionerTest method testConvertToForwardPartitioner.

@Test
public void testConvertToForwardPartitioner() {
    JobGraph jobGraph = StreamPartitionerTestUtils.createJobGraph("group1", "group1", new ForwardForUnspecifiedPartitioner<>());
    List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertThat(jobVertices.size(), is(1));
    JobVertex vertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
    StreamConfig sourceConfig = new StreamConfig(vertex.getConfiguration());
    StreamEdge edge = sourceConfig.getChainedOutputs(getClass().getClassLoader()).get(0);
    assertThat(edge.getPartitioner(), instanceOf(ForwardPartitioner.class));
}