Examples with StreamNode org.apache.flink.streaming.api.graph.StreamNode used on opensource projects

Example 1 with StreamNode

use of org.apache.flink.streaming.api.graph.StreamNode in project flink by apache.

the class TwoInputStreamTaskTestHarness method initializeInputs.

@Override
protected void initializeInputs() throws IOException, InterruptedException {
    inputGates = new StreamTestSingleInputGate[numInputGates];
    List<StreamEdge> inPhysicalEdges = new LinkedList<StreamEdge>();
    StreamOperator<IN1> dummyOperator = new AbstractStreamOperator<IN1>() {

        private static final long serialVersionUID = 1L;
    };
    StreamNode sourceVertexDummy = new StreamNode(null, 0, "default group", dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
    StreamNode targetVertexDummy = new StreamNode(null, 1, "default group", dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
    for (int i = 0; i < numInputGates; i++) {
        switch(inputGateAssignment[i]) {
            case 1:
                {
                    inputGates[i] = new StreamTestSingleInputGate<IN1>(numInputChannelsPerGate, bufferSize, inputSerializer1);
                    StreamEdge streamEdge = new StreamEdge(sourceVertexDummy, targetVertexDummy, 1, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null);
                    inPhysicalEdges.add(streamEdge);
                    break;
                }
            case 2:
                {
                    inputGates[i] = new StreamTestSingleInputGate<IN2>(numInputChannelsPerGate, bufferSize, inputSerializer2);
                    StreamEdge streamEdge = new StreamEdge(sourceVertexDummy, targetVertexDummy, 2, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null);
                    inPhysicalEdges.add(streamEdge);
                    break;
                }
            default:
                throw new IllegalStateException("Wrong input gate assignment.");
        }
        this.mockEnv.addInputGate(inputGates[i].getInputGate());
    }
    streamConfig.setInPhysicalEdges(inPhysicalEdges);
    streamConfig.setNumberOfInputs(numInputGates);
    streamConfig.setTypeSerializerIn1(inputSerializer1);
    streamConfig.setTypeSerializerIn2(inputSerializer2);
}

Example 2 with StreamNode

use of org.apache.flink.streaming.api.graph.StreamNode in project flink by apache.

the class IterateITCase method testmultipleHeadsTailsWithTailPartitioning.

@Test
public void testmultipleHeadsTailsWithTailPartitioning() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> source1 = env.fromElements(1, 2, 3, 4, 5).shuffle().map(NoOpIntMap);
    DataStream<Integer> source2 = env.fromElements(1, 2, 3, 4, 5).map(NoOpIntMap);
    IterativeStream<Integer> iter1 = source1.union(source2).iterate();
    DataStream<Integer> head1 = iter1.map(NoOpIntMap).name("map1");
    DataStream<Integer> head2 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2).name("shuffle").rebalance();
    DataStreamSink<Integer> head3 = iter1.map(NoOpIntMap).setParallelism(DEFAULT_PARALLELISM / 2).addSink(new ReceiveCheckNoOpSink<Integer>());
    DataStreamSink<Integer> head4 = iter1.map(NoOpIntMap).addSink(new ReceiveCheckNoOpSink<Integer>());
    SplitStream<Integer> source3 = env.fromElements(1, 2, 3, 4, 5).map(NoOpIntMap).name("split").split(new EvenOddOutputSelector());
    iter1.closeWith(source3.select("even").union(head1.map(NoOpIntMap).name("bc").broadcast(), head2.map(NoOpIntMap).shuffle()));
    StreamGraph graph = env.getStreamGraph();
    JobGraph jg = graph.getJobGraph();
    assertEquals(1, graph.getIterationSourceSinkPairs().size());
    Tuple2<StreamNode, StreamNode> sourceSinkPair = graph.getIterationSourceSinkPairs().iterator().next();
    StreamNode itSource = sourceSinkPair.f0;
    StreamNode itSink = sourceSinkPair.f1;
    assertEquals(4, itSource.getOutEdges().size());
    assertEquals(3, itSink.getInEdges().size());
    assertEquals(itSource.getParallelism(), itSink.getParallelism());
    for (StreamEdge edge : itSource.getOutEdges()) {
        if (edge.getTargetVertex().getOperatorName().equals("map1")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
            assertEquals(4, edge.getTargetVertex().getParallelism());
        } else if (edge.getTargetVertex().getOperatorName().equals("shuffle")) {
            assertTrue(edge.getPartitioner() instanceof RebalancePartitioner);
            assertEquals(2, edge.getTargetVertex().getParallelism());
        }
    }
    for (StreamEdge edge : itSink.getInEdges()) {
        String tailName = edge.getSourceVertex().getOperatorName();
        if (tailName.equals("split")) {
            assertTrue(edge.getPartitioner() instanceof ForwardPartitioner);
            assertTrue(edge.getSelectedNames().contains("even"));
        } else if (tailName.equals("bc")) {
            assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
        } else if (tailName.equals("shuffle")) {
            assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
        }
    }
    // Test co-location
    JobVertex itSource1 = null;
    JobVertex itSink1 = null;
    for (JobVertex vertex : jg.getVertices()) {
        if (vertex.getName().contains("IterationSource")) {
            itSource1 = vertex;
        } else if (vertex.getName().contains("IterationSink")) {
            itSink1 = vertex;
        }
    }
    assertTrue(itSource1.getCoLocationGroup() != null);
    assertTrue(itSink1.getCoLocationGroup() != null);
    assertEquals(itSource1.getCoLocationGroup(), itSink1.getCoLocationGroup());
}

Example 3 with StreamNode

use of org.apache.flink.streaming.api.graph.StreamNode in project flink by apache.

the class IterateITCase method testImmutabilityWithCoiteration.

@Test
public void testImmutabilityWithCoiteration() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // for rebalance
    DataStream<Integer> source = env.fromElements(1, 10).map(NoOpIntMap);
    IterativeStream<Integer> iter1 = source.iterate();
    // Calling withFeedbackType should create a new iteration
    ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);
    iter1.closeWith(iter1.map(NoOpIntMap)).print();
    iter2.closeWith(iter2.map(NoOpCoMap)).print();
    StreamGraph graph = env.getStreamGraph();
    assertEquals(2, graph.getIterationSourceSinkPairs().size());
    for (Tuple2<StreamNode, StreamNode> sourceSinkPair : graph.getIterationSourceSinkPairs()) {
        assertEquals(sourceSinkPair.f0.getOutEdges().get(0).getTargetVertex(), sourceSinkPair.f1.getInEdges().get(0).getSourceVertex());
    }
}

Example 4 with StreamNode

use of org.apache.flink.streaming.api.graph.StreamNode in project flink by apache.

the class StreamGraphHasherV1 method traverseStreamGraphAndGenerateHashes.

@Override
public Map<Integer, byte[]> traverseStreamGraphAndGenerateHashes(StreamGraph streamGraph) {
    // The hash function used to generate the hash
    final HashFunction hashFunction = Hashing.murmur3_128(0);
    final Map<Integer, byte[]> hashes = new HashMap<>();
    Set<Integer> visited = new HashSet<>();
    Queue<StreamNode> remaining = new ArrayDeque<>();
    // We need to make the source order deterministic. The source IDs are
    // not returned in the same order, which means that submitting the same
    // program twice might result in different traversal, which breaks the
    // deterministic hash assignment.
    List<Integer> sources = new ArrayList<>();
    for (Integer sourceNodeId : streamGraph.getSourceIDs()) {
        sources.add(sourceNodeId);
    }
    Collections.sort(sources);
    // Start with source nodes
    for (Integer sourceNodeId : sources) {
        remaining.add(streamGraph.getStreamNode(sourceNodeId));
        visited.add(sourceNodeId);
    }
    StreamNode currentNode;
    while ((currentNode = remaining.poll()) != null) {
        // generate the hash code.
        if (generateNodeHash(currentNode, hashFunction, hashes, streamGraph.isChainingEnabled())) {
            // Add the child nodes
            for (StreamEdge outEdge : currentNode.getOutEdges()) {
                StreamNode child = outEdge.getTargetVertex();
                if (!visited.contains(child.getId())) {
                    remaining.add(child);
                    visited.add(child.getId());
                }
            }
        } else {
            // We will revisit this later.
            visited.remove(currentNode.getId());
        }
    }
    return hashes;
}

Example 5 with StreamNode

use of org.apache.flink.streaming.api.graph.StreamNode in project flink by apache.

the class StreamGraphHasherV1 method isChainable.

private boolean isChainable(StreamEdge edge, boolean isChainingEnabled) {
    StreamNode upStreamVertex = edge.getSourceVertex();
    StreamNode downStreamVertex = edge.getTargetVertex();
    StreamOperator<?> headOperator = upStreamVertex.getOperator();
    StreamOperator<?> outOperator = downStreamVertex.getOperator();
    return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled;
}