Examples with MergedInputsBoltWrapper org.apache.flink.storm.wrappers.MergedInputsBoltWrapper used on opensource projects

Example 1 with MergedInputsBoltWrapper

use of org.apache.flink.storm.wrappers.MergedInputsBoltWrapper in project flink by apache.

the class FlinkTopology method createOutput.

@SuppressWarnings({ "unchecked", "rawtypes" })
private SingleOutputStreamOperator<?> createOutput(String boltId, IRichBolt bolt, Map<GlobalStreamId, DataStream<Tuple>> inputStreams) {
    assert (boltId != null);
    assert (bolt != null);
    assert (inputStreams != null);
    Iterator<Entry<GlobalStreamId, DataStream<Tuple>>> iterator = inputStreams.entrySet().iterator();
    Entry<GlobalStreamId, DataStream<Tuple>> input1 = iterator.next();
    GlobalStreamId streamId1 = input1.getKey();
    String inputStreamId1 = streamId1.get_streamId();
    String inputComponentId1 = streamId1.get_componentId();
    Fields inputSchema1 = this.outputStreams.get(inputComponentId1).get(inputStreamId1);
    DataStream<Tuple> singleInputStream = input1.getValue();
    DataStream<StormTuple<Tuple>> mergedInputStream = null;
    while (iterator.hasNext()) {
        Entry<GlobalStreamId, DataStream<Tuple>> input2 = iterator.next();
        GlobalStreamId streamId2 = input2.getKey();
        DataStream<Tuple> inputStream2 = input2.getValue();
        if (mergedInputStream == null) {
            mergedInputStream = singleInputStream.connect(inputStream2).flatMap(new TwoFlinkStreamsMerger(streamId1, inputSchema1, streamId2, this.outputStreams.get(streamId2.get_componentId()).get(streamId2.get_streamId()))).returns(StormTuple.class);
        } else {
            mergedInputStream = mergedInputStream.connect(inputStream2).flatMap(new StormFlinkStreamMerger(streamId2, this.outputStreams.get(streamId2.get_componentId()).get(streamId2.get_streamId()))).returns(StormTuple.class);
        }
    }
    final HashMap<String, Fields> boltOutputs = this.outputStreams.get(boltId);
    final FlinkOutputFieldsDeclarer declarer = this.declarers.get(boltId);
    final SingleOutputStreamOperator<?> outputStream;
    if (boltOutputs.size() < 2) {
        // single output stream or sink
        String outputStreamId;
        if (boltOutputs.size() == 1) {
            outputStreamId = (String) boltOutputs.keySet().toArray()[0];
        } else {
            outputStreamId = null;
        }
        final TypeInformation<Tuple> outType = declarer.getOutputType(outputStreamId);
        final SingleOutputStreamOperator<Tuple> outStream;
        // only one input
        if (inputStreams.entrySet().size() == 1) {
            BoltWrapper<Tuple, Tuple> boltWrapper = new BoltWrapper<>(bolt, boltId, inputStreamId1, inputComponentId1, inputSchema1, null);
            boltWrapper.setStormTopology(stormTopology);
            outStream = singleInputStream.transform(boltId, outType, boltWrapper);
        } else {
            MergedInputsBoltWrapper<Tuple, Tuple> boltWrapper = new MergedInputsBoltWrapper<Tuple, Tuple>(bolt, boltId, null);
            boltWrapper.setStormTopology(stormTopology);
            outStream = mergedInputStream.transform(boltId, outType, boltWrapper);
        }
        if (outType != null) {
            // only for non-sink nodes
            final HashMap<String, DataStream<Tuple>> op = new HashMap<>();
            op.put(outputStreamId, outStream);
            availableInputs.put(boltId, op);
        }
        outputStream = outStream;
    } else {
        final TypeInformation<SplitStreamType<Tuple>> outType = (TypeInformation) TypeExtractor.getForClass(SplitStreamType.class);
        final SingleOutputStreamOperator<SplitStreamType<Tuple>> multiStream;
        // only one input
        if (inputStreams.entrySet().size() == 1) {
            final BoltWrapper<Tuple, SplitStreamType<Tuple>> boltWrapperMultipleOutputs = new BoltWrapper<>(bolt, boltId, inputStreamId1, inputComponentId1, inputSchema1, null);
            boltWrapperMultipleOutputs.setStormTopology(stormTopology);
            multiStream = singleInputStream.transform(boltId, outType, boltWrapperMultipleOutputs);
        } else {
            final MergedInputsBoltWrapper<Tuple, SplitStreamType<Tuple>> boltWrapperMultipleOutputs = new MergedInputsBoltWrapper<Tuple, SplitStreamType<Tuple>>(bolt, boltId, null);
            boltWrapperMultipleOutputs.setStormTopology(stormTopology);
            multiStream = mergedInputStream.transform(boltId, outType, boltWrapperMultipleOutputs);
        }
        final SplitStream<SplitStreamType<Tuple>> splitStream = multiStream.split(new StormStreamSelector<Tuple>());
        final HashMap<String, DataStream<Tuple>> op = new HashMap<>();
        for (String outputStreamId : boltOutputs.keySet()) {
            op.put(outputStreamId, splitStream.select(outputStreamId).map(new SplitStreamMapper<Tuple>()));
            SingleOutputStreamOperator<Tuple> outStream = splitStream.select(outputStreamId).map(new SplitStreamMapper<Tuple>());
            outStream.getTransformation().setOutputType(declarer.getOutputType(outputStreamId));
            op.put(outputStreamId, outStream);
        }
        availableInputs.put(boltId, op);
        outputStream = multiStream;
    }
    return outputStream;
}