Examples with StreamInfo org.apache.storm.generated.StreamInfo used on opensource projects

Example 1 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class Thrift method prepareComponentCommon.

public static ComponentCommon prepareComponentCommon(Map<GlobalStreamId, Grouping> inputs, Map<String, StreamInfo> outputs, Integer parallelismHint, Map conf) {
    Map<GlobalStreamId, Grouping> mappedInputs = new HashMap<>();
    Map<String, StreamInfo> mappedOutputs = new HashMap<>();
    if (inputs != null && !inputs.isEmpty()) {
        mappedInputs.putAll(inputs);
    }
    if (outputs != null && !outputs.isEmpty()) {
        mappedOutputs.putAll(outputs);
    }
    ComponentCommon component = new ComponentCommon(mappedInputs, mappedOutputs);
    if (parallelismHint != null) {
        component.set_parallelism_hint(parallelismHint);
    }
    if (conf != null) {
        component.set_json_conf(JSONValue.toJSONString(conf));
    }
    return component;
}

Example 2 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class StormCommon method addAcker.

@SuppressWarnings("unchecked")
public static void addAcker(Map conf, StormTopology topology) {
    int ackerNum = Utils.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), Utils.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
    Map<GlobalStreamId, Grouping> inputs = ackerInputs(topology);
    Map<String, StreamInfo> outputStreams = new HashMap<String, StreamInfo>();
    outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    Map<String, Object> ackerConf = new HashMap<>();
    ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum);
    ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
    Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf);
    for (Bolt bolt : topology.get_bolts().values()) {
        ComponentCommon common = bolt.get_common();
        common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val")));
        common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
        common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
    }
    for (SpoutSpec spout : topology.get_spouts().values()) {
        ComponentCommon common = spout.get_common();
        Map spoutConf = componentConf(spout);
        spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
        common.set_json_conf(JSONValue.toJSONString(spoutConf));
        common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task")));
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping());
    }
    topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
}

Example 3 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class Testing method captureTopology.

/**
 * Rewrites a topology so that all the tuples flowing through it are captured.
 * @param topology the topology to rewrite
 * @return the modified topology and a new Bolt that can retrieve the
 *     captured tuples.
 */
public static CapturedTopology<StormTopology> captureTopology(StormTopology topology) {
    // Don't modify the original
    topology = topology.deepCopy();
    TupleCaptureBolt capturer = new TupleCaptureBolt();
    Map<GlobalStreamId, Grouping> captureBoltInputs = new HashMap<>();
    for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) {
        String id = spoutEntry.getKey();
        for (Entry<String, StreamInfo> streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) {
            String stream = streamEntry.getKey();
            StreamInfo info = streamEntry.getValue();
            if (info.is_direct()) {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
            } else {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
            }
        }
    }
    for (Entry<String, Bolt> boltEntry : topology.get_bolts().entrySet()) {
        String id = boltEntry.getKey();
        for (Entry<String, StreamInfo> streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) {
            String stream = streamEntry.getKey();
            StreamInfo info = streamEntry.getValue();
            if (info.is_direct()) {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
            } else {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
            }
        }
    }
    topology.put_to_bolts(Utils.uuid(), new Bolt(Thrift.serializeComponentObject(capturer), Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null)));
    return new CapturedTopology<>(topology, capturer);
}

Example 4 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class ClojureSpout method declareOutputFields.

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    for (String stream : fields.keySet()) {
        StreamInfo info = fields.get(stream);
        declarer.declareStream(stream, info.is_direct(), new Fields(info.get_output_fields()));
    }
}

Example 5 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class StormCommon method addSystemStreams.

public static void addSystemStreams(StormTopology topology) {
    for (Object component : allComponents(topology).values()) {
        ComponentCommon common = getComponentCommon(component);
        StreamInfo streamInfo = Thrift.outputFields(Arrays.asList("event"));
        common.put_to_streams(SYSTEM_STREAM_ID, streamInfo);
    }
}