Examples with GlobalStreamId org.apache.storm.generated.GlobalStreamId used on opensource projects

Example 1 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class Testing method captureTopology.

/**
     * Rewrites a topology so that all the tuples flowing through it are captured
     * @param topology the topology to rewrite
     * @return the modified topology and a new Bolt that can retrieve the
     * captured tuples.
     */
public static CapturedTopology<StormTopology> captureTopology(StormTopology topology) {
    //Don't modify the original
    topology = topology.deepCopy();
    TupleCaptureBolt capturer = new TupleCaptureBolt();
    Map<GlobalStreamId, Grouping> captureBoltInputs = new HashMap<>();
    for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) {
        String id = spoutEntry.getKey();
        for (Entry<String, StreamInfo> streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) {
            String stream = streamEntry.getKey();
            StreamInfo info = streamEntry.getValue();
            if (info.is_direct()) {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
            } else {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
            }
        }
    }
    for (Entry<String, Bolt> boltEntry : topology.get_bolts().entrySet()) {
        String id = boltEntry.getKey();
        for (Entry<String, StreamInfo> streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) {
            String stream = streamEntry.getKey();
            StreamInfo info = streamEntry.getValue();
            if (info.is_direct()) {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
            } else {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
            }
        }
    }
    topology.put_to_bolts(Utils.uuid(), new Bolt(Thrift.serializeComponentObject(capturer), Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null)));
    return new CapturedTopology<>(topology, capturer);
}

Example 2 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class Thrift method prepareComponentCommon.

public static ComponentCommon prepareComponentCommon(Map<GlobalStreamId, Grouping> inputs, Map<String, StreamInfo> outputs, Integer parallelismHint, Map conf) {
    Map<GlobalStreamId, Grouping> mappedInputs = new HashMap<>();
    Map<String, StreamInfo> mappedOutputs = new HashMap<>();
    if (inputs != null && !inputs.isEmpty()) {
        mappedInputs.putAll(inputs);
    }
    if (outputs != null && !outputs.isEmpty()) {
        mappedOutputs.putAll(outputs);
    }
    ComponentCommon component = new ComponentCommon(mappedInputs, mappedOutputs);
    if (parallelismHint != null) {
        component.set_parallelism_hint(parallelismHint);
    }
    if (conf != null) {
        component.set_json_conf(JSONValue.toJSONString(conf));
    }
    return component;
}

Example 3 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class StormCommon method addAcker.

@SuppressWarnings("unchecked")
public static void addAcker(Map conf, StormTopology topology) {
    int ackerNum = Utils.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), Utils.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
    Map<GlobalStreamId, Grouping> inputs = ackerInputs(topology);
    Map<String, StreamInfo> outputStreams = new HashMap<String, StreamInfo>();
    outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    Map<String, Object> ackerConf = new HashMap<>();
    ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum);
    ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
    Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf);
    for (Bolt bolt : topology.get_bolts().values()) {
        ComponentCommon common = bolt.get_common();
        common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val")));
        common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
        common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
    }
    for (SpoutSpec spout : topology.get_spouts().values()) {
        ComponentCommon common = spout.get_common();
        Map spoutConf = componentConf(spout);
        spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
        common.set_json_conf(JSONValue.toJSONString(spoutConf));
        common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task")));
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping());
    }
    topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
}

Example 4 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class GeneralTopologyContext method getTargets.

/**
     * Gets information about who is consuming the outputs of the specified component,
     * and how.
     *
     * @return Map from stream id to component id to the Grouping used.
     */
public Map<String, Map<String, Grouping>> getTargets(String componentId) {
    Map<String, Map<String, Grouping>> ret = new HashMap<>();
    for (String otherComponentId : getComponentIds()) {
        Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> entry : inputs.entrySet()) {
            GlobalStreamId id = entry.getKey();
            if (id.get_componentId().equals(componentId)) {
                Map<String, Grouping> curr = ret.get(id.get_streamId());
                if (curr == null)
                    curr = new HashMap<>();
                curr.put(otherComponentId, entry.getValue());
                ret.put(id.get_streamId(), curr);
            }
        }
    }
    return ret;
}

Example 5 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class TridentTopologyBuilder method getBoltBatchToComponentSubscriptions.

Map<String, Set<String>> getBoltBatchToComponentSubscriptions(String id) {
    Map<String, Set<String>> ret = new HashMap();
    for (GlobalStreamId s : getBoltSubscriptionStreams(id)) {
        String b = _batchIds.get(s);
        if (!ret.containsKey(b))
            ret.put(b, new HashSet());
        ret.get(b).add(s.get_componentId());
    }
    return ret;
}