Examples with Grouping org.apache.storm.generated.Grouping used on opensource projects

Example 1 with Grouping

use of org.apache.storm.generated.Grouping in project storm by apache.

the class Testing method captureTopology.

/**
     * Rewrites a topology so that all the tuples flowing through it are captured
     * @param topology the topology to rewrite
     * @return the modified topology and a new Bolt that can retrieve the
     * captured tuples.
     */
public static CapturedTopology<StormTopology> captureTopology(StormTopology topology) {
    //Don't modify the original
    topology = topology.deepCopy();
    TupleCaptureBolt capturer = new TupleCaptureBolt();
    Map<GlobalStreamId, Grouping> captureBoltInputs = new HashMap<>();
    for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) {
        String id = spoutEntry.getKey();
        for (Entry<String, StreamInfo> streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) {
            String stream = streamEntry.getKey();
            StreamInfo info = streamEntry.getValue();
            if (info.is_direct()) {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
            } else {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
            }
        }
    }
    for (Entry<String, Bolt> boltEntry : topology.get_bolts().entrySet()) {
        String id = boltEntry.getKey();
        for (Entry<String, StreamInfo> streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) {
            String stream = streamEntry.getKey();
            StreamInfo info = streamEntry.getValue();
            if (info.is_direct()) {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
            } else {
                captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
            }
        }
    }
    topology.put_to_bolts(Utils.uuid(), new Bolt(Thrift.serializeComponentObject(capturer), Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null)));
    return new CapturedTopology<>(topology, capturer);
}

Example 2 with Grouping

use of org.apache.storm.generated.Grouping in project storm by apache.

the class Thrift method prepareComponentCommon.

public static ComponentCommon prepareComponentCommon(Map<GlobalStreamId, Grouping> inputs, Map<String, StreamInfo> outputs, Integer parallelismHint, Map conf) {
    Map<GlobalStreamId, Grouping> mappedInputs = new HashMap<>();
    Map<String, StreamInfo> mappedOutputs = new HashMap<>();
    if (inputs != null && !inputs.isEmpty()) {
        mappedInputs.putAll(inputs);
    }
    if (outputs != null && !outputs.isEmpty()) {
        mappedOutputs.putAll(outputs);
    }
    ComponentCommon component = new ComponentCommon(mappedInputs, mappedOutputs);
    if (parallelismHint != null) {
        component.set_parallelism_hint(parallelismHint);
    }
    if (conf != null) {
        component.set_json_conf(JSONValue.toJSONString(conf));
    }
    return component;
}

Example 3 with Grouping

use of org.apache.storm.generated.Grouping in project storm by apache.

the class StormCommon method addAcker.

@SuppressWarnings("unchecked")
public static void addAcker(Map conf, StormTopology topology) {
    int ackerNum = Utils.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), Utils.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
    Map<GlobalStreamId, Grouping> inputs = ackerInputs(topology);
    Map<String, StreamInfo> outputStreams = new HashMap<String, StreamInfo>();
    outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    Map<String, Object> ackerConf = new HashMap<>();
    ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum);
    ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
    Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf);
    for (Bolt bolt : topology.get_bolts().values()) {
        ComponentCommon common = bolt.get_common();
        common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val")));
        common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
        common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
    }
    for (SpoutSpec spout : topology.get_spouts().values()) {
        ComponentCommon common = spout.get_common();
        Map spoutConf = componentConf(spout);
        spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
        common.set_json_conf(JSONValue.toJSONString(spoutConf));
        common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task")));
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping());
    }
    topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
}

Example 4 with Grouping

use of org.apache.storm.generated.Grouping in project storm by apache.

the class Executor method outboundComponents.

/**
     * Returns map of stream id to component id to grouper
     */
private Map<String, Map<String, LoadAwareCustomStreamGrouping>> outboundComponents(WorkerTopologyContext workerTopologyContext, String componentId, Map stormConf) {
    Map<String, Map<String, LoadAwareCustomStreamGrouping>> ret = new HashMap<>();
    Map<String, Map<String, Grouping>> outputGroupings = workerTopologyContext.getTargets(componentId);
    for (Map.Entry<String, Map<String, Grouping>> entry : outputGroupings.entrySet()) {
        String streamId = entry.getKey();
        Map<String, Grouping> componentGrouping = entry.getValue();
        Fields outFields = workerTopologyContext.getComponentOutputFields(componentId, streamId);
        Map<String, LoadAwareCustomStreamGrouping> componentGrouper = new HashMap<String, LoadAwareCustomStreamGrouping>();
        for (Map.Entry<String, Grouping> cg : componentGrouping.entrySet()) {
            String component = cg.getKey();
            Grouping grouping = cg.getValue();
            List<Integer> outTasks = workerTopologyContext.getComponentTasks(component);
            LoadAwareCustomStreamGrouping grouper = GrouperFactory.mkGrouper(workerTopologyContext, componentId, streamId, outFields, grouping, outTasks, stormConf);
            componentGrouper.put(component, grouper);
        }
        if (componentGrouper.size() > 0) {
            ret.put(streamId, componentGrouper);
        }
    }
    for (String stream : workerTopologyContext.getComponentCommon(componentId).get_streams().keySet()) {
        if (!ret.containsKey(stream)) {
            ret.put(stream, null);
        }
    }
    return ret;
}

Example 5 with Grouping

use of org.apache.storm.generated.Grouping in project storm by apache.

the class WorkerState method workerOutboundTasks.

/**
     *
     * @return seq of task ids that receive messages from this worker
     */
private Set<Integer> workerOutboundTasks() {
    WorkerTopologyContext context = getWorkerTopologyContext();
    Set<String> components = new HashSet<>();
    for (Integer taskId : taskIds) {
        for (Map<String, Grouping> value : context.getTargets(context.getComponentId(taskId)).values()) {
            components.addAll(value.keySet());
        }
    }
    Set<Integer> outboundTasks = new HashSet<>();
    for (Map.Entry<String, List<Integer>> entry : Utils.reverseMap(taskToComponent).entrySet()) {
        if (components.contains(entry.getKey())) {
            outboundTasks.addAll(entry.getValue());
        }
    }
    return outboundTasks;
}