Examples with ComponentCommon org.apache.storm.generated.ComponentCommon used on opensource projects

Example 21 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class StormCommon method validateStructure.

public static void validateStructure(StormTopology topology) throws InvalidTopologyException {
    Map<String, Object> componentMap = allComponents(topology);
    for (Map.Entry<String, Object> entry : componentMap.entrySet()) {
        String componentId = entry.getKey();
        ComponentCommon common = getComponentCommon(entry.getValue());
        Map<GlobalStreamId, Grouping> inputs = common.get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
            String sourceStreamId = input.getKey().get_streamId();
            String sourceComponentId = input.getKey().get_componentId();
            if (!componentMap.keySet().contains(sourceComponentId)) {
                throw new WrappedInvalidTopologyException("Component: [" + componentId + "] subscribes from non-existent component [" + sourceComponentId + "]");
            }
            ComponentCommon sourceComponent = getComponentCommon(componentMap.get(sourceComponentId));
            if (!sourceComponent.get_streams().containsKey(sourceStreamId)) {
                throw new WrappedInvalidTopologyException("Component: [" + componentId + "] subscribes from non-existent stream: " + "[" + sourceStreamId + "] of component [" + sourceComponentId + "]");
            }
            Grouping grouping = input.getValue();
            if (Thrift.groupingType(grouping) == Grouping._Fields.FIELDS) {
                List<String> fields = new ArrayList<>(grouping.get_fields());
                Map<String, StreamInfo> streams = sourceComponent.get_streams();
                Set<String> sourceOutputFields = getStreamOutputFields(streams);
                fields.removeAll(sourceOutputFields);
                if (fields.size() != 0) {
                    throw new WrappedInvalidTopologyException("Component: [" + componentId + "] subscribes from stream: [" + sourceStreamId + "] of component " + "[" + sourceComponentId + "] + with non-existent fields: " + fields);
                }
            }
        }
    }
}

Example 22 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class StormCommon method validateBasic.

@SuppressWarnings("unchecked")
public static void validateBasic(StormTopology topology) throws InvalidTopologyException {
    validateIds(topology);
    for (StormTopology._Fields field : Thrift.getSpoutFields()) {
        Map<String, Object> spoutComponents = (Map<String, Object>) topology.getFieldValue(field);
        if (spoutComponents != null) {
            for (Object obj : spoutComponents.values()) {
                ComponentCommon common = getComponentCommon(obj);
                if (!isEmptyInputs(common)) {
                    throw new WrappedInvalidTopologyException("May not declare inputs for a spout");
                }
            }
        }
    }
    Map<String, Object> componentMap = allComponents(topology);
    for (Object componentObj : componentMap.values()) {
        Map<String, Object> conf = componentConf(componentObj);
        ComponentCommon common = getComponentCommon(componentObj);
        int parallelismHintNum = Thrift.getParallelismHint(common);
        Integer taskNum = ObjectReader.getInt(conf.get(Config.TOPOLOGY_TASKS), 0);
        if (taskNum > 0 && parallelismHintNum <= 0) {
            throw new WrappedInvalidTopologyException("Number of executors must be greater than 0 when number of tasks is greater than 0");
        }
    }
}

Example 23 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class StormCommon method addAcker.

@SuppressWarnings("unchecked")
public static void addAcker(Map<String, Object> conf, StormTopology topology) {
    Map<String, StreamInfo> outputStreams = new HashMap<String, StreamInfo>();
    outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    Map<String, Object> ackerConf = new HashMap<>();
    int ackerNum = ObjectReader.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), ObjectReader.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
    ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum);
    ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
    Map<GlobalStreamId, Grouping> inputs = ackerInputs(topology);
    Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf);
    for (Bolt bolt : topology.get_bolts().values()) {
        ComponentCommon common = bolt.get_common();
        common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val")));
        common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
        common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
    }
    for (SpoutSpec spout : topology.get_spouts().values()) {
        ComponentCommon common = spout.get_common();
        Map<String, Object> spoutConf = componentConf(spout);
        spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
        common.set_json_conf(JSONValue.toJSONString(spoutConf));
        common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task")));
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping());
    }
    topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
}

Example 24 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class StormCommon method componentConf.

@SuppressWarnings("unchecked")
public static Map<String, Object> componentConf(Object component) {
    try {
        Map<String, Object> conf = new HashMap<>();
        ComponentCommon common = getComponentCommon(component);
        String jconf = common.get_json_conf();
        if (jconf != null) {
            conf.putAll((Map<String, Object>) JSONValue.parseWithException(jconf));
        }
        return conf;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 25 with ComponentCommon

use of org.apache.storm.generated.ComponentCommon in project storm by apache.

the class StormCommon method validateIds.

private static Set<String> validateIds(Map<String, ?> componentMap) throws InvalidTopologyException {
    Set<String> keys = componentMap.keySet();
    for (String id : keys) {
        if (Utils.isSystemId(id)) {
            throw new WrappedInvalidTopologyException(id + " is not a valid component id.");
        }
    }
    for (Object componentObj : componentMap.values()) {
        ComponentCommon common = getComponentCommon(componentObj);
        Set<String> streamIds = common.get_streams().keySet();
        for (String id : streamIds) {
            if (Utils.isSystemId(id)) {
                throw new WrappedInvalidTopologyException(id + " is not a valid stream id.");
            }
        }
    }
    return keys;
}