Examples with ComponentCommon backtype.storm.generated.ComponentCommon used on opensource projects

Example 1 with ComponentCommon

use of backtype.storm.generated.ComponentCommon in project storm by nathanmarz.

the class GeneralTopologyContext method maxTopologyMessageTimeout.

public int maxTopologyMessageTimeout() {
    Integer max = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
    for (String spout : getRawTopology().get_spouts().keySet()) {
        ComponentCommon common = getComponentCommon(spout);
        String jsonConf = common.get_json_conf();
        if (jsonConf != null) {
            Map conf = (Map) JSONValue.parse(jsonConf);
            Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS);
            if (comp != null) {
                max = Math.max(Utils.getInt(comp), max);
            }
        }
    }
    return max;
}

Example 2 with ComponentCommon

use of backtype.storm.generated.ComponentCommon in project storm by nathanmarz.

the class TopologyBuilder method getComponentCommon.

private ComponentCommon getComponentCommon(String id, IComponent component) {
    ComponentCommon ret = new ComponentCommon(_commons.get(id));
    OutputFieldsGetter getter = new OutputFieldsGetter();
    component.declareOutputFields(getter);
    ret.set_streams(getter.getFieldsDeclaration());
    return ret;
}

Example 3 with ComponentCommon

use of backtype.storm.generated.ComponentCommon in project storm by nathanmarz.

the class TopologyBuilder method createTopology.

public StormTopology createTopology() {
    Map<String, Bolt> boltSpecs = new HashMap<String, Bolt>();
    Map<String, SpoutSpec> spoutSpecs = new HashMap<String, SpoutSpec>();
    for (String boltId : _bolts.keySet()) {
        IRichBolt bolt = _bolts.get(boltId);
        ComponentCommon common = getComponentCommon(boltId, bolt);
        boltSpecs.put(boltId, new Bolt(ComponentObject.serialized_java(Utils.serialize(bolt)), common));
    }
    for (String spoutId : _spouts.keySet()) {
        IRichSpout spout = _spouts.get(spoutId);
        ComponentCommon common = getComponentCommon(spoutId, spout);
        spoutSpecs.put(spoutId, new SpoutSpec(ComponentObject.serialized_java(Utils.serialize(spout)), common));
    }
    return new StormTopology(spoutSpecs, boltSpecs, new HashMap<String, StateSpoutSpec>());
}

Example 4 with ComponentCommon

use of backtype.storm.generated.ComponentCommon in project jstorm by alibaba.

the class NimbusUtils method normalizeConf.

/**
     * Normalize stormConf
     *
     * @param conf      cluster conf
     * @param stormConf storm topology conf
     * @param topology  storm topology
     * @return normalized conf
     * @throws Exception
     */
@SuppressWarnings("rawtypes")
public static Map normalizeConf(Map conf, Map stormConf, StormTopology topology) throws Exception {
    List kryoRegisterList = new ArrayList();
    List kryoDecoratorList = new ArrayList();
    Map totalConf = new HashMap();
    totalConf.putAll(conf);
    totalConf.putAll(stormConf);
    Object totalRegister = totalConf.get(Config.TOPOLOGY_KRYO_REGISTER);
    if (totalRegister != null) {
        LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", TOPOLOGY_KRYO_REGISTER" + totalRegister.getClass().getName());
        JStormUtils.mergeList(kryoRegisterList, totalRegister);
    }
    Object totalDecorator = totalConf.get(Config.TOPOLOGY_KRYO_DECORATORS);
    if (totalDecorator != null) {
        LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", TOPOLOGY_KRYO_DECORATOR" + totalDecorator.getClass().getName());
        JStormUtils.mergeList(kryoDecoratorList, totalDecorator);
    }
    Set<String> cids = ThriftTopologyUtils.getComponentIds(topology);
    for (Iterator it = cids.iterator(); it.hasNext(); ) {
        String componentId = (String) it.next();
        ComponentCommon common = ThriftTopologyUtils.getComponentCommon(topology, componentId);
        String json = common.get_json_conf();
        if (json == null) {
            continue;
        }
        Map mtmp = (Map) JStormUtils.from_json(json);
        if (mtmp == null) {
            StringBuilder sb = new StringBuilder();
            sb.append("Failed to deserilaize " + componentId);
            sb.append(" json configuration: ");
            sb.append(json);
            LOG.info(sb.toString());
            throw new Exception(sb.toString());
        }
        Object componentKryoRegister = mtmp.get(Config.TOPOLOGY_KRYO_REGISTER);
        if (componentKryoRegister != null) {
            LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", componentId:" + componentId + ", TOPOLOGY_KRYO_REGISTER" + componentKryoRegister.getClass().getName());
            JStormUtils.mergeList(kryoRegisterList, componentKryoRegister);
        }
        Object componentDecorator = mtmp.get(Config.TOPOLOGY_KRYO_DECORATORS);
        if (componentDecorator != null) {
            LOG.info("topology:" + stormConf.get(Config.TOPOLOGY_NAME) + ", componentId:" + componentId + ", TOPOLOGY_KRYO_DECORATOR" + componentDecorator.getClass().getName());
            JStormUtils.mergeList(kryoDecoratorList, componentDecorator);
        }
    }
    Map kryoRegisterMap = mapifySerializations(kryoRegisterList);
    List decoratorList = JStormUtils.distinctList(kryoDecoratorList);
    Integer ackerNum = JStormUtils.parseInt(totalConf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Map rtn = new HashMap();
    //ensure to be cluster_mode
    rtn.put(Config.STORM_CLUSTER_MODE, conf.get(Config.STORM_CLUSTER_MODE));
    rtn.putAll(stormConf);
    rtn.put(Config.TOPOLOGY_KRYO_DECORATORS, decoratorList);
    rtn.put(Config.TOPOLOGY_KRYO_REGISTER, kryoRegisterMap);
    rtn.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackerNum);
    rtn.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, totalConf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM));
    return rtn;
}

Example 5 with ComponentCommon

use of backtype.storm.generated.ComponentCommon in project jstorm by alibaba.

the class GeneralTopologyContext method maxTopologyMessageTimeout.

public int maxTopologyMessageTimeout() {
    Integer max = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
    for (String spout : getRawTopology().get_spouts().keySet()) {
        ComponentCommon common = getComponentCommon(spout);
        String jsonConf = common.get_json_conf();
        if (jsonConf != null) {
            Map conf = (Map) Utils.from_json(jsonConf);
            Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS);
            if (comp != null) {
                max = Math.max(Utils.getInt(comp), max);
            }
        }
    }
    return max;
}