Examples with StateSpoutSpec backtype.storm.generated.StateSpoutSpec used on opensource projects

Example 1 with StateSpoutSpec

use of backtype.storm.generated.StateSpoutSpec in project storm by nathanmarz.

the class TopologyBuilder method createTopology.

public StormTopology createTopology() {
    Map<String, Bolt> boltSpecs = new HashMap<String, Bolt>();
    Map<String, SpoutSpec> spoutSpecs = new HashMap<String, SpoutSpec>();
    for (String boltId : _bolts.keySet()) {
        IRichBolt bolt = _bolts.get(boltId);
        ComponentCommon common = getComponentCommon(boltId, bolt);
        boltSpecs.put(boltId, new Bolt(ComponentObject.serialized_java(Utils.serialize(bolt)), common));
    }
    for (String spoutId : _spouts.keySet()) {
        IRichSpout spout = _spouts.get(spoutId);
        ComponentCommon common = getComponentCommon(spoutId, spout);
        spoutSpecs.put(spoutId, new SpoutSpec(ComponentObject.serialized_java(Utils.serialize(spout)), common));
    }
    return new StormTopology(spoutSpecs, boltSpecs, new HashMap<String, StateSpoutSpec>());
}

Example 2 with StateSpoutSpec

use of backtype.storm.generated.StateSpoutSpec in project jstorm by alibaba.

the class TopologyBuilder method createTopology.

public StormTopology createTopology() {
    Map<String, Bolt> boltSpecs = new HashMap<>();
    Map<String, SpoutSpec> spoutSpecs = new HashMap<>();
    maybeAddCheckpointSpout();
    for (String boltId : _bolts.keySet()) {
        IRichBolt bolt = _bolts.get(boltId);
        bolt = maybeAddCheckpointTupleForwarder(bolt);
        ComponentCommon common = getComponentCommon(boltId, bolt);
        try {
            maybeAddCheckpointInputs(common);
            boltSpecs.put(boltId, new Bolt(ComponentObject.serialized_java(Utils.javaSerialize(bolt)), common));
        } catch (RuntimeException wrapperCause) {
            if (wrapperCause.getCause() != null && NotSerializableException.class.equals(wrapperCause.getCause().getClass())) {
                throw new IllegalStateException("Bolt '" + boltId + "' contains a non-serializable field of type " + wrapperCause.getCause().getMessage() + ", " + "which was instantiated prior to topology creation. " + wrapperCause.getCause().getMessage() + " " + "should be instantiated within the prepare method of '" + boltId + " at the earliest.", wrapperCause);
            }
            throw wrapperCause;
        }
    }
    for (String spoutId : _spouts.keySet()) {
        IRichSpout spout = _spouts.get(spoutId);
        ComponentCommon common = getComponentCommon(spoutId, spout);
        try {
            spoutSpecs.put(spoutId, new SpoutSpec(ComponentObject.serialized_java(Utils.javaSerialize(spout)), common));
        } catch (RuntimeException wrapperCause) {
            if (wrapperCause.getCause() != null && NotSerializableException.class.equals(wrapperCause.getCause().getClass())) {
                throw new IllegalStateException("Spout '" + spoutId + "' contains a non-serializable field of type " + wrapperCause.getCause().getMessage() + ", " + "which was instantiated prior to topology creation. " + wrapperCause.getCause().getMessage() + " " + "should be instantiated within the prepare method of '" + spoutId + " at the earliest.", wrapperCause);
            }
            throw wrapperCause;
        }
    }
    StormTopology stormTopology = new StormTopology(spoutSpecs, boltSpecs, new HashMap<String, StateSpoutSpec>());
    //stormTopology.set_worker_hooks(_workerHooks);
    return stormTopology;
}

Example 3 with StateSpoutSpec

use of backtype.storm.generated.StateSpoutSpec in project jstorm by alibaba.

the class NimbusUtils method normalizeTopology.

/**
     * finalize component's task parallism
     *
     * @param stormConf storm conf
     * @param topology  storm topology
     * @param fromConf  means if the paralism is read from conf file instead of reading from topology code
     * @return normalized topology
     */
public static StormTopology normalizeTopology(Map stormConf, StormTopology topology, boolean fromConf) {
    StormTopology ret = topology.deepCopy();
    Map<String, Object> rawComponents = ThriftTopologyUtils.getComponents(topology);
    Map<String, Object> components = ThriftTopologyUtils.getComponents(ret);
    if (!rawComponents.keySet().equals(components.keySet())) {
        String errMsg = "Failed to normalize topology binary, maybe due to wrong dependency";
        LOG.info(errMsg + " raw components:" + rawComponents.keySet() + ", normalized " + components.keySet());
        throw new InvalidParameterException(errMsg);
    }
    for (Entry<String, Object> entry : components.entrySet()) {
        Object component = entry.getValue();
        String componentName = entry.getKey();
        ComponentCommon common = null;
        if (component instanceof Bolt) {
            common = ((Bolt) component).get_common();
            if (fromConf) {
                Integer paraNum = ConfigExtension.getBoltParallelism(stormConf, componentName);
                if (paraNum != null) {
                    LOG.info("Set " + componentName + " as " + paraNum);
                    common.set_parallelism_hint(paraNum);
                }
            }
        }
        if (component instanceof SpoutSpec) {
            common = ((SpoutSpec) component).get_common();
            if (fromConf) {
                Integer paraNum = ConfigExtension.getSpoutParallelism(stormConf, componentName);
                if (paraNum != null) {
                    LOG.info("Set " + componentName + " as " + paraNum);
                    common.set_parallelism_hint(paraNum);
                }
            }
        }
        if (component instanceof StateSpoutSpec) {
            common = ((StateSpoutSpec) component).get_common();
            if (fromConf) {
                Integer paraNum = ConfigExtension.getSpoutParallelism(stormConf, componentName);
                if (paraNum != null) {
                    LOG.info("Set " + componentName + " as " + paraNum);
                    common.set_parallelism_hint(paraNum);
                }
            }
        }
        Map componentMap = new HashMap();
        String jsonConfString = common.get_json_conf();
        if (jsonConfString != null) {
            componentMap.putAll((Map) JStormUtils.from_json(jsonConfString));
        }
        Integer taskNum = componentParalism(stormConf, common);
        componentMap.put(Config.TOPOLOGY_TASKS, taskNum);
        // change the executor's task number
        common.set_parallelism_hint(taskNum);
        LOG.info("Set " + componentName + " parallelism " + taskNum);
        common.set_json_conf(JStormUtils.to_json(componentMap));
    }
    return ret;
}