Examples with SpoutSpec org.apache.storm.generated.SpoutSpec used on opensource projects

Example 21 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project flink by apache.

the class WrapperSetupHelper method createTopologyContext.

/**
	 * Creates a {@link TopologyContext} for a Spout or Bolt instance (ie, Flink task / Storm executor).
	 * 
	 * @param context
	 *            The Flink runtime context.
	 * @param spoutOrBolt
	 *            The Spout or Bolt this context is created for.
	 * @param stormTopology
	 *            The original Storm topology.
	 * @param stormConfig
	 *            The user provided configuration.
	 * @return The created {@link TopologyContext}.
	 */
@SuppressWarnings({ "rawtypes", "unchecked" })
static synchronized TopologyContext createTopologyContext(final StreamingRuntimeContext context, final IComponent spoutOrBolt, final String operatorName, StormTopology stormTopology, final Map stormConfig) {
    final int dop = context.getNumberOfParallelSubtasks();
    final Map<Integer, String> taskToComponents = new HashMap<Integer, String>();
    final Map<String, List<Integer>> componentToSortedTasks = new HashMap<String, List<Integer>>();
    final Map<String, Map<String, Fields>> componentToStreamToFields = new HashMap<String, Map<String, Fields>>();
    String stormId = (String) stormConfig.get(TOPOLOGY_NAME);
    // not supported
    String codeDir = null;
    // not supported
    String pidDir = null;
    Integer taskId = -1;
    // not supported
    Integer workerPort = null;
    List<Integer> workerTasks = new ArrayList<Integer>();
    final Map<String, Object> defaultResources = new HashMap<String, Object>();
    final Map<String, Object> userResources = new HashMap<String, Object>();
    final Map<String, Object> executorData = new HashMap<String, Object>();
    final Map registeredMetrics = new HashMap();
    Atom openOrPrepareWasCalled = null;
    if (stormTopology == null) {
        // embedded mode
        ComponentCommon common = new ComponentCommon();
        common.set_parallelism_hint(dop);
        HashMap<String, SpoutSpec> spouts = new HashMap<String, SpoutSpec>();
        HashMap<String, Bolt> bolts = new HashMap<String, Bolt>();
        if (spoutOrBolt instanceof IRichSpout) {
            spouts.put(operatorName, new SpoutSpec(null, common));
        } else {
            assert (spoutOrBolt instanceof IRichBolt);
            bolts.put(operatorName, new Bolt(null, common));
        }
        stormTopology = new StormTopology(spouts, bolts, new HashMap<String, StateSpoutSpec>());
        List<Integer> sortedTasks = new ArrayList<Integer>(dop);
        for (int i = 1; i <= dop; ++i) {
            taskToComponents.put(i, operatorName);
            sortedTasks.add(i);
        }
        componentToSortedTasks.put(operatorName, sortedTasks);
        SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer();
        spoutOrBolt.declareOutputFields(declarer);
        componentToStreamToFields.put(operatorName, declarer.outputStreams);
    } else {
        // whole topology is built (i.e. FlinkTopology is used)
        Map<String, SpoutSpec> spouts = stormTopology.get_spouts();
        Map<String, Bolt> bolts = stormTopology.get_bolts();
        Map<String, StateSpoutSpec> stateSpouts = stormTopology.get_state_spouts();
        tid = 1;
        for (Entry<String, SpoutSpec> spout : spouts.entrySet()) {
            Integer rc = processSingleOperator(spout.getKey(), spout.getValue().get_common(), operatorName, context.getIndexOfThisSubtask(), dop, taskToComponents, componentToSortedTasks, componentToStreamToFields);
            if (rc != null) {
                taskId = rc;
            }
        }
        for (Entry<String, Bolt> bolt : bolts.entrySet()) {
            Integer rc = processSingleOperator(bolt.getKey(), bolt.getValue().get_common(), operatorName, context.getIndexOfThisSubtask(), dop, taskToComponents, componentToSortedTasks, componentToStreamToFields);
            if (rc != null) {
                taskId = rc;
            }
        }
        for (Entry<String, StateSpoutSpec> stateSpout : stateSpouts.entrySet()) {
            Integer rc = processSingleOperator(stateSpout.getKey(), stateSpout.getValue().get_common(), operatorName, context.getIndexOfThisSubtask(), dop, taskToComponents, componentToSortedTasks, componentToStreamToFields);
            if (rc != null) {
                taskId = rc;
            }
        }
        assert (taskId != null);
    }
    if (!stormConfig.containsKey(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
        // Storm default value
        stormConfig.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 30);
    }
    return new FlinkTopologyContext(stormTopology, stormConfig, taskToComponents, componentToSortedTasks, componentToStreamToFields, stormId, codeDir, pidDir, taskId, workerPort, workerTasks, defaultResources, userResources, executorData, registeredMetrics, openOrPrepareWasCalled);
}

Example 22 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project incubator-heron by apache.

the class GeneralTopologyContext method getRawTopology.

/**
 * Gets the Thrift object representing the topology.
 *
 * @return the Thrift definition representing the topology
 */
@SuppressWarnings("deprecation")
public StormTopology getRawTopology() {
    StormTopology stormTopology = new StormTopology();
    Map<String, SpoutSpec> spouts = new HashMap<>();
    for (TopologyAPI.Spout spout : this.delegate.getRawTopology().getSpoutsList()) {
        spouts.put(spout.getComp().getName(), new SpoutSpec(spout));
    }
    Map<String, Bolt> bolts = new HashMap<>();
    for (TopologyAPI.Bolt bolt : this.delegate.getRawTopology().getBoltsList()) {
        bolts.put(bolt.getComp().getName(), new Bolt(bolt));
    }
    stormTopology.set_spouts(spouts);
    stormTopology.set_bolts(bolts);
    return stormTopology;
}

Example 23 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project heron by twitter.

the class GeneralTopologyContext method getRawTopology.

/**
 * Gets the Thrift object representing the topology.
 *
 * @return the Thrift definition representing the topology
 */
@SuppressWarnings("deprecation")
public StormTopology getRawTopology() {
    StormTopology stormTopology = new StormTopology();
    Map<String, SpoutSpec> spouts = new HashMap<>();
    for (TopologyAPI.Spout spout : this.delegate.getRawTopology().getSpoutsList()) {
        spouts.put(spout.getComp().getName(), new SpoutSpec(spout));
    }
    Map<String, Bolt> bolts = new HashMap<>();
    for (TopologyAPI.Bolt bolt : this.delegate.getRawTopology().getBoltsList()) {
        bolts.put(bolt.getComp().getName(), new Bolt(bolt));
    }
    stormTopology.set_spouts(spouts);
    stormTopology.set_bolts(bolts);
    return stormTopology;
}

Example 24 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project storm by apache.

the class ThriftTopologyUtils method getComponentCommon.

public static ComponentCommon getComponentCommon(StormTopology topology, String componentId) {
    Bolt b = topology.get_bolts().get(componentId);
    if (b != null) {
        return b.get_common();
    }
    SpoutSpec s = topology.get_spouts().get(componentId);
    if (s != null) {
        return s.get_common();
    }
    StateSpoutSpec ss = topology.get_state_spouts().get(componentId);
    if (ss != null) {
        return ss.get_common();
    }
    throw new IllegalArgumentException("Could not find component common for " + componentId);
}

Example 25 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project storm by apache.

the class TopologyDetails method computeComponentMap.

/**
 * Compute a map of user topology specific components.
 *
 * @param topology                 userTopology
 * @return a map of Component Id to Component
 */
private Map<String, Component> computeComponentMap(StormTopology topology) {
    Map<String, Component> ret = new HashMap<>();
    Map<String, SpoutSpec> spouts = topology.get_spouts();
    Map<String, Bolt> bolts = topology.get_bolts();
    if (spouts != null) {
        for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) {
            String compId = entry.getKey();
            SpoutSpec spout = entry.getValue();
            if (!Utils.isSystemId(compId)) {
                Component comp = new Component(ComponentType.SPOUT, compId, componentToExecs(compId), spout.get_common().get_inputs());
                ret.put(compId, comp);
            }
        }
    }
    if (bolts != null) {
        for (Map.Entry<String, Bolt> entry : bolts.entrySet()) {
            String compId = entry.getKey();
            Bolt bolt = entry.getValue();
            if (!Utils.isSystemId(compId)) {
                Component comp = new Component(ComponentType.BOLT, compId, componentToExecs(compId), bolt.get_common().get_inputs());
                ret.put(compId, comp);
            }
        }
    }
    // Link the components together
    if (spouts != null) {
        for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) {
            Component spout = ret.get(entry.getKey());
            for (String parentId : getInputsTo(entry.getValue().get_common())) {
                ret.get(parentId).addChild(spout);
            }
        }
    }
    if (bolts != null) {
        for (Map.Entry<String, Bolt> entry : bolts.entrySet()) {
            Component bolt = ret.get(entry.getKey());
            for (String parentId : getInputsTo(entry.getValue().get_common())) {
                ret.get(parentId).addChild(bolt);
            }
        }
    }
    return ret;
}