Examples with StreamInfo org.apache.storm.generated.StreamInfo used on opensource projects

Example 6 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class StormCommon method addAcker.

@SuppressWarnings("unchecked")
public static void addAcker(Map conf, StormTopology topology) {
    int ackerNum = Utils.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), Utils.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
    Map<GlobalStreamId, Grouping> inputs = ackerInputs(topology);
    Map<String, StreamInfo> outputStreams = new HashMap<String, StreamInfo>();
    outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    Map<String, Object> ackerConf = new HashMap<>();
    ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum);
    ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
    Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf);
    for (Bolt bolt : topology.get_bolts().values()) {
        ComponentCommon common = bolt.get_common();
        common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val")));
        common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
        common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
    }
    for (SpoutSpec spout : topology.get_spouts().values()) {
        ComponentCommon common = spout.get_common();
        Map spoutConf = componentConf(spout);
        spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
        common.set_json_conf(JSONValue.toJSONString(spoutConf));
        common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task")));
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping());
    }
    topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
}

Example 7 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class Node method getOutputFields.

static Fields getOutputFields(IComponent component, String streamId) {
    OutputFieldsGetter getter = new OutputFieldsGetter();
    component.declareOutputFields(getter);
    Map<String, StreamInfo> fieldsDeclaration = getter.getFieldsDeclaration();
    if ((fieldsDeclaration != null) && fieldsDeclaration.containsKey(streamId)) {
        return new Fields(fieldsDeclaration.get(streamId).get_output_fields());
    }
    return new Fields();
}

Example 8 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project flink by apache.

the class WrapperSetupHelper method processSingleOperator.

/**
	 * Sets up {@code taskToComponents}, {@code componentToSortedTasks}, and {@code componentToStreamToFields} for a
	 * single instance of a Spout or Bolt (ie, task or executor). Furthermore, is computes the unique task-id.
	 * 
	 * @param componentId
	 *            The ID of the Spout/Bolt in the topology.
	 * @param common
	 *            The common operator object (that is all Spouts and Bolts have).
	 * @param operatorName
	 *            The Flink operator name.
	 * @param index
	 *            The index of the currently processed tasks with its operator.
	 * @param dop
	 *            The parallelism of the operator.
	 * @param taskToComponents
	 *            OUTPUT: A map from all task IDs of the topology to their component IDs.
	 * @param componentToSortedTasks
	 *            OUTPUT: A map from all component IDs to their sorted list of corresponding task IDs.
	 * @param componentToStreamToFields
	 *            OUTPUT: A map from all component IDs to there output streams and output fields.
	 * 
	 * @return A unique task ID if the currently processed Spout or Bolt ({@code componentId}) is equal to the current
	 *         Flink operator {@code operatorName} -- {@code null} otherwise.
	 */
private static Integer processSingleOperator(final String componentId, final ComponentCommon common, final String operatorName, final int index, final int dop, final Map<Integer, String> taskToComponents, final Map<String, List<Integer>> componentToSortedTasks, final Map<String, Map<String, Fields>> componentToStreamToFields) {
    final int parallelism_hint = common.get_parallelism_hint();
    Integer taskId = null;
    if (componentId.equals(operatorName)) {
        taskId = tid + index;
    }
    List<Integer> sortedTasks = new ArrayList<Integer>(dop);
    for (int i = 0; i < parallelism_hint; ++i) {
        taskToComponents.put(tid, componentId);
        sortedTasks.add(tid);
        ++tid;
    }
    componentToSortedTasks.put(componentId, sortedTasks);
    Map<String, Fields> outputStreams = new HashMap<String, Fields>();
    for (Entry<String, StreamInfo> outStream : common.get_streams().entrySet()) {
        outputStreams.put(outStream.getKey(), new Fields(outStream.getValue().get_output_fields()));
    }
    componentToStreamToFields.put(componentId, outputStreams);
    return taskId;
}

Example 9 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class StormCommon method addSystemComponents.

@SuppressWarnings("unused")
public static void addSystemComponents(Map conf, StormTopology topology) {
    Map<String, StreamInfo> outputStreams = new HashMap<>();
    outputStreams.put(Constants.SYSTEM_TICK_STREAM_ID, Thrift.outputFields(Arrays.asList("rate_secs")));
    outputStreams.put(Constants.METRICS_TICK_STREAM_ID, Thrift.outputFields(Arrays.asList("interval")));
    outputStreams.put(Constants.CREDENTIALS_CHANGED_STREAM_ID, Thrift.outputFields(Arrays.asList("creds")));
    Map<String, Object> boltConf = new HashMap<>();
    boltConf.put(Config.TOPOLOGY_TASKS, 0);
    Bolt systemBoltSpec = Thrift.prepareSerializedBoltDetails(null, new SystemBolt(), outputStreams, 0, boltConf);
    topology.put_to_bolts(Constants.SYSTEM_COMPONENT_ID, systemBoltSpec);
}

Example 10 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class StormCommon method addMetricStreams.

public static void addMetricStreams(StormTopology topology) {
    for (Object component : allComponents(topology).values()) {
        ComponentCommon common = getComponentCommon(component);
        StreamInfo streamInfo = Thrift.outputFields(Arrays.asList("task-info", "data-points"));
        common.put_to_streams(Constants.METRICS_STREAM_ID, streamInfo);
    }
}