Examples with StreamInfo org.apache.storm.generated.StreamInfo used on opensource projects

Example 11 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class StormCommon method addMetricStreams.

public static void addMetricStreams(StormTopology topology) {
    for (Object component : allComponents(topology).values()) {
        ComponentCommon common = getComponentCommon(component);
        StreamInfo streamInfo = Thrift.outputFields(Arrays.asList("task-info", "data-points"));
        common.put_to_streams(Constants.METRICS_STREAM_ID, streamInfo);
    }
}

Example 12 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class StormCommon method validateStructure.

public static void validateStructure(StormTopology topology) throws InvalidTopologyException {
    Map<String, Object> componentMap = allComponents(topology);
    for (Map.Entry<String, Object> entry : componentMap.entrySet()) {
        String componentId = entry.getKey();
        ComponentCommon common = getComponentCommon(entry.getValue());
        Map<GlobalStreamId, Grouping> inputs = common.get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
            String sourceStreamId = input.getKey().get_streamId();
            String sourceComponentId = input.getKey().get_componentId();
            if (!componentMap.keySet().contains(sourceComponentId)) {
                throw new InvalidTopologyException("Component: [" + componentId + "] subscribes from non-existent component [" + sourceComponentId + "]");
            }
            ComponentCommon sourceComponent = getComponentCommon(componentMap.get(sourceComponentId));
            if (!sourceComponent.get_streams().containsKey(sourceStreamId)) {
                throw new InvalidTopologyException("Component: [" + componentId + "] subscribes from non-existent stream: " + "[" + sourceStreamId + "] of component [" + sourceComponentId + "]");
            }
            Grouping grouping = input.getValue();
            if (Thrift.groupingType(grouping) == Grouping._Fields.FIELDS) {
                List<String> fields = new ArrayList<>(grouping.get_fields());
                Map<String, StreamInfo> streams = sourceComponent.get_streams();
                Set<String> sourceOutputFields = getStreamOutputFields(streams);
                fields.removeAll(sourceOutputFields);
                if (fields.size() != 0) {
                    throw new InvalidTopologyException("Component: [" + componentId + "] subscribes from stream: [" + sourceStreamId + "] of component " + "[" + sourceComponentId + "] + with non-existent fields: " + fields);
                }
            }
        }
    }
}

Example 13 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project flink by apache.

the class WrapperSetupHelper method processSingleOperator.

/**
	 * Sets up {@code taskToComponents}, {@code componentToSortedTasks}, and {@code componentToStreamToFields} for a
	 * single instance of a Spout or Bolt (ie, task or executor). Furthermore, is computes the unique task-id.
	 * 
	 * @param componentId
	 *            The ID of the Spout/Bolt in the topology.
	 * @param common
	 *            The common operator object (that is all Spouts and Bolts have).
	 * @param operatorName
	 *            The Flink operator name.
	 * @param index
	 *            The index of the currently processed tasks with its operator.
	 * @param dop
	 *            The parallelism of the operator.
	 * @param taskToComponents
	 *            OUTPUT: A map from all task IDs of the topology to their component IDs.
	 * @param componentToSortedTasks
	 *            OUTPUT: A map from all component IDs to their sorted list of corresponding task IDs.
	 * @param componentToStreamToFields
	 *            OUTPUT: A map from all component IDs to there output streams and output fields.
	 * 
	 * @return A unique task ID if the currently processed Spout or Bolt ({@code componentId}) is equal to the current
	 *         Flink operator {@code operatorName} -- {@code null} otherwise.
	 */
private static Integer processSingleOperator(final String componentId, final ComponentCommon common, final String operatorName, final int index, final int dop, final Map<Integer, String> taskToComponents, final Map<String, List<Integer>> componentToSortedTasks, final Map<String, Map<String, Fields>> componentToStreamToFields) {
    final int parallelism_hint = common.get_parallelism_hint();
    Integer taskId = null;
    if (componentId.equals(operatorName)) {
        taskId = tid + index;
    }
    List<Integer> sortedTasks = new ArrayList<Integer>(dop);
    for (int i = 0; i < parallelism_hint; ++i) {
        taskToComponents.put(tid, componentId);
        sortedTasks.add(tid);
        ++tid;
    }
    componentToSortedTasks.put(componentId, sortedTasks);
    Map<String, Fields> outputStreams = new HashMap<String, Fields>();
    for (Entry<String, StreamInfo> outStream : common.get_streams().entrySet()) {
        outputStreams.put(outStream.getKey(), new Fields(outStream.getValue().get_output_fields()));
    }
    componentToStreamToFields.put(componentId, outputStreams);
    return taskId;
}

Example 14 with StreamInfo

use of org.apache.storm.generated.StreamInfo in project storm by apache.

the class ClojureBolt method declareOutputFields.

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    for (String stream : _fields.keySet()) {
        StreamInfo info = _fields.get(stream);
        declarer.declareStream(stream, info.is_direct(), new Fields(info.get_output_fields()));
    }
}