Examples with Fields org.apache.storm.tuple.Fields used on opensource projects

Example 96 with Fields

use of org.apache.storm.tuple.Fields in project flink by apache.

the class WordCountTopology method buildTopology.

public static TopologyBuilder buildTopology(boolean indexOrName) {
    final TopologyBuilder builder = new TopologyBuilder();
    // get input data
    if (fileInputOutput) {
        // read the text file from given input path
        final String[] tokens = textPath.split(":");
        final String inputFile = tokens[tokens.length - 1];
        // inserting NullTerminatingSpout only required to stabilize integration test
        builder.setSpout(spoutId, new NullTerminatingSpout(new WordCountFileSpout(inputFile)));
    } else {
        builder.setSpout(spoutId, new WordCountInMemorySpout());
    }
    if (indexOrName) {
        // split up the lines in pairs (2-tuples) containing: (word,1)
        builder.setBolt(tokenierzerId, new BoltTokenizer(), 4).shuffleGrouping(spoutId);
        // group by the tuple field "0" and sum up tuple field "1"
        builder.setBolt(counterId, new BoltCounter(), 4).fieldsGrouping(tokenierzerId, new Fields(BoltTokenizer.ATTRIBUTE_WORD));
    } else {
        // split up the lines in pairs (2-tuples) containing: (word,1)
        builder.setBolt(tokenierzerId, new BoltTokenizerByName(), 4).shuffleGrouping(spoutId);
        // group by the tuple field "0" and sum up tuple field "1"
        builder.setBolt(counterId, new BoltCounterByName(), 4).fieldsGrouping(tokenierzerId, new Fields(BoltTokenizerByName.ATTRIBUTE_WORD));
    }
    // emit result
    if (fileInputOutput) {
        // read the text file from given input path
        final String[] tokens = outputPath.split(":");
        final String outputFile = tokens[tokens.length - 1];
        builder.setBolt(sinkId, new BoltFileSink(outputFile, formatter)).shuffleGrouping(counterId);
    } else {
        builder.setBolt(sinkId, new BoltPrintSink(formatter), 4).shuffleGrouping(counterId);
    }
    return builder;
}

Example 97 with Fields

use of org.apache.storm.tuple.Fields in project flink by apache.

the class WrapperSetupHelper method processSingleOperator.

/**
	 * Sets up {@code taskToComponents}, {@code componentToSortedTasks}, and {@code componentToStreamToFields} for a
	 * single instance of a Spout or Bolt (ie, task or executor). Furthermore, is computes the unique task-id.
	 * 
	 * @param componentId
	 *            The ID of the Spout/Bolt in the topology.
	 * @param common
	 *            The common operator object (that is all Spouts and Bolts have).
	 * @param operatorName
	 *            The Flink operator name.
	 * @param index
	 *            The index of the currently processed tasks with its operator.
	 * @param dop
	 *            The parallelism of the operator.
	 * @param taskToComponents
	 *            OUTPUT: A map from all task IDs of the topology to their component IDs.
	 * @param componentToSortedTasks
	 *            OUTPUT: A map from all component IDs to their sorted list of corresponding task IDs.
	 * @param componentToStreamToFields
	 *            OUTPUT: A map from all component IDs to there output streams and output fields.
	 * 
	 * @return A unique task ID if the currently processed Spout or Bolt ({@code componentId}) is equal to the current
	 *         Flink operator {@code operatorName} -- {@code null} otherwise.
	 */
private static Integer processSingleOperator(final String componentId, final ComponentCommon common, final String operatorName, final int index, final int dop, final Map<Integer, String> taskToComponents, final Map<String, List<Integer>> componentToSortedTasks, final Map<String, Map<String, Fields>> componentToStreamToFields) {
    final int parallelism_hint = common.get_parallelism_hint();
    Integer taskId = null;
    if (componentId.equals(operatorName)) {
        taskId = tid + index;
    }
    List<Integer> sortedTasks = new ArrayList<Integer>(dop);
    for (int i = 0; i < parallelism_hint; ++i) {
        taskToComponents.put(tid, componentId);
        sortedTasks.add(tid);
        ++tid;
    }
    componentToSortedTasks.put(componentId, sortedTasks);
    Map<String, Fields> outputStreams = new HashMap<String, Fields>();
    for (Entry<String, StreamInfo> outStream : common.get_streams().entrySet()) {
        outputStreams.put(outStream.getKey(), new Fields(outStream.getValue().get_output_fields()));
    }
    componentToStreamToFields.put(componentId, outputStreams);
    return taskId;
}

Example 98 with Fields

use of org.apache.storm.tuple.Fields in project flink by apache.

the class SplitBolt method declareOutputFields.

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    Fields schema = new Fields("number");
    declarer.declareStream(EVEN_STREAM, schema);
    declarer.declareStream(ODD_STREAM, schema);
}

Example 99 with Fields

use of org.apache.storm.tuple.Fields in project flink by apache.

the class WrapperSetupHelperTest method testTupleTypes.

private void testTupleTypes(final int numberOfAttributes) throws Exception {
    String[] schema;
    if (numberOfAttributes == -1) {
        schema = new String[1];
    } else {
        schema = new String[numberOfAttributes];
    }
    for (int i = 0; i < schema.length; ++i) {
        schema[i] = "a" + i;
    }
    IComponent boltOrSpout;
    if (this.r.nextBoolean()) {
        boltOrSpout = mock(IRichSpout.class);
    } else {
        boltOrSpout = mock(IRichBolt.class);
    }
    final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer();
    declarer.declare(new Fields(schema));
    PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
    HashMap<String, Integer> attributes = new HashMap<String, Integer>();
    attributes.put(Utils.DEFAULT_STREAM_ID, numberOfAttributes);
    Assert.assertEquals(attributes, WrapperSetupHelper.getNumberOfAttributes(boltOrSpout, numberOfAttributes == -1 ? new HashSet<String>(singleton(Utils.DEFAULT_STREAM_ID)) : null));
}

Example 100 with Fields

use of org.apache.storm.tuple.Fields in project flink by apache.

the class WrapperSetupHelperTest method testToManyAttributes.

@Test(expected = IllegalArgumentException.class)
public void testToManyAttributes() throws Exception {
    IComponent boltOrSpout;
    if (this.r.nextBoolean()) {
        boltOrSpout = mock(IRichSpout.class);
    } else {
        boltOrSpout = mock(IRichBolt.class);
    }
    final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer();
    final String[] schema = new String[26];
    for (int i = 0; i < schema.length; ++i) {
        schema[i] = "a" + i;
    }
    declarer.declare(new Fields(schema));
    PowerMockito.whenNew(SetupOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
    WrapperSetupHelper.getNumberOfAttributes(boltOrSpout, null);
}