use of org.apache.flink.storm.util.NullTerminatingSpout in project flink by apache.
the class SingleJoinExample method main.
public static void main(String[] args) throws Exception {
final FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender", "hobbies"));
final FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));
Config conf = new Config();
TopologyBuilder builder = new TopologyBuilder();
// only required to stabilize integration test
conf.put(FlinkLocalCluster.SUBMIT_BLOCKING, true);
final NullTerminatingSpout finalGenderSpout = new NullTerminatingSpout(genderSpout);
final NullTerminatingSpout finalAgeSpout = new NullTerminatingSpout(ageSpout);
builder.setSpout("gender", finalGenderSpout);
builder.setSpout("age", finalAgeSpout);
builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id")).fieldsGrouping("age", new Fields("id"));
// emit result
if (args.length > 0) {
// read the text file from given input path
builder.setBolt("fileOutput", new BoltFileSink(args[0], new TupleOutputFormatter())).shuffleGrouping("join");
} else {
builder.setBolt("print", new PrinterBolt()).shuffleGrouping("join");
}
String[] hobbies = new String[] { "reading", "biking", "travelling", "watching tv" };
for (int i = 0; i < 10; i++) {
String gender;
if (i % 2 == 0) {
gender = "male";
} else {
gender = "female";
}
genderSpout.feed(new Values(i, gender, hobbies[i % hobbies.length]));
}
for (int i = 9; i >= 0; i--) {
ageSpout.feed(new Values(i, i + 20));
}
final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster();
cluster.submitTopology("joinTopology", conf, FlinkTopology.createTopology(builder));
cluster.shutdown();
}
use of org.apache.flink.storm.util.NullTerminatingSpout in project flink by apache.
the class WordCountTopology method buildTopology.
public static TopologyBuilder buildTopology(boolean indexOrName) {
final TopologyBuilder builder = new TopologyBuilder();
// get input data
if (fileInputOutput) {
// read the text file from given input path
final String[] tokens = textPath.split(":");
final String inputFile = tokens[tokens.length - 1];
// inserting NullTerminatingSpout only required to stabilize integration test
builder.setSpout(spoutId, new NullTerminatingSpout(new WordCountFileSpout(inputFile)));
} else {
builder.setSpout(spoutId, new WordCountInMemorySpout());
}
if (indexOrName) {
// split up the lines in pairs (2-tuples) containing: (word,1)
builder.setBolt(tokenierzerId, new BoltTokenizer(), 4).shuffleGrouping(spoutId);
// group by the tuple field "0" and sum up tuple field "1"
builder.setBolt(counterId, new BoltCounter(), 4).fieldsGrouping(tokenierzerId, new Fields(BoltTokenizer.ATTRIBUTE_WORD));
} else {
// split up the lines in pairs (2-tuples) containing: (word,1)
builder.setBolt(tokenierzerId, new BoltTokenizerByName(), 4).shuffleGrouping(spoutId);
// group by the tuple field "0" and sum up tuple field "1"
builder.setBolt(counterId, new BoltCounterByName(), 4).fieldsGrouping(tokenierzerId, new Fields(BoltTokenizerByName.ATTRIBUTE_WORD));
}
// emit result
if (fileInputOutput) {
// read the text file from given input path
final String[] tokens = outputPath.split(":");
final String outputFile = tokens[tokens.length - 1];
builder.setBolt(sinkId, new BoltFileSink(outputFile, formatter)).shuffleGrouping(counterId);
} else {
builder.setBolt(sinkId, new BoltPrintSink(formatter), 4).shuffleGrouping(counterId);
}
return builder;
}
Aggregations