use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.
the class StatefulWordCountTopology method main.
/**
* Main method
*/
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
if (args.length < 1) {
throw new RuntimeException("Specify topology name");
}
int parallelism = 1;
if (args.length > 1) {
parallelism = Integer.parseInt(args[1]);
}
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("word", new WordSpout(), parallelism);
builder.setBolt("consumer", new ConsumerBolt(), parallelism).fieldsGrouping("word", new Fields("word"));
Config conf = new Config();
conf.setNumStmgrs(parallelism);
conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
conf.setTopologyStatefulCheckpointIntervalSecs(20);
// configure component resources
conf.setComponentRam("word", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
conf.setComponentRam("consumer", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
// configure container resources
conf.setContainerDiskRequested(ExampleResources.getContainerDisk(2 * parallelism, parallelism));
conf.setContainerRamRequested(ExampleResources.getContainerRam(4 * parallelism, parallelism));
conf.setContainerCpuRequested(ExampleResources.getContainerCpu(2 * parallelism, parallelism));
HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.
the class WindowedWordCountTopology method main.
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
int parallelism = 1;
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("sentence", new SentenceSpout(), parallelism);
builder.setBolt("split", new SplitSentence(), parallelism).shuffleGrouping("sentence");
builder.setBolt("consumer", new WindowSumBolt().withWindow(BaseWindowedBolt.Count.of(10000), BaseWindowedBolt.Count.of(5000)), parallelism).fieldsGrouping("split", new Fields("word"));
Config conf = new Config();
conf.setMaxSpoutPending(1000000);
HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.
the class MultiSpoutExclamationTopology method main.
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("word0", new TestWordSpout(), 2);
builder.setSpout("word1", new TestWordSpout(), 2);
builder.setSpout("word2", new TestWordSpout(), 2);
builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word0").shuffleGrouping("word1").shuffleGrouping("word2");
Config conf = new Config();
conf.setDebug(true);
conf.setMaxSpoutPending(10);
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
// component resource configuration
conf.setComponentRam("word0", ExampleResources.getComponentRam());
conf.setComponentRam("word1", ExampleResources.getComponentRam());
conf.setComponentRam("word2", ExampleResources.getComponentRam());
conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
// container resource configuration
conf.setContainerDiskRequested(ByteAmount.fromGigabytes(3));
conf.setContainerRamRequested(ByteAmount.fromGigabytes(5));
conf.setContainerCpuRequested(4);
if (args != null && args.length > 0) {
conf.setNumStmgrs(3);
HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
} else {
Simulator simulator = new Simulator();
simulator.submitTopology("test", conf, builder.createTopology());
Utils.sleep(10000);
simulator.killTopology("test");
simulator.shutdown();
}
}
use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.
the class StatefulSlidingWindowTopology method main.
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("integer", new IntegerSpout(), 1);
builder.setBolt("sumbolt", new WindowSumBolt().withWindow(BaseWindowedBolt.Count.of(5), BaseWindowedBolt.Count.of(3)), 1).shuffleGrouping("integer");
builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("sumbolt");
Config conf = new Config();
conf.setDebug(true);
String topoName = "test";
Config.setComponentRam(conf, "integer", ByteAmount.fromGigabytes(1));
Config.setComponentRam(conf, "sumbolt", ByteAmount.fromGigabytes(1));
Config.setComponentRam(conf, "printer", ByteAmount.fromGigabytes(1));
Config.setContainerDiskRequested(conf, ByteAmount.fromGigabytes(5));
Config.setContainerCpuRequested(conf, 4);
conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
conf.setTopologyStatefulCheckpointIntervalSecs(20);
conf.setMaxSpoutPending(1000);
if (args != null && args.length > 0) {
topoName = args[0];
}
HeronSubmitter.submitTopology(topoName, conf, builder.createTopology());
}
use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.
the class TopologyManagerTest method getTestTopology.
/**
* Construct the test topology
*/
public static TopologyAPI.Topology getTestTopology() {
TopologyBuilder topologyBuilder = new TopologyBuilder();
topologyBuilder.setSpout(STREAM_ID, new BaseRichSpout() {
private static final long serialVersionUID = 5406114907377311020L;
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields(STREAM_ID));
}
@Override
public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
}
@Override
public void nextTuple() {
}
}, 2);
topologyBuilder.setBolt(BOLT_ID, new BaseBasicBolt() {
private static final long serialVersionUID = 4398578755681473899L;
@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
}
}, 2).shuffleGrouping(STREAM_ID);
Config conf = new Config();
conf.setDebug(true);
conf.setMaxSpoutPending(10);
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
conf.setComponentRam(STREAM_ID, ByteAmount.fromMegabytes(500));
conf.setComponentRam(BOLT_ID, ByteAmount.fromGigabytes(1));
conf.setMessageTimeoutSecs(1);
return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}
Aggregations