use of org.apache.heron.simulator.Simulator in project heron by twitter.
the class ComponentConfigTopology method main.
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("word", new TestWordSpout(), 2).addConfiguration(Config.TOPOLOGY_COMPONENT_OUTPUT_BPS, 1000);
builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word");
Config conf = new Config();
conf.setDebug(true);
conf.setMaxSpoutPending(10);
// component resource configuration
conf.setComponentCpu("word", 0.5);
conf.setComponentRam("word", ByteAmount.fromMegabytes(512));
conf.setComponentDisk("word", ByteAmount.fromMegabytes(512));
conf.setComponentCpu("exclaim1", 0.5);
conf.setComponentRam("exclaim1", ByteAmount.fromMegabytes(512));
conf.setComponentDisk("exclaim1", ByteAmount.fromMegabytes(512));
// container resource configuration
conf.setContainerDiskRequested(ByteAmount.fromGigabytes(2));
conf.setContainerRamRequested(ByteAmount.fromGigabytes(3));
conf.setContainerCpuRequested(2);
// Specify the size of RAM padding to per container.
// Notice, this config will be considered as a hint,
// and it's up to the packing algorithm to determine whether to apply this hint
conf.setContainerRamPadding(ByteAmount.fromGigabytes(2));
if (args != null && args.length > 0) {
conf.setNumStmgrs(2);
HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
} else {
Simulator simulator = new Simulator();
simulator.submitTopology("test", conf, builder.createTopology());
Utils.sleep(10000);
simulator.killTopology("test");
simulator.shutdown();
}
}
use of org.apache.heron.simulator.Simulator in project heron by twitter.
the class MultiStageAckingTopology method main.
public static void main(String[] args) throws Exception {
if (args.length != 1) {
throw new RuntimeException("Please specify the name of the topology");
}
TopologyBuilder builder = new TopologyBuilder();
int parallelism = 2;
builder.setSpout("word", new AckingTestWordSpout(), parallelism);
builder.setBolt("exclaim1", new ExclamationBolt(true), parallelism).shuffleGrouping("word");
builder.setBolt("exclaim2", new ExclamationBolt(false), parallelism).shuffleGrouping("exclaim1");
Config conf = new Config();
conf.setDebug(true);
// Put an arbitrary large number here if you don't want to slow the topology down
conf.setMaxSpoutPending(1000 * 1000 * 1000);
// To enable acking, we need to setEnableAcking true
conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.ATLEAST_ONCE);
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
// component resource configuration
conf.setComponentRam("word", ExampleResources.getComponentRam());
conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
conf.setComponentRam("exclaim2", ExampleResources.getComponentRam());
// container resource configuration
conf.setContainerDiskRequested(ExampleResources.getContainerDisk(3 * parallelism, parallelism));
conf.setContainerRamRequested(ExampleResources.getContainerRam(3 * parallelism, parallelism));
conf.setContainerCpuRequested(ExampleResources.getContainerCpu(3 * parallelism, parallelism));
if (args != null && args.length > 0) {
conf.setNumStmgrs(parallelism);
HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
} else {
Simulator simulator = new Simulator();
simulator.submitTopology("test", conf, builder.createTopology());
Utils.sleep(10000);
simulator.killTopology("test");
simulator.shutdown();
}
}
use of org.apache.heron.simulator.Simulator in project heron by twitter.
the class MultiSpoutExclamationTopology method main.
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("word0", new TestWordSpout(), 2);
builder.setSpout("word1", new TestWordSpout(), 2);
builder.setSpout("word2", new TestWordSpout(), 2);
builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word0").shuffleGrouping("word1").shuffleGrouping("word2");
Config conf = new Config();
conf.setDebug(true);
conf.setMaxSpoutPending(10);
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
// component resource configuration
conf.setComponentRam("word0", ExampleResources.getComponentRam());
conf.setComponentRam("word1", ExampleResources.getComponentRam());
conf.setComponentRam("word2", ExampleResources.getComponentRam());
conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
// container resource configuration
conf.setContainerDiskRequested(ByteAmount.fromGigabytes(3));
conf.setContainerRamRequested(ByteAmount.fromGigabytes(5));
conf.setContainerCpuRequested(4);
if (args != null && args.length > 0) {
conf.setNumStmgrs(3);
HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
} else {
Simulator simulator = new Simulator();
simulator.submitTopology("test", conf, builder.createTopology());
Utils.sleep(10000);
simulator.killTopology("test");
simulator.shutdown();
}
}
use of org.apache.heron.simulator.Simulator in project heron by twitter.
the class HeronKafkaSpoutSampleTopology method main.
public static void main(String[] args) {
Map<String, Object> kafkaConsumerConfig = new HashMap<>();
kafkaConsumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
kafkaConsumerConfig.put(ConsumerConfig.GROUP_ID_CONFIG, "sample-kafka-spout");
kafkaConsumerConfig.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
kafkaConsumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
LOG.info("Kafka Consumer Config: {}", kafkaConsumerConfig);
KafkaConsumerFactory<String, String> kafkaConsumerFactory = new DefaultKafkaConsumerFactory<>(kafkaConsumerConfig);
TopologyBuilder topologyBuilder = new TopologyBuilder();
topologyBuilder.setSpout(KAFKA_SPOUT_NAME, new KafkaSpout<>(kafkaConsumerFactory, Collections.singletonList("test-topic")));
topologyBuilder.setBolt(LOGGING_BOLT_NAME, new LoggingBolt()).shuffleGrouping(KAFKA_SPOUT_NAME);
Config config = new Config();
config.setNumStmgrs(1);
config.setContainerCpuRequested(1);
config.setContainerRamRequested(ByteAmount.fromGigabytes(1));
config.setContainerDiskRequested(ByteAmount.fromGigabytes(1));
config.setComponentCpu(KAFKA_SPOUT_NAME, 0.25);
config.setComponentRam(KAFKA_SPOUT_NAME, ByteAmount.fromMegabytes(256));
config.setComponentDisk(KAFKA_SPOUT_NAME, ByteAmount.fromMegabytes(512));
config.setComponentCpu(LOGGING_BOLT_NAME, 0.25);
config.setComponentRam(LOGGING_BOLT_NAME, ByteAmount.fromMegabytes(256));
config.setComponentDisk(LOGGING_BOLT_NAME, ByteAmount.fromMegabytes(256));
Simulator simulator = new Simulator();
simulator.submitTopology("heron-kafka-spout-sample-topology", config, topologyBuilder.createTopology());
}
use of org.apache.heron.simulator.Simulator in project heron by twitter.
the class ExclamationTopology method main.
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
int parallelism = 2;
int spouts = parallelism;
builder.setSpout("word", new TestWordSpout(Duration.ofMillis(0)), spouts);
int bolts = 2 * parallelism;
builder.setBolt("exclaim1", new ExclamationBolt(), bolts).shuffleGrouping("word");
Config conf = new Config();
conf.setDebug(true);
conf.setMaxSpoutPending(10);
conf.setMessageTimeoutSecs(600);
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
// resources configuration
conf.setComponentRam("word", ExampleResources.getComponentRam());
conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
conf.setContainerDiskRequested(ExampleResources.getContainerDisk(spouts + bolts, parallelism));
conf.setContainerRamRequested(ExampleResources.getContainerRam(spouts + bolts, parallelism));
conf.setContainerCpuRequested(ExampleResources.getContainerCpu(spouts + bolts, parallelism));
if (args != null && args.length > 0) {
conf.setNumStmgrs(parallelism);
HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
} else {
System.out.println("Topology name not provided as an argument, running in simulator mode.");
Simulator simulator = new Simulator();
simulator.submitTopology("test", conf, builder.createTopology());
Utils.sleep(10000);
simulator.killTopology("test");
simulator.shutdown();
}
}
Aggregations