Examples with TopologyBuilder org.apache.heron.api.topology.TopologyBuilder used on opensource projects

Example 11 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
 * Create Topology proto object using HeronSubmitter API.
 *
 * @param heronConfig desired config params.
 * @param spouts spoutName -> parallelism
 * @param bolts boltName -> parallelism
 * @param connections connect default stream from value to key.
 * @return topology proto.
 */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 12 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class ComponentConfigTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 2).addConfiguration(Config.TOPOLOGY_COMPONENT_OUTPUT_BPS, 1000);
    builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    // component resource configuration
    conf.setComponentCpu("word", 0.5);
    conf.setComponentRam("word", ByteAmount.fromMegabytes(512));
    conf.setComponentDisk("word", ByteAmount.fromMegabytes(512));
    conf.setComponentCpu("exclaim1", 0.5);
    conf.setComponentRam("exclaim1", ByteAmount.fromMegabytes(512));
    conf.setComponentDisk("exclaim1", ByteAmount.fromMegabytes(512));
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(2));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerCpuRequested(2);
    // Specify the size of RAM padding to per container.
    // Notice, this config will be considered as a hint,
    // and it's up to the packing algorithm to determine whether to apply this hint
    conf.setContainerRamPadding(ByteAmount.fromGigabytes(2));
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(2);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 13 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class CustomGroupingTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 2);
    builder.setBolt("mybolt", new MyBolt(), 2).customGrouping("word", new MyCustomStreamGrouping());
    Config conf = new Config();
    // component resource configuration
    conf.setComponentRam("word", ByteAmount.fromMegabytes(512));
    conf.setComponentRam("mybolt", ByteAmount.fromMegabytes(512));
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(2));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerCpuRequested(2);
    conf.setNumStmgrs(2);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 14 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class MultiStageAckingTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new RuntimeException("Please specify the name of the topology");
    }
    TopologyBuilder builder = new TopologyBuilder();
    int parallelism = 2;
    builder.setSpout("word", new AckingTestWordSpout(), parallelism);
    builder.setBolt("exclaim1", new ExclamationBolt(true), parallelism).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(false), parallelism).shuffleGrouping("exclaim1");
    Config conf = new Config();
    conf.setDebug(true);
    // Put an arbitrary large number here if you don't want to slow the topology down
    conf.setMaxSpoutPending(1000 * 1000 * 1000);
    // To enable acking, we need to setEnableAcking true
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.ATLEAST_ONCE);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // component resource configuration
    conf.setComponentRam("word", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim2", ExampleResources.getComponentRam());
    // container resource configuration
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(3 * parallelism, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(3 * parallelism, parallelism));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(3 * parallelism, parallelism));
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(parallelism);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 15 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class TaskHookTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new AckingTestWordSpout(), 2);
    builder.setBolt("count", new CountBolt(), 2).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    // Put an arbitrary large number here if you don't want to slow the topology down
    conf.setMaxSpoutPending(1000 * 1000 * 1000);
    // To enable acking, we need to setEnableAcking true
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.ATLEAST_ONCE);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // Set the task hook
    List<String> taskHooks = new LinkedList<>();
    taskHooks.add("org.apache.heron.examples.TaskHookTopology$TestTaskHook");
    conf.setAutoTaskHooks(taskHooks);
    // component resource configuration
    conf.setComponentRam("word", ByteAmount.fromMegabytes(512));
    conf.setComponentRam("count", ByteAmount.fromMegabytes(512));
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(2));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerCpuRequested(3);
    conf.setNumStmgrs(2);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}