Examples with Config org.apache.heron.api.Config used on opensource projects

Example 56 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class ExclamationTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    int parallelism = 2;
    int spouts = parallelism;
    builder.setSpout("word", new TestWordSpout(Duration.ofMillis(0)), spouts);
    int bolts = 2 * parallelism;
    builder.setBolt("exclaim1", new ExclamationBolt(), bolts).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.setMessageTimeoutSecs(600);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // resources configuration
    conf.setComponentRam("word", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(spouts + bolts, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(spouts + bolts, parallelism));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(spouts + bolts, parallelism));
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(parallelism);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        System.out.println("Topology name not provided as an argument, running in simulator mode.");
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 57 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class SlidingWindowTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(BaseWindowedBolt.Count.of(30), BaseWindowedBolt.Count.of(10)), 1).shuffleGrouping("integer");
    builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(BaseWindowedBolt.Count.of(3)), 1).shuffleGrouping("slidingsum");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    conf.setComponentRam("integer", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("slidingsum", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("tumblingavg", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("printer", ByteAmount.fromGigabytes(1));
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(5));
    conf.setContainerCpuRequested(4);
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    HeronSubmitter.submitTopology(topoName, conf, builder.createTopology());
}

Example 58 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class StatefulTumblingWindowTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new IntegerSpout(), 1);
    WindowSumBolt windowSumBolt = new WindowSumBolt();
    windowSumBolt.withTumblingWindow(Duration.ofMinutes(10));
    builder.setBolt("sumbolt", windowSumBolt, 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("sumbolt");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    Config.setComponentRam(conf, "integer", ByteAmount.fromGigabytes(1));
    Config.setComponentRam(conf, "sumbolt", ByteAmount.fromGigabytes(1));
    Config.setComponentRam(conf, "printer", ByteAmount.fromGigabytes(1));
    Config.setContainerDiskRequested(conf, ByteAmount.fromGigabytes(5));
    Config.setContainerCpuRequested(conf, 4);
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
    conf.setTopologyStatefulCheckpointIntervalSecs(20);
    conf.setMaxSpoutPending(1000);
    conf.setMessageTimeoutSecs(1500);
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    HeronSubmitter.submitTopology(topoName, conf, builder.createTopology());
}

Example 59 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class WordCountTopology method main.

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 1) {
        throw new RuntimeException("Specify topology name");
    }
    int parallelism = 1;
    if (args.length > 1) {
        parallelism = Integer.parseInt(args[1]);
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism).fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
    // configure component resources
    conf.setComponentRam("word", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
    conf.setComponentRam("consumer", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
    conf.setComponentCpu("word", 1.0);
    conf.setComponentCpu("consumer", 1.0);
    // configure container resources
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(2 * parallelism, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(2 * parallelism, parallelism));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(2 * parallelism, parallelism));
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 60 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class AckingTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }
    TopologyBuilder builder = new TopologyBuilder();
    int spouts = 2;
    int bolts = 2;
    builder.setSpout("word", new AckingTestWordSpout(), spouts);
    builder.setBolt("exclaim1", new ExclamationBolt(), bolts).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setSerializationClassName(Config.HERON_KRYO_SERIALIZER_CLASS_NAME);
    // Specifies that all tuples will be automatically failed if not acked within 10 seconds
    conf.setMessageTimeoutSecs(10);
    // Put an arbitrarily large number here if you don't want to slow the topology down
    conf.setMaxSpoutPending(1000 * 1000 * 1000);
    // To enable at-least-once delivery semantics
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.ATLEAST_ONCE);
    // Extra JVM options
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // Component resource configuration
    conf.setComponentRam("word", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
    // Container resource configuration
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(spouts + bolts, 2));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(spouts + bolts, 2));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(spouts + bolts, 2));
    // Set the number of workers or stream managers
    conf.setNumStmgrs(2);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}