Examples with Config com.twitter.heron.api.Config used on opensource projects

Example 76 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class WordCountTopology method main.

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 1) {
        throw new RuntimeException("Specify topology name");
    }
    int parallelism = 1;
    if (args.length > 1) {
        parallelism = Integer.parseInt(args[1]);
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism).fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
    // configure component resources
    conf.setComponentRam("word", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB * 2));
    conf.setComponentRam("consumer", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB * 2));
    // configure container resources
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(2 * parallelism, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(2 * parallelism, parallelism));
    conf.setContainerCpuRequested(2);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 77 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class SlidingWindowTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(BaseWindowedBolt.Count.of(30), BaseWindowedBolt.Count.of(10)), 1).shuffleGrouping("integer");
    builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(BaseWindowedBolt.Count.of(3)), 1).shuffleGrouping("slidingsum");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    conf.setComponentRam("integer", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("slidingsum", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("tumblingavg", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("printer", ByteAmount.fromGigabytes(1));
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(5));
    conf.setContainerCpuRequested(4);
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    HeronSubmitter.submitTopology(topoName, conf, builder.createTopology());
}

Example 78 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class EcoSubmitterTest method submitTopology_AllGood_BehavesAsExpected.

@Test
public void submitTopology_AllGood_BehavesAsExpected() throws Exception {
    Config config = new Config();
    StormTopology topology = new StormTopology();
    PowerMockito.spy(StormSubmitter.class);
    PowerMockito.doNothing().when(StormSubmitter.class, "submitTopology", any(String.class), any(Config.class), any(StormTopology.class));
    subject.submitTopology("name", config, topology);
    PowerMockito.verifyStatic(times(1));
    StormSubmitter.submitTopology(anyString(), any(Config.class), any(StormTopology.class));
}

Example 79 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class ComponentJVMOptionsTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 2);
    builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word").addConfiguration("test-config", // Sample adding component-specific config
    "test-key");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    // TOPOLOGY_WORKER_CHILDOPTS will be a global one
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // For each component, both the global and if any the component one will be appended.
    // And the component one will take precedence
    conf.setComponentJvmOptions("word", "-XX:NewSize=300m");
    conf.setComponentJvmOptions("exclaim1", "-XX:NewSize=300m");
    // component resource configuration
    conf.setComponentCpu("word", 0.5);
    conf.setComponentRam("word", ByteAmount.fromMegabytes(512));
    conf.setComponentDisk("word", ByteAmount.fromMegabytes(512));
    conf.setComponentCpu("exclaim1", 0.5);
    conf.setComponentRam("exclaim1", ByteAmount.fromMegabytes(512));
    conf.setComponentDisk("exclaim1", ByteAmount.fromMegabytes(512));
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(2));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(2));
    conf.setContainerCpuRequested(2);
    // Specify the size of ram padding to per container.
    // Notice, this config will be considered as a hint,
    // and it's up to the packing algorithm to determine whether to apply this hint
    conf.setContainerRamPadding(ByteAmount.fromGigabytes(2));
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(2);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 80 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class WindowedWordCountTopology method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    int parallelism = 1;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("sentence", new SentenceSpout(), parallelism);
    builder.setBolt("split", new SplitSentence(), parallelism).shuffleGrouping("sentence");
    builder.setBolt("consumer", new WindowSumBolt().withWindow(BaseWindowedBolt.Count.of(10000), BaseWindowedBolt.Count.of(5000)), parallelism).fieldsGrouping("split", new Fields("word"));
    Config conf = new Config();
    conf.setMaxSpoutPending(1000000);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}