Examples with Config org.apache.heron.api.Config used on opensource projects

Example 26 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class MultiStageAckingTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new RuntimeException("Please specify the name of the topology");
    }
    TopologyBuilder builder = new TopologyBuilder();
    int parallelism = 2;
    builder.setSpout("word", new AckingTestWordSpout(), parallelism);
    builder.setBolt("exclaim1", new ExclamationBolt(true), parallelism).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(false), parallelism).shuffleGrouping("exclaim1");
    Config conf = new Config();
    conf.setDebug(true);
    // Put an arbitrary large number here if you don't want to slow the topology down
    conf.setMaxSpoutPending(1000 * 1000 * 1000);
    // To enable acking, we need to setEnableAcking true
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.ATLEAST_ONCE);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // component resource configuration
    conf.setComponentRam("word", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim2", ExampleResources.getComponentRam());
    // container resource configuration
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(3 * parallelism, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(3 * parallelism, parallelism));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(3 * parallelism, parallelism));
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(parallelism);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 27 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class TaskHookTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new AckingTestWordSpout(), 2);
    builder.setBolt("count", new CountBolt(), 2).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    // Put an arbitrary large number here if you don't want to slow the topology down
    conf.setMaxSpoutPending(1000 * 1000 * 1000);
    // To enable acking, we need to setEnableAcking true
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.ATLEAST_ONCE);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // Set the task hook
    List<String> taskHooks = new LinkedList<>();
    taskHooks.add("org.apache.heron.examples.TaskHookTopology$TestTaskHook");
    conf.setAutoTaskHooks(taskHooks);
    // component resource configuration
    conf.setComponentRam("word", ByteAmount.fromMegabytes(512));
    conf.setComponentRam("count", ByteAmount.fromMegabytes(512));
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(2));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerCpuRequested(3);
    conf.setNumStmgrs(2);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 28 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class ConfigBuilderTest method testBuildConfig_IncorrectComponentJVMOptions_ExceptionThrown.

@Test(expected = IllegalArgumentException.class)
public void testBuildConfig_IncorrectComponentJVMOptions_ExceptionThrown() throws Exception {
    Config config = null;
    try {
        EcoParser ecoParser = new EcoParser();
        InputStream inputStream = new ByteArrayInputStream(INCORRECT_JVM_OPTIONS_CONFIG.getBytes());
        FileInputStream mockPropsStream = PowerMockito.mock(FileInputStream.class);
        EcoTopologyDefinition ecoTopologyDefinition = ecoParser.parseFromInputStream(inputStream, mockPropsStream, false);
        config = subject.buildConfig(ecoTopologyDefinition);
    } finally {
        assertNull(config);
    }
}

Example 29 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class StatefulWordCountTopology method main.

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 1) {
        throw new RuntimeException("Specify topology name");
    }
    int parallelism = 1;
    if (args.length > 1) {
        parallelism = Integer.parseInt(args[1]);
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism).fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
    conf.setTopologyStatefulCheckpointIntervalSecs(20);
    // configure component resources
    conf.setComponentRam("word", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
    conf.setComponentRam("consumer", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
    // configure container resources
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(2 * parallelism, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(4 * parallelism, parallelism));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(2 * parallelism, parallelism));
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 30 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class WindowedWordCountTopology method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    int parallelism = 1;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("sentence", new SentenceSpout(), parallelism);
    builder.setBolt("split", new SplitSentence(), parallelism).shuffleGrouping("sentence");
    builder.setBolt("consumer", new WindowSumBolt().withWindow(BaseWindowedBolt.Count.of(10000), BaseWindowedBolt.Count.of(5000)), parallelism).fieldsGrouping("split", new Fields("word"));
    Config conf = new Config();
    conf.setMaxSpoutPending(1000000);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}