Examples with Config com.twitter.heron.api.Config used on opensource projects

Example 26 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class Eco method submit.

public void submit(FileInputStream fileInputStream, FileInputStream propertiesFile, boolean envFilter) throws Exception {
    EcoTopologyDefinition topologyDefinition = ecoParser.parseFromInputStream(fileInputStream, propertiesFile, envFilter);
    String topologyName = topologyDefinition.getName();
    Config topologyConfig = ecoBuilder.buildConfig(topologyDefinition);
    EcoExecutionContext executionContext = new EcoExecutionContext(topologyDefinition, topologyConfig);
    printTopologyInfo(executionContext);
    ObjectBuilder objectBuilder = new ObjectBuilder();
    objectBuilder.setBuilderUtility(new BuilderUtility());
    TopologyBuilder builder = ecoBuilder.buildTopologyBuilder(executionContext, objectBuilder);
    ecoSubmitter.submitTopology(topologyName, topologyConfig, builder.createTopology());
}

Example 27 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class ConfigBuilder method buildConfig.

protected Config buildConfig(EcoTopologyDefinition topologyDefinition) throws IllegalArgumentException {
    Map<String, Object> configMap = topologyDefinition.getConfig();
    Config config = new Config();
    for (Map.Entry<String, Object> entry : configMap.entrySet()) {
        if (entry.getKey().equals(COMPONENT_RESOURCE_MAP)) {
            setComponentLevelResource(config, entry);
        } else if (entry.getKey().equals(COMPONENT_JVM_OPTIONS)) {
            List<Object> objects = (List<Object>) entry.getValue();
            for (Object obj : objects) {
                String objString = obj.toString();
                objString = objString.replace(LEFT_BRACE, WHITESPACE);
                objString = objString.replace(RIGHT_BRACE, WHITESPACE);
                int idIndex = objString.indexOf(ID);
                int optionsIndex = objString.indexOf(OPTIONS);
                String id = getIdValue(objString, idIndex);
                String jvmOptions;
                if (optionsIndex != -1) {
                    int equalsIndex = objString.indexOf(EQUALS, optionsIndex);
                    jvmOptions = objString.substring(equalsIndex + 1, objString.length());
                    jvmOptions = jvmOptions.replace(LEFT_BRACKET, "").replace(RIGHT_BRACKET, "");
                } else {
                    throw new IllegalArgumentException("You must specify the JVM options for your component");
                }
                config.setComponentJvmOptions(id, jvmOptions);
            }
        } else {
            config.put(entry.getKey(), entry.getValue());
        }
    }
    return config;
}

Example 28 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class MultiSpoutExclamationTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word0", new TestWordSpout(), 2);
    builder.setSpout("word1", new TestWordSpout(), 2);
    builder.setSpout("word2", new TestWordSpout(), 2);
    builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word0").shuffleGrouping("word1").shuffleGrouping("word2");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // component resource configuration
    conf.setComponentRam("word0", ExampleResources.getComponentRam());
    conf.setComponentRam("word1", ExampleResources.getComponentRam());
    conf.setComponentRam("word2", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(2));
    conf.setContainerCpuRequested(1);
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(3);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 29 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class SentenceWordCountTopology method main.

public static void main(String[] args) throws Exception {
    String name = "fast-word-count-topology";
    if (args != null && args.length > 0) {
        name = args[0];
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new FastRandomSentenceSpout(), 1);
    builder.setBolt("split", new SplitSentence(), 2).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 2).fieldsGrouping("split", new Fields("word"));
    Config conf = new Config();
    // component resource configuration
    conf.setComponentRam("spout", ByteAmount.fromMegabytes(512));
    conf.setComponentRam("split", ByteAmount.fromMegabytes(512));
    conf.setComponentRam("count", ByteAmount.fromMegabytes(512));
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerCpuRequested(2);
    conf.setNumStmgrs(2);
    HeronSubmitter.submitTopology(name, conf, builder.createTopology());
}

Example 30 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class StatefulSlidingWindowTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new IntegerSpout(), 1);
    builder.setBolt("sumbolt", new WindowSumBolt().withWindow(BaseWindowedBolt.Count.of(5), BaseWindowedBolt.Count.of(3)), 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("sumbolt");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    Config.setComponentRam(conf, "integer", ByteAmount.fromGigabytes(1));
    Config.setComponentRam(conf, "sumbolt", ByteAmount.fromGigabytes(1));
    Config.setComponentRam(conf, "printer", ByteAmount.fromGigabytes(1));
    Config.setContainerDiskRequested(conf, ByteAmount.fromGigabytes(5));
    Config.setContainerCpuRequested(conf, 4);
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
    conf.setTopologyStatefulCheckpointIntervalSecs(20);
    conf.setMaxSpoutPending(1000);
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    HeronSubmitter.submitTopology(topoName, conf, builder.createTopology());
}