Examples with Config org.apache.heron.api.Config used on opensource projects

Example 31 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class MultiSpoutExclamationTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word0", new TestWordSpout(), 2);
    builder.setSpout("word1", new TestWordSpout(), 2);
    builder.setSpout("word2", new TestWordSpout(), 2);
    builder.setBolt("exclaim1", new ExclamationBolt(), 2).shuffleGrouping("word0").shuffleGrouping("word1").shuffleGrouping("word2");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // component resource configuration
    conf.setComponentRam("word0", ExampleResources.getComponentRam());
    conf.setComponentRam("word1", ExampleResources.getComponentRam());
    conf.setComponentRam("word2", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
    // container resource configuration
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(3));
    conf.setContainerRamRequested(ByteAmount.fromGigabytes(5));
    conf.setContainerCpuRequested(4);
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(3);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 32 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class StatefulSlidingWindowTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new IntegerSpout(), 1);
    builder.setBolt("sumbolt", new WindowSumBolt().withWindow(BaseWindowedBolt.Count.of(5), BaseWindowedBolt.Count.of(3)), 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("sumbolt");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    Config.setComponentRam(conf, "integer", ByteAmount.fromGigabytes(1));
    Config.setComponentRam(conf, "sumbolt", ByteAmount.fromGigabytes(1));
    Config.setComponentRam(conf, "printer", ByteAmount.fromGigabytes(1));
    Config.setContainerDiskRequested(conf, ByteAmount.fromGigabytes(5));
    Config.setContainerCpuRequested(conf, 4);
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
    conf.setTopologyStatefulCheckpointIntervalSecs(20);
    conf.setMaxSpoutPending(1000);
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    HeronSubmitter.submitTopology(topoName, conf, builder.createTopology());
}

Example 33 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class TopologyManagerTest method getTestTopology.

/**
 * Construct the test topology
 */
public static TopologyAPI.Topology getTestTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(STREAM_ID, new BaseRichSpout() {

        private static final long serialVersionUID = 5406114907377311020L;

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
            outputFieldsDeclarer.declare(new Fields(STREAM_ID));
        }

        @Override
        public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        }

        @Override
        public void nextTuple() {
        }
    }, 2);
    topologyBuilder.setBolt(BOLT_ID, new BaseBasicBolt() {

        private static final long serialVersionUID = 4398578755681473899L;

        @Override
        public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        }

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        }
    }, 2).shuffleGrouping(STREAM_ID);
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam(STREAM_ID, ByteAmount.fromMegabytes(500));
    conf.setComponentRam(BOLT_ID, ByteAmount.fromGigabytes(1));
    conf.setMessageTimeoutSecs(1);
    return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 34 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class ConfigUtils method translateConfig.

/**
 * Translate storm config to heron config
 * @param stormConfig the storm config
 * @return a heron config
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static Config translateConfig(Map stormConfig) {
    Config heronConfig;
    if (stormConfig != null) {
        heronConfig = new Config((Map<String, Object>) stormConfig);
    } else {
        heronConfig = new Config();
    }
    // Look at serialization stuff first
    doSerializationTranslation(heronConfig);
    // Now look at supported apis
    doStormTranslation(heronConfig);
    doTaskHooksTranslation(heronConfig);
    doTopologyLevelTranslation(heronConfig);
    return heronConfig;
}

Example 35 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class HeronKafkaSpoutSampleTopology method main.

public static void main(String[] args) {
    Map<String, Object> kafkaConsumerConfig = new HashMap<>();
    kafkaConsumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    kafkaConsumerConfig.put(ConsumerConfig.GROUP_ID_CONFIG, "sample-kafka-spout");
    kafkaConsumerConfig.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
    kafkaConsumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
    LOG.info("Kafka Consumer Config: {}", kafkaConsumerConfig);
    KafkaConsumerFactory<String, String> kafkaConsumerFactory = new DefaultKafkaConsumerFactory<>(kafkaConsumerConfig);
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(KAFKA_SPOUT_NAME, new KafkaSpout<>(kafkaConsumerFactory, Collections.singletonList("test-topic")));
    topologyBuilder.setBolt(LOGGING_BOLT_NAME, new LoggingBolt()).shuffleGrouping(KAFKA_SPOUT_NAME);
    Config config = new Config();
    config.setNumStmgrs(1);
    config.setContainerCpuRequested(1);
    config.setContainerRamRequested(ByteAmount.fromGigabytes(1));
    config.setContainerDiskRequested(ByteAmount.fromGigabytes(1));
    config.setComponentCpu(KAFKA_SPOUT_NAME, 0.25);
    config.setComponentRam(KAFKA_SPOUT_NAME, ByteAmount.fromMegabytes(256));
    config.setComponentDisk(KAFKA_SPOUT_NAME, ByteAmount.fromMegabytes(512));
    config.setComponentCpu(LOGGING_BOLT_NAME, 0.25);
    config.setComponentRam(LOGGING_BOLT_NAME, ByteAmount.fromMegabytes(256));
    config.setComponentDisk(LOGGING_BOLT_NAME, ByteAmount.fromMegabytes(256));
    Simulator simulator = new Simulator();
    simulator.submitTopology("heron-kafka-spout-sample-topology", config, topologyBuilder.createTopology());
}