Examples with TopologyBuilder org.apache.heron.api.topology.TopologyBuilder used on opensource projects

Example 21 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class EcoBuilder method buildTopologyBuilder.

public TopologyBuilder buildTopologyBuilder(EcoExecutionContext executionContext, ObjectBuilder objectBuilder) throws InstantiationException, IllegalAccessException, ClassNotFoundException, NoSuchFieldException, InvocationTargetException {
    TopologyBuilder builder = new TopologyBuilder();
    LOG.info("Building components");
    componentBuilder.buildComponents(executionContext, objectBuilder);
    LOG.info("Building spouts");
    spoutBuilder.buildSpouts(executionContext, builder, objectBuilder);
    LOG.info("Building bolts");
    boltBuilder.buildBolts(executionContext, objectBuilder);
    LOG.info("Building streams");
    streamBuilder.buildStreams(executionContext, builder, objectBuilder);
    return builder;
}

Example 22 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class HeronKafkaSpoutSampleTopology method main.

public static void main(String[] args) {
    Map<String, Object> kafkaConsumerConfig = new HashMap<>();
    kafkaConsumerConfig.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    kafkaConsumerConfig.put(ConsumerConfig.GROUP_ID_CONFIG, "sample-kafka-spout");
    kafkaConsumerConfig.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
    kafkaConsumerConfig.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
    LOG.info("Kafka Consumer Config: {}", kafkaConsumerConfig);
    KafkaConsumerFactory<String, String> kafkaConsumerFactory = new DefaultKafkaConsumerFactory<>(kafkaConsumerConfig);
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(KAFKA_SPOUT_NAME, new KafkaSpout<>(kafkaConsumerFactory, Collections.singletonList("test-topic")));
    topologyBuilder.setBolt(LOGGING_BOLT_NAME, new LoggingBolt()).shuffleGrouping(KAFKA_SPOUT_NAME);
    Config config = new Config();
    config.setNumStmgrs(1);
    config.setContainerCpuRequested(1);
    config.setContainerRamRequested(ByteAmount.fromGigabytes(1));
    config.setContainerDiskRequested(ByteAmount.fromGigabytes(1));
    config.setComponentCpu(KAFKA_SPOUT_NAME, 0.25);
    config.setComponentRam(KAFKA_SPOUT_NAME, ByteAmount.fromMegabytes(256));
    config.setComponentDisk(KAFKA_SPOUT_NAME, ByteAmount.fromMegabytes(512));
    config.setComponentCpu(LOGGING_BOLT_NAME, 0.25);
    config.setComponentRam(LOGGING_BOLT_NAME, ByteAmount.fromMegabytes(256));
    config.setComponentDisk(LOGGING_BOLT_NAME, ByteAmount.fromMegabytes(256));
    Simulator simulator = new Simulator();
    simulator.submitTopology("heron-kafka-spout-sample-topology", config, topologyBuilder.createTopology());
}

Example 23 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class AbstractTupleRoutingTest method constructPhysicalPlan.

private PhysicalPlans.PhysicalPlan constructPhysicalPlan() {
    PhysicalPlans.PhysicalPlan.Builder physicalPlanBuilder = PhysicalPlans.PhysicalPlan.newBuilder();
    // Set topology protobuf
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    initSpout(topologyBuilder, Component.SPOUT.getName());
    initBoltA(topologyBuilder, Component.BOLT_A.getName(), Component.SPOUT.getName());
    initBoltB(topologyBuilder, Component.BOLT_B.getName(), Component.BOLT_A.getName());
    Config conf = new Config();
    conf.setTeamEmail("some-team@company.com");
    conf.setTeamName("some-team");
    conf.setTopologyProjectName("heron-integration-test");
    conf.setNumStmgrs(1);
    conf.setMaxSpoutPending(100);
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.ATMOST_ONCE);
    TopologyAPI.Topology topology = topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
    physicalPlanBuilder.setTopology(topology);
    // Set instances
    int taskId = 0;
    for (Component component : Component.values()) {
        addComponent(physicalPlanBuilder, component, taskId++);
    }
    // Set stream mgr
    PhysicalPlans.StMgr.Builder stmgr = PhysicalPlans.StMgr.newBuilder();
    stmgr.setId("stream-manager-id");
    stmgr.setHostName("127.0.0.1");
    stmgr.setDataPort(8888);
    stmgr.setLocalEndpoint("endpoint");
    physicalPlanBuilder.addStmgrs(stmgr);
    return physicalPlanBuilder.build();
}

Example 24 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class ExclamationTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    int parallelism = 2;
    int spouts = parallelism;
    builder.setSpout("word", new TestWordSpout(Duration.ofMillis(0)), spouts);
    int bolts = 2 * parallelism;
    builder.setBolt("exclaim1", new ExclamationBolt(), bolts).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.setMessageTimeoutSecs(600);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    // resources configuration
    conf.setComponentRam("word", ExampleResources.getComponentRam());
    conf.setComponentRam("exclaim1", ExampleResources.getComponentRam());
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(spouts + bolts, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(spouts + bolts, parallelism));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(spouts + bolts, parallelism));
    if (args != null && args.length > 0) {
        conf.setNumStmgrs(parallelism);
        HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        System.out.println("Topology name not provided as an argument, running in simulator mode.");
        Simulator simulator = new Simulator();
        simulator.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10000);
        simulator.killTopology("test");
        simulator.shutdown();
    }
}

Example 25 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class SlidingWindowTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(BaseWindowedBolt.Count.of(30), BaseWindowedBolt.Count.of(10)), 1).shuffleGrouping("integer");
    builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(BaseWindowedBolt.Count.of(3)), 1).shuffleGrouping("slidingsum");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    conf.setComponentRam("integer", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("slidingsum", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("tumblingavg", ByteAmount.fromGigabytes(1));
    conf.setComponentRam("printer", ByteAmount.fromGigabytes(1));
    conf.setContainerDiskRequested(ByteAmount.fromGigabytes(5));
    conf.setContainerCpuRequested(4);
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    HeronSubmitter.submitTopology(topoName, conf, builder.createTopology());
}