Examples with TopologyBuilder com.twitter.heron.api.topology.TopologyBuilder used on opensource projects

Example 1 with TopologyBuilder

use of com.twitter.heron.api.topology.TopologyBuilder in project heron by twitter.

the class LaunchRunnerTest method createTopology.

public static TopologyAPI.Topology createTopology(com.twitter.heron.api.Config heronConfig) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new BaseRichSpout() {

        private static final long serialVersionUID = -762965195665496156L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    }, 2);
    builder.setBolt("bolt-1", new BaseBasicBolt() {

        private static final long serialVersionUID = -5738458486388778812L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    }, 1);
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(TOPOLOGY_NAME).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 2 with TopologyBuilder

use of com.twitter.heron.api.topology.TopologyBuilder in project heron by twitter.

the class HeronExecutorTaskTest method createTestTopology.

Topology createTestTopology(String name) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new TestSpout(), 2);
    builder.setBolt("bolt-1", new TestBolt(), 1).shuffleGrouping("spout-1");
    HeronTopology topology = builder.createTopology();
    com.twitter.heron.api.Config config = new com.twitter.heron.api.Config();
    return topology.setName(name).setConfig(config).setState(TopologyState.RUNNING).getTopology();
}

Example 3 with TopologyBuilder

use of com.twitter.heron.api.topology.TopologyBuilder in project heron by twitter.

the class UnitTestHelper method setTopology.

private static void setTopology(PhysicalPlans.PhysicalPlan.Builder pPlan, boolean ackEnabled, int messageTimeout, TopologyAPI.TopologyState topologyState) {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("test-spout", new TestSpout(), 1);
    // Here we need case switch to corresponding grouping
    topologyBuilder.setBolt("test-bolt", new TestBolt(), 1).shuffleGrouping("test-spout");
    Config conf = new Config();
    conf.setTeamEmail("streaming-compute@twitter.com");
    conf.setTeamName("stream-computing");
    conf.setTopologyProjectName("heron-integration-test");
    conf.setNumStmgrs(1);
    conf.setMaxSpoutPending(100);
    if (ackEnabled) {
        conf.setEnableAcking(true);
    } else {
        conf.setEnableAcking(false);
    }
    if (messageTimeout != -1) {
        conf.setMessageTimeoutSecs(messageTimeout);
        conf.put("topology.enable.message.timeouts", "true");
    }
    TopologyAPI.Topology fTopology = topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(topologyState).getTopology();
    pPlan.setTopology(fTopology);
}

Example 4 with TopologyBuilder

use of com.twitter.heron.api.topology.TopologyBuilder in project heron by twitter.

the class CustomGroupingTest method constructPhysicalPlan.

private PhysicalPlans.PhysicalPlan constructPhysicalPlan(MyCustomGrouping myCustomGrouping) {
    PhysicalPlans.PhysicalPlan.Builder pPlan = PhysicalPlans.PhysicalPlan.newBuilder();
    // Set topology protobuf
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("test-spout", new TestSpout(), 1);
    // Here we need case switch to corresponding grouping
    topologyBuilder.setBolt("test-bolt", new TestBolt(), 1).customGrouping("test-spout", myCustomGrouping);
    Config conf = new Config();
    conf.setTeamEmail("streaming-compute@twitter.com");
    conf.setTeamName("stream-computing");
    conf.setTopologyProjectName("heron-integration-test");
    conf.setNumStmgrs(1);
    conf.setMaxSpoutPending(100);
    conf.setEnableAcking(false);
    TopologyAPI.Topology fTopology = topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
    pPlan.setTopology(fTopology);
    // Set instances
    // Construct the spoutInstance
    PhysicalPlans.InstanceInfo.Builder spoutInstanceInfo = PhysicalPlans.InstanceInfo.newBuilder();
    spoutInstanceInfo.setComponentName("test-spout");
    spoutInstanceInfo.setTaskId(0);
    spoutInstanceInfo.setComponentIndex(0);
    PhysicalPlans.Instance.Builder spoutInstance = PhysicalPlans.Instance.newBuilder();
    spoutInstance.setInstanceId("spout-id");
    spoutInstance.setStmgrId("stream-manager-id");
    spoutInstance.setInfo(spoutInstanceInfo);
    // Construct the boltInstanceInfo
    PhysicalPlans.InstanceInfo.Builder boltInstanceInfo = PhysicalPlans.InstanceInfo.newBuilder();
    boltInstanceInfo.setComponentName("test-bolt");
    boltInstanceInfo.setTaskId(1);
    boltInstanceInfo.setComponentIndex(0);
    PhysicalPlans.Instance.Builder boltInstance = PhysicalPlans.Instance.newBuilder();
    boltInstance.setInstanceId("bolt-id");
    boltInstance.setStmgrId("stream-manager-id");
    boltInstance.setInfo(boltInstanceInfo);
    pPlan.addInstances(spoutInstance);
    pPlan.addInstances(boltInstance);
    // Set stream mgr
    PhysicalPlans.StMgr.Builder stmgr = PhysicalPlans.StMgr.newBuilder();
    stmgr.setId("stream-manager-id");
    stmgr.setHostName("127.0.0.1");
    stmgr.setDataPort(8888);
    stmgr.setLocalEndpoint("endpoint");
    pPlan.addStmgrs(stmgr);
    return pPlan.build();
}

Example 5 with TopologyBuilder

use of com.twitter.heron.api.topology.TopologyBuilder in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
   * Create Topology proto object using HeronSubmitter API.
   *
   * @param heronConfig desired config params.
   * @param spouts spoutName -> parallelism
   * @param bolts boltName -> parallelism
   * @param connections connect default stream from value to key.
   * @return topology proto.
   */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}