Examples with HeronTopology org.apache.heron.api.HeronTopology used on opensource projects

Example 1 with HeronTopology

use of org.apache.heron.api.HeronTopology in project heron by twitter.

the class TestTopologyBuilder method createTopology.

// By default, will use AggregatorBolt, which writes to HTTP Server and takes URL as input
@Override
public HeronTopology createTopology() {
    // We will add the aggregation_bolt to be serialized
    final String AGGREGATOR_BOLT = "__integration_test_aggregator_bolt";
    BaseBatchBolt aggregatorBolt;
    try {
        // Terminal Bolt will be initialized using reflection, based on the value of terminal bolt
        // class.
        // class should be built on top of BaseBatchBolt abstract class, and can be changed using
        // setTerminalBolt function
        aggregatorBolt = (BaseBatchBolt) Class.forName(terminalBoltClass).getConstructor(String.class).newInstance(this.outputLocation);
    } catch (NoSuchMethodException e) {
        throw new RuntimeException(e + " Terminal Bolt class must have a single String constructor.");
    } catch (InstantiationException e) {
        throw new RuntimeException(e + " Terminal bolt class could not be instantiated.");
    } catch (IllegalAccessException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor is not accessible.");
    } catch (InvocationTargetException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor could not be invoked.");
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e + " Terminal Bolt class must be a class path.");
    }
    setBolt(AGGREGATOR_BOLT, aggregatorBolt, 1);
    // We get the user-defined TopologyAPI.Topology.Builder
    TopologyAPI.Topology.Builder topologyBlr = super.createTopology().setConfig(new Config()).setName("").setState(TopologyAPI.TopologyState.RUNNING).getTopology().toBuilder();
    // Clear unnecessary fields to make the state of TopologyAPI.Topology.Builder clean
    topologyBlr.clearTopologyConfig().clearName().clearState();
    for (TopologyAPI.Spout.Builder spout : topologyBlr.getSpoutsBuilderList()) {
        String name = spout.getComp().getName();
        spouts.put(name, spout);
    }
    // by looking only on bolts, since spout will not have parents
    for (TopologyAPI.Bolt.Builder bolt : topologyBlr.getBoltsBuilderList()) {
        String name = bolt.getComp().getName();
        bolts.put(name, bolt);
        if (name.equals(AGGREGATOR_BOLT)) {
            // since it is not user-defined
            continue;
        }
        // To get the parent's component to construct a graph of topology structure
        for (TopologyAPI.InputStream inputStream : bolt.getInputsList()) {
            String parent = inputStream.getStream().getComponentName();
            if (prev.containsKey(name)) {
                prev.get(name).add(parent);
            } else {
                HashSet<String> parents = new HashSet<String>();
                parents.add(parent);
                prev.put(name, parents);
            }
        }
    }
    // To find the terminal bolts defined by users and link them with "AggregatorBolt"
    // First, "it" of course needs upstream component, we don't want the isolated bolt
    HashSet<String> terminals = new HashSet<>();
    // Second, "it" should not exists in the prev.valueSet, which means, it has no downstream
    HashSet<String> nonTerminals = new HashSet<>();
    for (HashSet<String> set : prev.values()) {
        nonTerminals.addAll(set);
    }
    // a isolated bolt, including AggregatorBolt
    for (String bolt : prev.keySet()) {
        if (!nonTerminals.contains(bolt)) {
            terminals.add(bolt);
        }
    }
    // We will also consider the cases with spouts without children
    for (String spout : spouts.keySet()) {
        if (!nonTerminals.contains(spout)) {
            terminals.add(spout);
        }
    }
    // Now first, we will add all grouping to components
    for (String child : prev.keySet()) {
        for (String parent : prev.get(child)) {
            addAllGrouping(child, parent, Constants.INTEGRATION_TEST_CONTROL_STREAM_ID);
        }
    }
    // We could use any grouping but for convenience we would use allGrouping here
    for (String t : terminals) {
        List<TopologyAPI.OutputStream> osList;
        if (bolts.get(t) != null) {
            osList = bolts.get(t).getOutputsList();
        } else {
            osList = spouts.get(t).getOutputsList();
        }
        for (TopologyAPI.OutputStream os : osList) {
            addAllGrouping(AGGREGATOR_BOLT, t, os.getStream().getId());
        }
    }
    // We wrap it to the new topologyBuilder
    return new HeronTopology(topologyBlr);
}

Example 2 with HeronTopology

use of org.apache.heron.api.HeronTopology in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
 * Create Topology proto object using HeronSubmitter API.
 *
 * @param heronConfig desired config params.
 * @param spouts spoutName -> parallelism
 * @param bolts boltName -> parallelism
 * @param connections connect default stream from value to key.
 * @return topology proto.
 */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 3 with HeronTopology

use of org.apache.heron.api.HeronTopology in project heron by twitter.

the class LaunchRunnerTest method createTopology.

public static TopologyAPI.Topology createTopology(org.apache.heron.api.Config heronConfig) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new BaseRichSpout() {

        private static final long serialVersionUID = -762965195665496156L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    }, 2);
    builder.setBolt("bolt-1", new BaseBasicBolt() {

        private static final long serialVersionUID = -5738458486388778812L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    }, 1);
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(TOPOLOGY_NAME).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 4 with HeronTopology

use of org.apache.heron.api.HeronTopology in project heron by twitter.

the class HeronExecutorTaskTest method createTestTopology.

Topology createTestTopology(String name) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new TestSpout(), 2);
    builder.setBolt("bolt-1", new TestBolt(), 1).shuffleGrouping("spout-1");
    HeronTopology topology = builder.createTopology();
    org.apache.heron.api.Config config = new org.apache.heron.api.Config();
    return topology.setName(name).setConfig(config).setState(TopologyState.RUNNING).getTopology();
}

Example 5 with HeronTopology

use of org.apache.heron.api.HeronTopology in project heron by twitter.

the class EcoSubmitterTest method submitHeronTopology_AllGood_BehavesAsExpected.

@Test
public void submitHeronTopology_AllGood_BehavesAsExpected() throws Exception {
    Config config = new Config();
    HeronTopology topology = new HeronTopology(null);
    PowerMockito.spy(HeronSubmitter.class);
    PowerMockito.doNothing().when(HeronSubmitter.class, "submitTopology", any(String.class), any(Config.class), any(HeronTopology.class));
    subject.submitHeronTopology("name", config, topology);
    PowerMockito.verifyStatic(times(1));
    HeronSubmitter.submitTopology(anyString(), any(Config.class), any(HeronTopology.class));
}