Examples with Config com.twitter.heron.api.Config used on opensource projects

Example 61 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class TestTopologyBuilder method createTopology.

// By default, will use AggregatorBolt, which writes to HTTP Server and takes URL as input
@Override
public HeronTopology createTopology() {
    // We will add the aggregation_bolt to be serialized
    final String AGGREGATOR_BOLT = "__integration_test_aggregator_bolt";
    BaseBatchBolt aggregatorBolt;
    try {
        // Terminal Bolt will be initialized using reflection, based on the value of terminal bolt
        // class.
        // class should be built on top of BaseBatchBolt abstract class, and can be changed using
        // setTerminalBolt function
        aggregatorBolt = (BaseBatchBolt) Class.forName(terminalBoltClass).getConstructor(String.class).newInstance(this.outputLocation);
    } catch (NoSuchMethodException e) {
        throw new RuntimeException(e + " Terminal Bolt class must have a single String constructor.");
    } catch (InstantiationException e) {
        throw new RuntimeException(e + " Terminal bolt class could not be instantiated.");
    } catch (IllegalAccessException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor is not accessible.");
    } catch (InvocationTargetException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor could not be invoked.");
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e + " Terminal Bolt class must be a class path.");
    }
    setBolt(AGGREGATOR_BOLT, aggregatorBolt, 1);
    // We get the user-defined TopologyAPI.Topology.Builder
    TopologyAPI.Topology.Builder topologyBlr = super.createTopology().setConfig(new Config()).setName("").setState(TopologyAPI.TopologyState.RUNNING).getTopology().toBuilder();
    // Clear unnecessary fields to make the state of TopologyAPI.Topology.Builder clean
    topologyBlr.clearTopologyConfig().clearName().clearState();
    for (TopologyAPI.Spout.Builder spout : topologyBlr.getSpoutsBuilderList()) {
        String name = spout.getComp().getName();
        spouts.put(name, spout);
    }
    // by looking only on bolts, since spout will not have parents
    for (TopologyAPI.Bolt.Builder bolt : topologyBlr.getBoltsBuilderList()) {
        String name = bolt.getComp().getName();
        bolts.put(name, bolt);
        if (name.equals(AGGREGATOR_BOLT)) {
            // since it is not user-defined
            continue;
        }
        // To get the parent's component to construct a graph of topology structure
        for (TopologyAPI.InputStream inputStream : bolt.getInputsList()) {
            String parent = inputStream.getStream().getComponentName();
            if (prev.containsKey(name)) {
                prev.get(name).add(parent);
            } else {
                HashSet<String> parents = new HashSet<String>();
                parents.add(parent);
                prev.put(name, parents);
            }
        }
    }
    // To find the terminal bolts defined by users and link them with "AggregatorBolt"
    // First, "it" of course needs upstream component, we don't want the isolated bolt
    HashSet<String> terminals = new HashSet<>();
    // Second, "it" should not exists in the prev.valueSet, which means, it has no downstream
    HashSet<String> nonTerminals = new HashSet<>();
    for (HashSet<String> set : prev.values()) {
        nonTerminals.addAll(set);
    }
    // a isolated bolt, including AggregatorBolt
    for (String bolt : prev.keySet()) {
        if (!nonTerminals.contains(bolt)) {
            terminals.add(bolt);
        }
    }
    // We will also consider the cases with spouts without children
    for (String spout : spouts.keySet()) {
        if (!nonTerminals.contains(spout)) {
            terminals.add(spout);
        }
    }
    // Now first, we will add all grouping to components
    for (String child : prev.keySet()) {
        for (String parent : prev.get(child)) {
            addAllGrouping(child, parent, Constants.INTEGRATION_TEST_CONTROL_STREAM_ID);
        }
    }
    // We could use any grouping but for convenience we would use allGrouping here
    for (String t : terminals) {
        List<TopologyAPI.OutputStream> osList;
        if (bolts.get(t) != null) {
            osList = bolts.get(t).getOutputsList();
        } else {
            osList = spouts.get(t).getOutputsList();
        }
        for (TopologyAPI.OutputStream os : osList) {
            addAllGrouping(AGGREGATOR_BOLT, t, os.getStream().getId());
        }
    }
    // We wrap it to the new topologyBuilder
    return new HeronTopology(topologyBlr);
}

Example 62 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class LocalReadWriteTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3 || args.length > 4) {
        throw new RuntimeException("Expects 3 or 4 arguments, topology name, " + "inputFile, outputFile and max emit count (optional)");
    }
    String topologyName = args[0];
    String inputFile = args[1];
    String outputFile = args[2];
    TestTopologyBuilder builder = new TestTopologyBuilder(outputFile);
    builder.setTerminalBoltClass(LOCAL_AGGREGATOR_BOLT_CLASS);
    if (args.length == 3) {
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1);
    } else {
        int maxEmits = Integer.parseInt(args[3]);
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1, maxEmits);
    }
    builder.setBolt("identity-bolt", new IdentityBolt(new Fields("line")), 1).shuffleGrouping("paused-local-spout");
    Config conf = new BasicConfig();
    HeronSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example 63 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class ConfigUtils method translateComponentConfig.

/**
 * Translate storm config to heron config for components
 * @param stormConfig the storm config
 * @return a heron config
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static Config translateComponentConfig(Map stormConfig) {
    Config heronConfig;
    if (stormConfig != null) {
        heronConfig = new Config((Map<String, Object>) stormConfig);
    } else {
        heronConfig = new Config();
    }
    doStormTranslation(heronConfig);
    return heronConfig;
}

Example 64 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class ConfigUtils method translateConfig.

/**
 * Translate storm config to heron config for topology
 * @param stormConfig the storm config
 * @return a heron config
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static Config translateConfig(Map stormConfig) {
    Config heronConfig;
    if (stormConfig != null) {
        heronConfig = new Config((Map<String, Object>) stormConfig);
    } else {
        heronConfig = new Config();
    }
    // Look at serialization stuff first
    doSerializationTranslation(heronConfig);
    // Now look at supported apis
    doStormTranslation(heronConfig);
    doTaskHooksTranslation(heronConfig);
    doTopologyLevelTranslation(heronConfig);
    return heronConfig;
}

Example 65 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class TopologyManagerTest method getTestTopology.

/**
 * Construct the test topology
 */
public static TopologyAPI.Topology getTestTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(STREAM_ID, new BaseRichSpout() {

        private static final long serialVersionUID = 5406114907377311020L;

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
            outputFieldsDeclarer.declare(new Fields(STREAM_ID));
        }

        @Override
        public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        }

        @Override
        public void nextTuple() {
        }
    }, 2);
    topologyBuilder.setBolt(BOLT_ID, new BaseBasicBolt() {

        private static final long serialVersionUID = 4398578755681473899L;

        @Override
        public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        }

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        }
    }, 2).shuffleGrouping(STREAM_ID);
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam(STREAM_ID, ByteAmount.fromMegabytes(500));
    conf.setComponentRam(BOLT_ID, ByteAmount.fromGigabytes(1));
    conf.setMessageTimeoutSecs(1);
    return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}