Examples with Config com.twitter.heron.api.Config used on opensource projects

Example 6 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class TestTopologyBuilder method createTopology.

// By default, will use AggregatorBolt, which writes to HTTP Server and takes URL as input
@Override
public HeronTopology createTopology() {
    // We will add the aggregation_bolt to be serialized
    final String AGGREGATOR_BOLT = "__integration_test_aggregator_bolt";
    BaseBatchBolt aggregatorBolt;
    try {
        // Terminal Bolt will be initialized using reflection, based on the value of terminal bolt
        // class.
        // class should be built on top of BaseBatchBolt abstract class, and can be changed using
        // setTerminalBolt function
        aggregatorBolt = (BaseBatchBolt) Class.forName(terminalBoltClass).getConstructor(String.class).newInstance(this.outputLocation);
    } catch (NoSuchMethodException e) {
        throw new RuntimeException(e + " Terminal Bolt class must have a single String constructor.");
    } catch (InstantiationException e) {
        throw new RuntimeException(e + " Terminal bolt class could not be instantiated.");
    } catch (IllegalAccessException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor is not accessible.");
    } catch (InvocationTargetException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor could not be invoked.");
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e + " Terminal Bolt class must be a class path.");
    }
    setBolt(AGGREGATOR_BOLT, aggregatorBolt, 1);
    // We get the user-defined TopologyAPI.Topology.Builder
    TopologyAPI.Topology.Builder topologyBlr = super.createTopology().setConfig(new Config()).setName("").setState(TopologyAPI.TopologyState.RUNNING).getTopology().toBuilder();
    // Clear unnecessary fields to make the state of TopologyAPI.Topology.Builder clean
    topologyBlr.clearTopologyConfig().clearName().clearState();
    for (TopologyAPI.Spout.Builder spout : topologyBlr.getSpoutsBuilderList()) {
        String name = spout.getComp().getName();
        spouts.put(name, spout);
    }
    // by looking only on bolts, since spout will not have parents
    for (TopologyAPI.Bolt.Builder bolt : topologyBlr.getBoltsBuilderList()) {
        String name = bolt.getComp().getName();
        bolts.put(name, bolt);
        if (name.equals(AGGREGATOR_BOLT)) {
            // since it is not user-defined
            continue;
        }
        // To get the parent's component to construct a graph of topology structure
        for (TopologyAPI.InputStream inputStream : bolt.getInputsList()) {
            String parent = inputStream.getStream().getComponentName();
            if (prev.containsKey(name)) {
                prev.get(name).add(parent);
            } else {
                HashSet<String> parents = new HashSet<String>();
                parents.add(parent);
                prev.put(name, parents);
            }
        }
    }
    // To find the terminal bolts defined by users and link them with "AggregatorBolt"
    // First, "it" of course needs upstream component, we don't want the isolated bolt
    HashSet<String> terminals = new HashSet<>();
    // Second, "it" should not exists in the prev.valueSet, which means, it has no downstream
    HashSet<String> nonTerminals = new HashSet<>();
    for (HashSet<String> set : prev.values()) {
        nonTerminals.addAll(set);
    }
    // a isolated bolt, including AggregatorBolt
    for (String bolt : prev.keySet()) {
        if (!nonTerminals.contains(bolt)) {
            terminals.add(bolt);
        }
    }
    // We will also consider the cases with spouts without children
    for (String spout : spouts.keySet()) {
        if (!nonTerminals.contains(spout)) {
            terminals.add(spout);
        }
    }
    // Now first, we will add all grouping to components
    for (String child : prev.keySet()) {
        for (String parent : prev.get(child)) {
            addAllGrouping(child, parent, Constants.INTEGRATION_TEST_CONTROL_STREAM_ID);
        }
    }
    // We could use any grouping but for convenience we would use allGrouping here
    for (String t : terminals) {
        List<TopologyAPI.OutputStream> osList;
        if (bolts.get(t) != null) {
            osList = bolts.get(t).getOutputsList();
        } else {
            osList = spouts.get(t).getOutputsList();
        }
        for (TopologyAPI.OutputStream os : osList) {
            addAllGrouping(AGGREGATOR_BOLT, t, os.getStream().getId());
        }
    }
    // We wrap it to the new topologyBuilder
    return new HeronTopology(topologyBlr);
}

Example 7 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class ConfigUtils method translateConfig.

/**
   * Translate storm config to heron config
   * @param stormConfig the storm config
   * @return a heron config
   */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static Config translateConfig(Map stormConfig) {
    Config heronConfig = new Config((Map<String, Object>) stormConfig);
    // Look at serialization stuff first
    doSerializationTranslation(heronConfig);
    // Now look at supported apis
    if (heronConfig.containsKey(org.apache.storm.Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS)) {
        heronConfig.put(org.apache.storm.Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, heronConfig.get(org.apache.storm.Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS).toString());
    }
    if (heronConfig.containsKey(org.apache.storm.Config.TOPOLOGY_WORKERS)) {
        Integer nWorkers = (Integer) heronConfig.get(org.apache.storm.Config.TOPOLOGY_WORKERS);
        com.twitter.heron.api.Config.setNumStmgrs(heronConfig, nWorkers);
    }
    if (heronConfig.containsKey(org.apache.storm.Config.TOPOLOGY_ACKER_EXECUTORS)) {
        Integer nAckers = (Integer) heronConfig.get(org.apache.storm.Config.TOPOLOGY_ACKER_EXECUTORS);
        com.twitter.heron.api.Config.setEnableAcking(heronConfig, nAckers > 0);
    }
    if (heronConfig.containsKey(org.apache.storm.Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
        Integer nSecs = (Integer) heronConfig.get(org.apache.storm.Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS);
        com.twitter.heron.api.Config.setMessageTimeoutSecs(heronConfig, nSecs);
    }
    if (heronConfig.containsKey(org.apache.storm.Config.TOPOLOGY_MAX_SPOUT_PENDING)) {
        Integer nPending = Integer.parseInt(heronConfig.get(org.apache.storm.Config.TOPOLOGY_MAX_SPOUT_PENDING).toString());
        com.twitter.heron.api.Config.setMaxSpoutPending(heronConfig, nPending);
    }
    if (heronConfig.containsKey(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS)) {
        Integer tSecs = Integer.parseInt(heronConfig.get(org.apache.storm.Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS).toString());
        com.twitter.heron.api.Config.setTickTupleFrequency(heronConfig, tSecs);
    }
    if (heronConfig.containsKey(org.apache.storm.Config.TOPOLOGY_DEBUG)) {
        Boolean dBg = Boolean.parseBoolean(heronConfig.get(org.apache.storm.Config.TOPOLOGY_DEBUG).toString());
        com.twitter.heron.api.Config.setDebug(heronConfig, dBg);
    }
    doTaskHooksTranslation(heronConfig);
    return heronConfig;
}

Example 8 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class LocalReadWriteTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3 || args.length > 4) {
        throw new RuntimeException("Expects 3 or 4 arguments, topology name, " + "inputFile, outputFile and max emit count (optional)");
    }
    String topologyName = args[0];
    String inputFile = args[1];
    String outputFile = args[2];
    TestTopologyBuilder builder = new TestTopologyBuilder(outputFile);
    builder.setTerminalBoltClass(LOCAL_AGGREGATOR_BOLT_CLASS);
    if (args.length == 3) {
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1);
    } else {
        int maxEmits = Integer.parseInt(args[3]);
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1, maxEmits);
    }
    builder.setBolt("identity-bolt", new IdentityBolt(new Fields("line")), 1).shuffleGrouping("paused-local-spout");
    Config conf = new BasicConfig();
    HeronSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example 9 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class AbstractTestTopology method submit.

public final void submit() throws AlreadyAliveException, InvalidTopologyException {
    TestTopologyBuilder builder = new TestTopologyBuilder(httpServerResultsUrl, httpServerStateUrl, stateUpdateToken, spoutWrapperType);
    Config conf = buildConfig(new BasicConfig());
    HeronSubmitter.submitTopology(topologyName, conf, buildTopology(builder).createTopology());
}

Example 10 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class UnitTestHelper method setTopology.

private static void setTopology(PhysicalPlans.PhysicalPlan.Builder pPlan, boolean ackEnabled, int messageTimeout, TopologyAPI.TopologyState topologyState) {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("test-spout", new TestSpout(), 1);
    // Here we need case switch to corresponding grouping
    topologyBuilder.setBolt("test-bolt", new TestBolt(), 1).shuffleGrouping("test-spout");
    Config conf = new Config();
    conf.setTeamEmail("streaming-compute@twitter.com");
    conf.setTeamName("stream-computing");
    conf.setTopologyProjectName("heron-integration-test");
    conf.setNumStmgrs(1);
    conf.setMaxSpoutPending(100);
    if (ackEnabled) {
        conf.setEnableAcking(true);
    } else {
        conf.setEnableAcking(false);
    }
    if (messageTimeout != -1) {
        conf.setMessageTimeoutSecs(messageTimeout);
        conf.put("topology.enable.message.timeouts", "true");
    }
    TopologyAPI.Topology fTopology = topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(topologyState).getTopology();
    pPlan.setTopology(fTopology);
}