Examples with Config org.apache.heron.api.Config used on opensource projects

Example 1 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class StreamletWithFilterAndTransform method main.

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    StreamletWithFilterAndTransform topology = new StreamletWithFilterAndTransform(args);
    topology.submit(conf);
}

Example 2 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class TestTopologyBuilder method createTopology.

// By default, will use AggregatorBolt, which writes to HTTP Server and takes URL as input
@Override
public HeronTopology createTopology() {
    // We will add the aggregation_bolt to be serialized
    final String AGGREGATOR_BOLT = "__integration_test_aggregator_bolt";
    BaseBatchBolt aggregatorBolt;
    try {
        // Terminal Bolt will be initialized using reflection, based on the value of terminal bolt
        // class.
        // class should be built on top of BaseBatchBolt abstract class, and can be changed using
        // setTerminalBolt function
        aggregatorBolt = (BaseBatchBolt) Class.forName(terminalBoltClass).getConstructor(String.class).newInstance(this.outputLocation);
    } catch (NoSuchMethodException e) {
        throw new RuntimeException(e + " Terminal Bolt class must have a single String constructor.");
    } catch (InstantiationException e) {
        throw new RuntimeException(e + " Terminal bolt class could not be instantiated.");
    } catch (IllegalAccessException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor is not accessible.");
    } catch (InvocationTargetException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor could not be invoked.");
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e + " Terminal Bolt class must be a class path.");
    }
    setBolt(AGGREGATOR_BOLT, aggregatorBolt, 1);
    // We get the user-defined TopologyAPI.Topology.Builder
    TopologyAPI.Topology.Builder topologyBlr = super.createTopology().setConfig(new Config()).setName("").setState(TopologyAPI.TopologyState.RUNNING).getTopology().toBuilder();
    // Clear unnecessary fields to make the state of TopologyAPI.Topology.Builder clean
    topologyBlr.clearTopologyConfig().clearName().clearState();
    for (TopologyAPI.Spout.Builder spout : topologyBlr.getSpoutsBuilderList()) {
        String name = spout.getComp().getName();
        spouts.put(name, spout);
    }
    // by looking only on bolts, since spout will not have parents
    for (TopologyAPI.Bolt.Builder bolt : topologyBlr.getBoltsBuilderList()) {
        String name = bolt.getComp().getName();
        bolts.put(name, bolt);
        if (name.equals(AGGREGATOR_BOLT)) {
            // since it is not user-defined
            continue;
        }
        // To get the parent's component to construct a graph of topology structure
        for (TopologyAPI.InputStream inputStream : bolt.getInputsList()) {
            String parent = inputStream.getStream().getComponentName();
            if (prev.containsKey(name)) {
                prev.get(name).add(parent);
            } else {
                HashSet<String> parents = new HashSet<String>();
                parents.add(parent);
                prev.put(name, parents);
            }
        }
    }
    // To find the terminal bolts defined by users and link them with "AggregatorBolt"
    // First, "it" of course needs upstream component, we don't want the isolated bolt
    HashSet<String> terminals = new HashSet<>();
    // Second, "it" should not exists in the prev.valueSet, which means, it has no downstream
    HashSet<String> nonTerminals = new HashSet<>();
    for (HashSet<String> set : prev.values()) {
        nonTerminals.addAll(set);
    }
    // a isolated bolt, including AggregatorBolt
    for (String bolt : prev.keySet()) {
        if (!nonTerminals.contains(bolt)) {
            terminals.add(bolt);
        }
    }
    // We will also consider the cases with spouts without children
    for (String spout : spouts.keySet()) {
        if (!nonTerminals.contains(spout)) {
            terminals.add(spout);
        }
    }
    // Now first, we will add all grouping to components
    for (String child : prev.keySet()) {
        for (String parent : prev.get(child)) {
            addAllGrouping(child, parent, Constants.INTEGRATION_TEST_CONTROL_STREAM_ID);
        }
    }
    // We could use any grouping but for convenience we would use allGrouping here
    for (String t : terminals) {
        List<TopologyAPI.OutputStream> osList;
        if (bolts.get(t) != null) {
            osList = bolts.get(t).getOutputsList();
        } else {
            osList = spouts.get(t).getOutputsList();
        }
        for (TopologyAPI.OutputStream os : osList) {
            addAllGrouping(AGGREGATOR_BOLT, t, os.getStream().getId());
        }
    }
    // We wrap it to the new topologyBuilder
    return new HeronTopology(topologyBlr);
}

Example 3 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class AbstractTestTopology method submit.

public final void submit(Config userConf) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = buildConfig(new BasicConfig());
    if (userConf != null) {
        conf.putAll(userConf);
    }
    if (this.httpServerResultsUrl == null) {
        TopologyBuilder builder = new TopologyBuilder();
        HeronSubmitter.submitTopology(topologyName, conf, buildTopology(builder).createTopology());
    } else {
        TopologyTestTopologyBuilder builder = new TopologyTestTopologyBuilder(httpServerResultsUrl);
        conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
        conf.setTopologyStatefulCheckpointIntervalSecs(CHECKPOINT_INTERVAL);
        HeronSubmitter.submitTopology(topologyName, conf, buildStatefulTopology(builder).createTopology());
    }
}

Example 4 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class LocalReadWriteTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3 || args.length > 4) {
        throw new RuntimeException("Expects 3 or 4 arguments, topology name, " + "inputFile, outputFile and max emit count (optional)");
    }
    String topologyName = args[0];
    String inputFile = args[1];
    String outputFile = args[2];
    TestTopologyBuilder builder = new TestTopologyBuilder(outputFile);
    builder.setTerminalBoltClass(LOCAL_AGGREGATOR_BOLT_CLASS);
    if (args.length == 3) {
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1);
    } else {
        int maxEmits = Integer.parseInt(args[3]);
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1, maxEmits);
    }
    builder.setBolt("identity-bolt", new IdentityBolt(new Fields("line")), 1).shuffleGrouping("paused-local-spout");
    Config conf = new BasicConfig();
    HeronSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example 5 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class StreamletWithKeybyCountAndReduce method main.

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    StreamletWithKeybyCountAndReduce topology = new StreamletWithKeybyCountAndReduce(args);
    topology.submit(conf);
}