Examples with Config com.twitter.heron.api.Config used on opensource projects

Example 36 with Config

use of com.twitter.heron.api.Config in project incubator-heron by apache.

the class WindowManagerTest method testTimeBasedWindow.

@Test
public void testTimeBasedWindow() throws Exception {
    EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(Duration.ofSeconds(1).toMillis());
    windowManager.setEvictionPolicy(evictionPolicy);
    /*
         * Don't wait for Timetrigger to fire since this could lead to timing issues in unit tests.
         * Set it to a large value and trigger manually.
          */
    TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(Duration.ofDays(1).toMillis());
    triggerPolicy.setTriggerHandler(windowManager);
    triggerPolicy.setEvictionPolicy(evictionPolicy);
    triggerPolicy.setTopologyConfig(new Config());
    triggerPolicy.start();
    windowManager.setTriggerPolicy(triggerPolicy);
    long now = System.currentTimeMillis();
    // add with past ts
    for (int i : seq(1, 50)) {
        windowManager.add(i, now - 1000);
    }
    // add with current ts
    for (int i : seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD)) {
        windowManager.add(i, now);
    }
    // first 50 should have expired due to expire events threshold
    assertEquals(50, listener.onExpiryEvents.size());
    // add more events with past ts
    for (int i : seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100)) {
        windowManager.add(i, now - 1000);
    }
    // simulate the time trigger by setting the reference time and invoking onTrigger() manually
    evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
    windowManager.onTrigger();
    // 100 events with past ts should expire
    assertEquals(100, listener.onExpiryEvents.size());
    assertEquals(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100), listener.onExpiryEvents);
    List<Integer> activationsEvents = seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD);
    assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationEvents);
    assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationNewEvents);
    // activation expired list should contain even the ones expired due to EXPIRE_EVENTS_THRESHOLD
    List<Integer> expiredList = seq(1, 50);
    expiredList.addAll(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100));
    assertEquals(expiredList, listener.onActivationExpiredEvents);
    listener.clear();
    // add more events with current ts
    List<Integer> newEvents = seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 101, WindowManager.EXPIRE_EVENTS_THRESHOLD + 200);
    for (int i : newEvents) {
        windowManager.add(i, now);
    }
    activationsEvents.addAll(newEvents);
    // simulate the time trigger by setting the reference time and invoking onTrigger() manually
    evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
    windowManager.onTrigger();
    assertTrue(listener.onExpiryEvents.isEmpty());
    assertEquals(activationsEvents, listener.onActivationEvents);
    assertEquals(newEvents, listener.onActivationNewEvents);
}

Example 37 with Config

use of com.twitter.heron.api.Config in project streaming-samples by ashvina.

the class NoAckWordCount2StageTopology method main.

public static void main(String[] args) throws Exception {
    TopologyArgParser parser = new TopologyArgParser(args, SPOUT, COUNT);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT, new RandomWordSpout(), parser.get(SPOUT));
    builder.setBolt(COUNT, new WordCount(), parser.get(COUNT)).fieldsGrouping(SPOUT, new Fields(WordCountTopologyHelper.FIELD_WORD));
    Config conf = new Config();
    conf.setDebug(false);
    conf.setEnableAcking(false);
    conf.setNumStmgrs(parser.getNumWorkers());
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 38 with Config

use of com.twitter.heron.api.Config in project streaming-samples by ashvina.

the class TopTrendingTopology method main.

public static void main(String[] args) throws Exception {
    TopologyArgParser parser = new TopologyArgParser(args, SPOUT, COUNTER, PARTIAL, TOTAL);
    TweetSpout tweetSpout = new TweetSpout("/tmp/tweets.txt");
    RollingCountBolt countBolt = new RollingCountBolt(Duration.ofSeconds(10), Duration.ofMillis(500));
    PartialRanker partialRanker = new PartialRanker(Duration.ofSeconds(5), 5);
    TotalRanker totalRanker = new TotalRanker(Duration.ofSeconds(5), 5);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT, tweetSpout, parser.get(SPOUT));
    builder.setBolt(COUNTER, countBolt, parser.get(COUNTER)).fieldsGrouping(SPOUT, new Fields(FIELD_TREND));
    builder.setBolt(PARTIAL, partialRanker, parser.get(PARTIAL)).fieldsGrouping(COUNTER, new Fields(FIELD_TREND));
    builder.setBolt(TOTAL, totalRanker, parser.get(TOTAL)).globalGrouping(PARTIAL);
    Config conf = new Config();
    conf.setDebug(false);
    conf.setEnableAcking(false);
    conf.setNumStmgrs(parser.getNumWorkers());
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 39 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class TestTopologyBuilder method createTopology.

// By default, will use AggregatorBolt, which writes to HTTP Server and takes URL as input
@Override
public HeronTopology createTopology() {
    // We will add the aggregation_bolt to be serialized
    final String AGGREGATOR_BOLT = "__integration_test_aggregator_bolt";
    BaseBatchBolt aggregatorBolt;
    try {
        // Terminal Bolt will be initialized using reflection, based on the value of terminal bolt
        // class.
        // class should be built on top of BaseBatchBolt abstract class, and can be changed using
        // setTerminalBolt function
        aggregatorBolt = (BaseBatchBolt) Class.forName(terminalBoltClass).getConstructor(String.class).newInstance(this.outputLocation);
    } catch (NoSuchMethodException e) {
        throw new RuntimeException(e + " Terminal Bolt class must have a single String constructor.");
    } catch (InstantiationException e) {
        throw new RuntimeException(e + " Terminal bolt class could not be instantiated.");
    } catch (IllegalAccessException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor is not accessible.");
    } catch (InvocationTargetException e) {
        throw new RuntimeException(e + " Terminal Bolt class constructor could not be invoked.");
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e + " Terminal Bolt class must be a class path.");
    }
    setBolt(AGGREGATOR_BOLT, aggregatorBolt, 1);
    // We get the user-defined TopologyAPI.Topology.Builder
    TopologyAPI.Topology.Builder topologyBlr = super.createTopology().setConfig(new Config()).setName("").setState(TopologyAPI.TopologyState.RUNNING).getTopology().toBuilder();
    // Clear unnecessary fields to make the state of TopologyAPI.Topology.Builder clean
    topologyBlr.clearTopologyConfig().clearName().clearState();
    for (TopologyAPI.Spout.Builder spout : topologyBlr.getSpoutsBuilderList()) {
        String name = spout.getComp().getName();
        spouts.put(name, spout);
    }
    // by looking only on bolts, since spout will not have parents
    for (TopologyAPI.Bolt.Builder bolt : topologyBlr.getBoltsBuilderList()) {
        String name = bolt.getComp().getName();
        bolts.put(name, bolt);
        if (name.equals(AGGREGATOR_BOLT)) {
            // since it is not user-defined
            continue;
        }
        // To get the parent's component to construct a graph of topology structure
        for (TopologyAPI.InputStream inputStream : bolt.getInputsList()) {
            String parent = inputStream.getStream().getComponentName();
            if (prev.containsKey(name)) {
                prev.get(name).add(parent);
            } else {
                HashSet<String> parents = new HashSet<String>();
                parents.add(parent);
                prev.put(name, parents);
            }
        }
    }
    // To find the terminal bolts defined by users and link them with "AggregatorBolt"
    // First, "it" of course needs upstream component, we don't want the isolated bolt
    HashSet<String> terminals = new HashSet<>();
    // Second, "it" should not exists in the prev.valueSet, which means, it has no downstream
    HashSet<String> nonTerminals = new HashSet<>();
    for (HashSet<String> set : prev.values()) {
        nonTerminals.addAll(set);
    }
    // a isolated bolt, including AggregatorBolt
    for (String bolt : prev.keySet()) {
        if (!nonTerminals.contains(bolt)) {
            terminals.add(bolt);
        }
    }
    // We will also consider the cases with spouts without children
    for (String spout : spouts.keySet()) {
        if (!nonTerminals.contains(spout)) {
            terminals.add(spout);
        }
    }
    // Now first, we will add all grouping to components
    for (String child : prev.keySet()) {
        for (String parent : prev.get(child)) {
            addAllGrouping(child, parent, Constants.INTEGRATION_TEST_CONTROL_STREAM_ID);
        }
    }
    // We could use any grouping but for convenience we would use allGrouping here
    for (String t : terminals) {
        List<TopologyAPI.OutputStream> osList;
        if (bolts.get(t) != null) {
            osList = bolts.get(t).getOutputsList();
        } else {
            osList = spouts.get(t).getOutputsList();
        }
        for (TopologyAPI.OutputStream os : osList) {
            addAllGrouping(AGGREGATOR_BOLT, t, os.getStream().getId());
        }
    }
    // We wrap it to the new topologyBuilder
    return new HeronTopology(topologyBlr);
}

Example 40 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class LocalReadWriteTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3 || args.length > 4) {
        throw new RuntimeException("Expects 3 or 4 arguments, topology name, " + "inputFile, outputFile and max emit count (optional)");
    }
    String topologyName = args[0];
    String inputFile = args[1];
    String outputFile = args[2];
    TestTopologyBuilder builder = new TestTopologyBuilder(outputFile);
    builder.setTerminalBoltClass(LOCAL_AGGREGATOR_BOLT_CLASS);
    if (args.length == 3) {
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1);
    } else {
        int maxEmits = Integer.parseInt(args[3]);
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1, maxEmits);
    }
    builder.setBolt("identity-bolt", new IdentityBolt(new Fields("line")), 1).shuffleGrouping("paused-local-spout");
    Config conf = new BasicConfig();
    HeronSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}