Examples with Tuple com.twitter.heron.api.tuple.Tuple used on opensource projects

Example 16 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class ReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private ReduceByKeyAndWindowOperator<String, Integer, String> getReduceByWindowOperator() {
    ReduceByKeyAndWindowOperator<String, Integer, String> reduceByWindowOperator = new ReduceByKeyAndWindowOperator<>(x -> x, x -> 1, (o, o2) -> o + o2);
    reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return reduceByWindowOperator;
}

Example 17 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class TopologyTests method createTopologyWithConnection.

/**
 * Create Topology proto object using HeronSubmitter API.
 *
 * @param heronConfig desired config params.
 * @param spouts spoutName -> parallelism
 * @param bolts boltName -> parallelism
 * @param connections connect default stream from value to key.
 * @return topology proto.
 */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 18 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class JoinOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
    Map<K, Pair<List<V1>, List<V2>>> joinMap = new HashMap<>();
    for (Tuple tuple : inputWindow.get()) {
        if (tuple.getSourceComponent().equals(leftComponent)) {
            V1 tup = (V1) tuple.getValue(0);
            if (tup != null) {
                addMapLeft(joinMap, tup);
            }
        } else {
            V2 tup = (V2) tuple.getValue(0);
            if (tup != null) {
                addMapRight(joinMap, tup);
            }
        }
    }
    evaluateJoinMap(joinMap, inputWindow);
}

Example 19 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class ReduceByKeyAndWindowOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
    Map<K, V> reduceMap = new HashMap<>();
    Map<K, Integer> windowCountMap = new HashMap<>();
    for (Tuple tuple : inputWindow.get()) {
        R tup = (R) tuple.getValue(0);
        addMap(reduceMap, windowCountMap, tup);
    }
    long startWindow;
    long endWindow;
    if (inputWindow.getStartTimestamp() == null) {
        startWindow = 0;
    } else {
        startWindow = inputWindow.getStartTimestamp();
    }
    if (inputWindow.getEndTimestamp() == null) {
        endWindow = 0;
    } else {
        endWindow = inputWindow.getEndTimestamp();
    }
    for (K key : reduceMap.keySet()) {
        Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
        KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
        collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
    }
}

Example 20 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class GeneralReduceByKeyAndWindowOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
    Map<K, VR> reduceMap = new HashMap<>();
    Map<K, Integer> windowCountMap = new HashMap<>();
    for (Tuple tuple : inputWindow.get()) {
        V tup = (V) tuple.getValue(0);
        addMap(reduceMap, windowCountMap, tup);
    }
    long startWindow;
    long endWindow;
    if (inputWindow.getStartTimestamp() == null) {
        startWindow = 0;
    } else {
        startWindow = inputWindow.getStartTimestamp();
    }
    if (inputWindow.getEndTimestamp() == null) {
        endWindow = 0;
    } else {
        endWindow = inputWindow.getEndTimestamp();
    }
    for (K key : reduceMap.keySet()) {
        Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
        KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
        collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
    }
}