Search in sources :

Example 1 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class JoinOperatorTest method getTupleWindow.

private TupleWindow getTupleWindow() {
    TopologyAPI.StreamId leftComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("leftComponent").setId("s1").build();
    TopologyAPI.StreamId rightComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("rightComponent").setId("s1").build();
    List<Tuple> tuples = new LinkedList<>();
    for (int i = 0; i < 5; i++) {
        Tuple tuple;
        if (i % 2 == 0) {
            tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
        } else {
            tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
        }
        tuples.add(tuple);
    }
    for (int i = 5; i < 8; i++) {
        Tuple tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key2", String.valueOf(i))));
        tuples.add(tuple);
    }
    for (int i = 8; i < 12; i++) {
        Tuple tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key3", String.valueOf(i))));
        tuples.add(tuple);
    }
    TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
    return tupleWindow;
}
Also used : KeyValue(org.apache.heron.streamlet.KeyValue) Values(org.apache.heron.api.tuple.Values) TupleWindow(org.apache.heron.api.windowing.TupleWindow) LinkedList(java.util.LinkedList) TopologyAPI(org.apache.heron.api.generated.TopologyAPI) Fields(org.apache.heron.api.tuple.Fields) TupleWindowImpl(org.apache.heron.api.windowing.TupleWindowImpl) Tuple(org.apache.heron.api.tuple.Tuple)

Example 2 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class JoinOperatorTest method testOuterLeftJoinOperator.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testOuterLeftJoinOperator() {
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = getJoinOperator(JoinType.OUTER_LEFT);
    TupleWindow tupleWindow = getTupleWindow();
    Set<String> expectedResultsK1 = new HashSet<>();
    expectedResultsK1.add("01");
    expectedResultsK1.add("03");
    expectedResultsK1.add("21");
    expectedResultsK1.add("23");
    expectedResultsK1.add("41");
    expectedResultsK1.add("43");
    Set<String> expectedResultsK2 = new HashSet<>();
    expectedResultsK2.add("5null");
    expectedResultsK2.add("6null");
    expectedResultsK2.add("7null");
    joinOperator.execute(tupleWindow);
    Assert.assertEquals(9, emittedTuples.size());
    for (Object object : emittedTuples) {
        KeyValue<KeyedWindow<String>, String> tuple = (KeyValue<KeyedWindow<String>, String>) object;
        KeyedWindow<String> keyedWindow = tuple.getKey();
        switch(keyedWindow.getKey()) {
            case "key1":
                Assert.assertTrue(expectedResultsK1.contains(tuple.getValue()));
                expectedResultsK1.remove(tuple.getValue());
                break;
            case "key2":
                Assert.assertTrue(expectedResultsK2.contains(tuple.getValue()));
                expectedResultsK2.remove(tuple.getValue());
                break;
            default:
                Assert.fail();
        }
        Assert.assertEquals(12, keyedWindow.getWindow().getCount());
        Assert.assertEquals(startTime, keyedWindow.getWindow().getStartTime());
        Assert.assertEquals(endTime, keyedWindow.getWindow().getEndTime());
    }
    Assert.assertEquals(0, expectedResultsK1.size());
    Assert.assertEquals(0, expectedResultsK2.size());
}
Also used : KeyValue(org.apache.heron.streamlet.KeyValue) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) TupleWindow(org.apache.heron.api.windowing.TupleWindow) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 3 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class KeyByOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    R obj = (R) tuple.getValue(0);
    K key = keyExtractor.apply(obj);
    V value = valueExtractor.apply(obj);
    collector.emit(new Values(new KeyValue<>(key, value)));
    collector.ack(tuple);
}
Also used : KeyValue(org.apache.heron.streamlet.KeyValue) Values(org.apache.heron.api.tuple.Values)

Example 4 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class ReduceByKeyAndWindowOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
    Map<K, T> reduceMap = new HashMap<>();
    Map<K, Integer> windowCountMap = new HashMap<>();
    for (Tuple tuple : inputWindow.get()) {
        R tup = (R) tuple.getValue(0);
        addMap(reduceMap, windowCountMap, tup);
    }
    long startWindow;
    long endWindow;
    if (inputWindow.getStartTimestamp() == null) {
        startWindow = 0;
    } else {
        startWindow = inputWindow.getStartTimestamp();
    }
    if (inputWindow.getEndTimestamp() == null) {
        endWindow = 0;
    } else {
        endWindow = inputWindow.getEndTimestamp();
    }
    for (K key : reduceMap.keySet()) {
        Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
        KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
        collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
    }
}
Also used : Window(org.apache.heron.streamlet.Window) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) TupleWindow(org.apache.heron.api.windowing.TupleWindow) KeyValue(org.apache.heron.streamlet.KeyValue) HashMap(java.util.HashMap) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) Values(org.apache.heron.api.tuple.Values) Tuple(org.apache.heron.api.tuple.Tuple)

Example 5 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class GeneralReduceByKeyAndWindowOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
    Map<K, T> reduceMap = new HashMap<>();
    Map<K, Integer> windowCountMap = new HashMap<>();
    for (Tuple tuple : inputWindow.get()) {
        R tup = (R) tuple.getValue(0);
        addMap(reduceMap, windowCountMap, tup);
    }
    long startWindow;
    long endWindow;
    if (inputWindow.getStartTimestamp() == null) {
        startWindow = 0;
    } else {
        startWindow = inputWindow.getStartTimestamp();
    }
    if (inputWindow.getEndTimestamp() == null) {
        endWindow = 0;
    } else {
        endWindow = inputWindow.getEndTimestamp();
    }
    for (K key : reduceMap.keySet()) {
        Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
        KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
        collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
    }
}
Also used : Window(org.apache.heron.streamlet.Window) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) TupleWindow(org.apache.heron.api.windowing.TupleWindow) KeyValue(org.apache.heron.streamlet.KeyValue) HashMap(java.util.HashMap) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) Values(org.apache.heron.api.tuple.Values) Tuple(org.apache.heron.api.tuple.Tuple)

Aggregations

KeyValue (org.apache.heron.streamlet.KeyValue)16 TupleWindow (org.apache.heron.api.windowing.TupleWindow)11 KeyedWindow (org.apache.heron.streamlet.KeyedWindow)9 Values (org.apache.heron.api.tuple.Values)8 Test (org.junit.Test)7 Tuple (org.apache.heron.api.tuple.Tuple)6 HashMap (java.util.HashMap)5 HashSet (java.util.HashSet)5 LinkedList (java.util.LinkedList)4 TopologyAPI (org.apache.heron.api.generated.TopologyAPI)3 Fields (org.apache.heron.api.tuple.Fields)3 TupleWindowImpl (org.apache.heron.api.windowing.TupleWindowImpl)3 Collection (java.util.Collection)2 List (java.util.List)2 Config (org.apache.heron.api.Config)2 IOutputCollector (org.apache.heron.api.bolt.IOutputCollector)2 OutputCollector (org.apache.heron.api.bolt.OutputCollector)2 TopologyContext (org.apache.heron.api.topology.TopologyContext)2 Window (org.apache.heron.streamlet.Window)2 DecimalFormat (java.text.DecimalFormat)1