use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class JoinOperatorTest method getTupleWindow.
private TupleWindow getTupleWindow() {
TopologyAPI.StreamId leftComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("leftComponent").setId("s1").build();
TopologyAPI.StreamId rightComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("rightComponent").setId("s1").build();
List<Tuple> tuples = new LinkedList<>();
for (int i = 0; i < 5; i++) {
Tuple tuple;
if (i % 2 == 0) {
tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
} else {
tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
}
tuples.add(tuple);
}
for (int i = 5; i < 8; i++) {
Tuple tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key2", String.valueOf(i))));
tuples.add(tuple);
}
for (int i = 8; i < 12; i++) {
Tuple tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key3", String.valueOf(i))));
tuples.add(tuple);
}
TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
return tupleWindow;
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class JoinOperatorTest method testOuterLeftJoinOperator.
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testOuterLeftJoinOperator() {
JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = getJoinOperator(JoinType.OUTER_LEFT);
TupleWindow tupleWindow = getTupleWindow();
Set<String> expectedResultsK1 = new HashSet<>();
expectedResultsK1.add("01");
expectedResultsK1.add("03");
expectedResultsK1.add("21");
expectedResultsK1.add("23");
expectedResultsK1.add("41");
expectedResultsK1.add("43");
Set<String> expectedResultsK2 = new HashSet<>();
expectedResultsK2.add("5null");
expectedResultsK2.add("6null");
expectedResultsK2.add("7null");
joinOperator.execute(tupleWindow);
Assert.assertEquals(9, emittedTuples.size());
for (Object object : emittedTuples) {
KeyValue<KeyedWindow<String>, String> tuple = (KeyValue<KeyedWindow<String>, String>) object;
KeyedWindow<String> keyedWindow = tuple.getKey();
switch(keyedWindow.getKey()) {
case "key1":
Assert.assertTrue(expectedResultsK1.contains(tuple.getValue()));
expectedResultsK1.remove(tuple.getValue());
break;
case "key2":
Assert.assertTrue(expectedResultsK2.contains(tuple.getValue()));
expectedResultsK2.remove(tuple.getValue());
break;
default:
Assert.fail();
}
Assert.assertEquals(12, keyedWindow.getWindow().getCount());
Assert.assertEquals(startTime, keyedWindow.getWindow().getStartTime());
Assert.assertEquals(endTime, keyedWindow.getWindow().getEndTime());
}
Assert.assertEquals(0, expectedResultsK1.size());
Assert.assertEquals(0, expectedResultsK2.size());
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class KeyByOperator method execute.
@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
R obj = (R) tuple.getValue(0);
K key = keyExtractor.apply(obj);
V value = valueExtractor.apply(obj);
collector.emit(new Values(new KeyValue<>(key, value)));
collector.ack(tuple);
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class ReduceByKeyAndWindowOperator method execute.
@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
Map<K, T> reduceMap = new HashMap<>();
Map<K, Integer> windowCountMap = new HashMap<>();
for (Tuple tuple : inputWindow.get()) {
R tup = (R) tuple.getValue(0);
addMap(reduceMap, windowCountMap, tup);
}
long startWindow;
long endWindow;
if (inputWindow.getStartTimestamp() == null) {
startWindow = 0;
} else {
startWindow = inputWindow.getStartTimestamp();
}
if (inputWindow.getEndTimestamp() == null) {
endWindow = 0;
} else {
endWindow = inputWindow.getEndTimestamp();
}
for (K key : reduceMap.keySet()) {
Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
}
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class GeneralReduceByKeyAndWindowOperator method execute.
@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
Map<K, T> reduceMap = new HashMap<>();
Map<K, Integer> windowCountMap = new HashMap<>();
for (Tuple tuple : inputWindow.get()) {
R tup = (R) tuple.getValue(0);
addMap(reduceMap, windowCountMap, tup);
}
long startWindow;
long endWindow;
if (inputWindow.getStartTimestamp() == null) {
startWindow = 0;
} else {
startWindow = inputWindow.getStartTimestamp();
}
if (inputWindow.getEndTimestamp() == null) {
endWindow = 0;
} else {
endWindow = inputWindow.getEndTimestamp();
}
for (K key : reduceMap.keySet()) {
Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
}
}
Aggregations