use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.
the class GeneralReduceByKeyAndWindowOperatorTest method getTupleWindow.
private TupleWindow getTupleWindow(int nkeys, int count) {
TopologyAPI.StreamId componentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("sourceComponent").setId("default").build();
List<Tuple> tuples = new LinkedList<>();
for (int i = 0; i < nkeys; i++) {
for (int j = 0; j < count; ++j) {
Tuple tuple = getTuple(componentStreamId, new Fields("a"), new Values(new KeyValue<>(String.valueOf(i), j)));
tuples.add(tuple);
}
}
TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
return tupleWindow;
}
use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.
the class GeneralReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.
@SuppressWarnings({ "rawtypes", "unchecked" })
private GeneralReduceByKeyAndWindowOperator<String, KeyValue<String, Integer>, Integer> getReduceByWindowOperator(Integer identity) {
GeneralReduceByKeyAndWindowOperator<String, KeyValue<String, Integer>, Integer> reduceByWindowOperator = new GeneralReduceByKeyAndWindowOperator<>(x -> x.getKey(), identity, (o, o2) -> o + o2.getValue());
reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {
@Override
public void reportError(Throwable error) {
}
@Override
public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
emittedTuples.addAll(tuple);
return null;
}
@Override
public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
}
@Override
public void ack(Tuple input) {
}
@Override
public void fail(Tuple input) {
}
}));
return reduceByWindowOperator;
}
use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.
the class GeneralReduceByKeyAndWindowOperatorTest method testReduceByWindowOperator.
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testReduceByWindowOperator() {
GeneralReduceByKeyAndWindowOperator<String, KeyValue<String, Integer>, Integer> reduceOperator = getReduceByWindowOperator(12);
TupleWindow tupleWindow = getTupleWindow(3, 5);
HashMap<String, Integer> expectedResults = new HashMap<>();
expectedResults.put("0", 22);
expectedResults.put("1", 22);
expectedResults.put("2", 22);
reduceOperator.execute(tupleWindow);
Assert.assertEquals(3, emittedTuples.size());
for (Object object : emittedTuples) {
KeyValue<KeyedWindow<String>, Integer> tuple = (KeyValue<KeyedWindow<String>, Integer>) object;
KeyedWindow<String> window = tuple.getKey();
String key = window.getKey();
Assert.assertEquals(5, window.getWindow().getCount());
Assert.assertEquals(startTime, window.getWindow().getStartTime());
Assert.assertEquals(endTime, window.getWindow().getEndTime());
Assert.assertEquals(expectedResults.get(key), tuple.getValue());
}
}
use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.
the class ReduceByKeyAndWindowOperatorTest method testReduceByWindowOperator.
@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testReduceByWindowOperator() {
ReduceByKeyAndWindowOperator<String, Integer, String> reduceOperator = getReduceByWindowOperator();
TupleWindow tupleWindow = getTupleWindow(3, 5);
HashMap<String, Integer> expectedResults = new HashMap<>();
expectedResults.put("0", 5);
expectedResults.put("1", 5);
expectedResults.put("2", 5);
reduceOperator.execute(tupleWindow);
Assert.assertEquals(3, emittedTuples.size());
for (Object object : emittedTuples) {
KeyValue<KeyedWindow<String>, Integer> tuple = (KeyValue<KeyedWindow<String>, Integer>) object;
KeyedWindow<String> window = tuple.getKey();
String key = window.getKey();
Assert.assertEquals(5, window.getWindow().getCount());
Assert.assertEquals(startTime, window.getWindow().getStartTime());
Assert.assertEquals(endTime, window.getWindow().getEndTime());
Assert.assertEquals(expectedResults.get(key), tuple.getValue());
}
}
use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.
the class ReduceByKeyAndWindowOperator method execute.
@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
Map<K, V> reduceMap = new HashMap<>();
Map<K, Integer> windowCountMap = new HashMap<>();
for (Tuple tuple : inputWindow.get()) {
R tup = (R) tuple.getValue(0);
addMap(reduceMap, windowCountMap, tup);
}
long startWindow;
long endWindow;
if (inputWindow.getStartTimestamp() == null) {
startWindow = 0;
} else {
startWindow = inputWindow.getStartTimestamp();
}
if (inputWindow.getEndTimestamp() == null) {
endWindow = 0;
} else {
endWindow = inputWindow.getEndTimestamp();
}
for (K key : reduceMap.keySet()) {
Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
}
}
Aggregations