Examples with KeyValue com.twitter.heron.streamlet.KeyValue used on opensource projects

Example 11 with KeyValue

use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.

the class JoinOperatorTest method testOuterJoinOperator.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testOuterJoinOperator() {
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = getJoinOperator(JoinType.OUTER);
    TupleWindow tupleWindow = getTupleWindow();
    Set<String> expectedResultsK1 = new HashSet<>();
    expectedResultsK1.add("01");
    expectedResultsK1.add("03");
    expectedResultsK1.add("21");
    expectedResultsK1.add("23");
    expectedResultsK1.add("41");
    expectedResultsK1.add("43");
    Set<String> expectedResultsK2 = new HashSet<>();
    expectedResultsK2.add("5null");
    expectedResultsK2.add("6null");
    expectedResultsK2.add("7null");
    Set<String> expectedResultsK3 = new HashSet<>();
    expectedResultsK3.add("null8");
    expectedResultsK3.add("null9");
    expectedResultsK3.add("null10");
    expectedResultsK3.add("null11");
    joinOperator.execute(tupleWindow);
    Assert.assertEquals(13, emittedTuples.size());
    for (Object object : emittedTuples) {
        KeyValue<KeyedWindow<String>, String> tuple = (KeyValue<KeyedWindow<String>, String>) object;
        KeyedWindow<String> keyedWindow = tuple.getKey();
        switch(keyedWindow.getKey()) {
            case "key1":
                Assert.assertTrue(expectedResultsK1.contains(tuple.getValue()));
                expectedResultsK1.remove(tuple.getValue());
                break;
            case "key2":
                Assert.assertTrue(expectedResultsK2.contains(tuple.getValue()));
                expectedResultsK2.remove(tuple.getValue());
                break;
            case "key3":
                Assert.assertTrue(expectedResultsK3.contains(tuple.getValue()));
                expectedResultsK3.remove(tuple.getValue());
                break;
            default:
                Assert.fail();
        }
        Assert.assertEquals(12, keyedWindow.getWindow().getCount());
        Assert.assertEquals(startTime, keyedWindow.getWindow().getStartTime());
        Assert.assertEquals(endTime, keyedWindow.getWindow().getEndTime());
    }
    Assert.assertEquals(0, expectedResultsK1.size());
    Assert.assertEquals(0, expectedResultsK3.size());
}

Example 12 with KeyValue

use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.

the class JoinOperatorTest method testOuterLeftJoinOperator.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testOuterLeftJoinOperator() {
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = getJoinOperator(JoinType.OUTER_LEFT);
    TupleWindow tupleWindow = getTupleWindow();
    Set<String> expectedResultsK1 = new HashSet<>();
    expectedResultsK1.add("01");
    expectedResultsK1.add("03");
    expectedResultsK1.add("21");
    expectedResultsK1.add("23");
    expectedResultsK1.add("41");
    expectedResultsK1.add("43");
    Set<String> expectedResultsK2 = new HashSet<>();
    expectedResultsK2.add("5null");
    expectedResultsK2.add("6null");
    expectedResultsK2.add("7null");
    joinOperator.execute(tupleWindow);
    Assert.assertEquals(9, emittedTuples.size());
    for (Object object : emittedTuples) {
        KeyValue<KeyedWindow<String>, String> tuple = (KeyValue<KeyedWindow<String>, String>) object;
        KeyedWindow<String> keyedWindow = tuple.getKey();
        switch(keyedWindow.getKey()) {
            case "key1":
                Assert.assertTrue(expectedResultsK1.contains(tuple.getValue()));
                expectedResultsK1.remove(tuple.getValue());
                break;
            case "key2":
                Assert.assertTrue(expectedResultsK2.contains(tuple.getValue()));
                expectedResultsK2.remove(tuple.getValue());
                break;
            default:
                Assert.fail();
        }
        Assert.assertEquals(12, keyedWindow.getWindow().getCount());
        Assert.assertEquals(startTime, keyedWindow.getWindow().getStartTime());
        Assert.assertEquals(endTime, keyedWindow.getWindow().getEndTime());
    }
    Assert.assertEquals(0, expectedResultsK1.size());
    Assert.assertEquals(0, expectedResultsK2.size());
}

Example 13 with KeyValue

use of com.twitter.heron.streamlet.KeyValue in project incubator-heron by apache.

the class GeneralReduceByKeyAndWindowOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
    Map<K, VR> reduceMap = new HashMap<>();
    Map<K, Integer> windowCountMap = new HashMap<>();
    for (Tuple tuple : inputWindow.get()) {
        V tup = (V) tuple.getValue(0);
        addMap(reduceMap, windowCountMap, tup);
    }
    long startWindow;
    long endWindow;
    if (inputWindow.getStartTimestamp() == null) {
        startWindow = 0;
    } else {
        startWindow = inputWindow.getStartTimestamp();
    }
    if (inputWindow.getEndTimestamp() == null) {
        endWindow = 0;
    } else {
        endWindow = inputWindow.getEndTimestamp();
    }
    for (K key : reduceMap.keySet()) {
        Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
        KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
        collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
    }
}