Examples with KeyValue org.apache.heron.streamlet.KeyValue used on opensource projects

Example 11 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class GeneralReduceByKeyAndWindowOperatorTest method testReduceByWindowOperator.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testReduceByWindowOperator() {
    GeneralReduceByKeyAndWindowOperator<KeyValue<String, Integer>, String, Integer> reduceOperator = getReduceByWindowOperator(12);
    TupleWindow tupleWindow = getTupleWindow(3, 5);
    HashMap<String, Integer> expectedResults = new HashMap<>();
    expectedResults.put("0", 22);
    expectedResults.put("1", 22);
    expectedResults.put("2", 22);
    reduceOperator.execute(tupleWindow);
    Assert.assertEquals(3, emittedTuples.size());
    for (Object object : emittedTuples) {
        KeyValue<KeyedWindow<String>, Integer> tuple = (KeyValue<KeyedWindow<String>, Integer>) object;
        KeyedWindow<String> window = tuple.getKey();
        String key = window.getKey();
        Assert.assertEquals(5, window.getWindow().getCount());
        Assert.assertEquals(startTime, window.getWindow().getStartTime());
        Assert.assertEquals(endTime, window.getWindow().getEndTime());
        Assert.assertEquals(expectedResults.get(key), tuple.getValue());
    }
}

Example 12 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class JoinOperatorTest method testOuterRightJoinOperator.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testOuterRightJoinOperator() {
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = getJoinOperator(JoinType.OUTER_RIGHT);
    TupleWindow tupleWindow = getTupleWindow();
    Set<String> expectedResultsK1 = new HashSet<>();
    expectedResultsK1.add("01");
    expectedResultsK1.add("03");
    expectedResultsK1.add("21");
    expectedResultsK1.add("23");
    expectedResultsK1.add("41");
    expectedResultsK1.add("43");
    Set<String> expectedResultsK2 = new HashSet<>();
    expectedResultsK2.add("null8");
    expectedResultsK2.add("null9");
    expectedResultsK2.add("null10");
    expectedResultsK2.add("null11");
    joinOperator.execute(tupleWindow);
    Assert.assertEquals(10, emittedTuples.size());
    for (Object object : emittedTuples) {
        KeyValue<KeyedWindow<String>, String> tuple = (KeyValue<KeyedWindow<String>, String>) object;
        KeyedWindow<String> keyedWindow = tuple.getKey();
        switch(keyedWindow.getKey()) {
            case "key1":
                Assert.assertTrue(expectedResultsK1.contains(tuple.getValue()));
                expectedResultsK1.remove(tuple.getValue());
                break;
            case "key2":
                Assert.assertTrue(expectedResultsK2.contains(tuple.getValue()));
                expectedResultsK2.remove(tuple.getValue());
                break;
            case "key3":
                Assert.assertTrue(expectedResultsK2.contains(tuple.getValue()));
                expectedResultsK2.remove(tuple.getValue());
                break;
            default:
                Assert.fail();
        }
        Assert.assertEquals(12, keyedWindow.getWindow().getCount());
        Assert.assertEquals(startTime, keyedWindow.getWindow().getStartTime());
        Assert.assertEquals(endTime, keyedWindow.getWindow().getEndTime());
    }
    Assert.assertEquals(0, expectedResultsK1.size());
    Assert.assertEquals(0, expectedResultsK2.size());
}

Example 13 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class JoinOperatorTest method testInnerJoinOperator.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testInnerJoinOperator() {
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = getJoinOperator(JoinType.INNER);
    TupleWindow tupleWindow = getTupleWindow();
    Set<String> expectedResults = new HashSet<>();
    expectedResults.add("01");
    expectedResults.add("03");
    expectedResults.add("21");
    expectedResults.add("23");
    expectedResults.add("41");
    expectedResults.add("43");
    joinOperator.execute(tupleWindow);
    Assert.assertEquals(2 * 3, emittedTuples.size());
    for (Object object : emittedTuples) {
        KeyValue<KeyedWindow<String>, String> tuple = (KeyValue<KeyedWindow<String>, String>) object;
        KeyedWindow<String> keyedWindow = tuple.getKey();
        Assert.assertEquals("key1", keyedWindow.getKey());
        Assert.assertEquals(12, keyedWindow.getWindow().getCount());
        Assert.assertEquals(startTime, keyedWindow.getWindow().getStartTime());
        Assert.assertEquals(endTime, keyedWindow.getWindow().getEndTime());
        Assert.assertTrue(expectedResults.contains(tuple.getValue()));
        expectedResults.remove(tuple.getValue());
    }
    Assert.assertEquals(0, expectedResults.size());
}

Example 14 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class JoinOperatorTest method testOuterJoinOperator.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testOuterJoinOperator() {
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = getJoinOperator(JoinType.OUTER);
    TupleWindow tupleWindow = getTupleWindow();
    Set<String> expectedResultsK1 = new HashSet<>();
    expectedResultsK1.add("01");
    expectedResultsK1.add("03");
    expectedResultsK1.add("21");
    expectedResultsK1.add("23");
    expectedResultsK1.add("41");
    expectedResultsK1.add("43");
    Set<String> expectedResultsK2 = new HashSet<>();
    expectedResultsK2.add("5null");
    expectedResultsK2.add("6null");
    expectedResultsK2.add("7null");
    Set<String> expectedResultsK3 = new HashSet<>();
    expectedResultsK3.add("null8");
    expectedResultsK3.add("null9");
    expectedResultsK3.add("null10");
    expectedResultsK3.add("null11");
    joinOperator.execute(tupleWindow);
    Assert.assertEquals(13, emittedTuples.size());
    for (Object object : emittedTuples) {
        KeyValue<KeyedWindow<String>, String> tuple = (KeyValue<KeyedWindow<String>, String>) object;
        KeyedWindow<String> keyedWindow = tuple.getKey();
        switch(keyedWindow.getKey()) {
            case "key1":
                Assert.assertTrue(expectedResultsK1.contains(tuple.getValue()));
                expectedResultsK1.remove(tuple.getValue());
                break;
            case "key2":
                Assert.assertTrue(expectedResultsK2.contains(tuple.getValue()));
                expectedResultsK2.remove(tuple.getValue());
                break;
            case "key3":
                Assert.assertTrue(expectedResultsK3.contains(tuple.getValue()));
                expectedResultsK3.remove(tuple.getValue());
                break;
            default:
                Assert.fail();
        }
        Assert.assertEquals(12, keyedWindow.getWindow().getCount());
        Assert.assertEquals(startTime, keyedWindow.getWindow().getStartTime());
        Assert.assertEquals(endTime, keyedWindow.getWindow().getEndTime());
    }
    Assert.assertEquals(0, expectedResultsK1.size());
    Assert.assertEquals(0, expectedResultsK3.size());
}

Example 15 with KeyValue

use of org.apache.heron.streamlet.KeyValue in project heron by twitter.

the class JoinOperatorTest method getJoinOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> getJoinOperator(JoinType type) {
    SerializableFunction<KeyValue<String, String>, String> f = x -> x == null ? "null" : x.getKey();
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = new JoinOperator(type, "leftComponent", "rightComponent", f, f, (SerializableBiFunction<KeyValue<String, String>, KeyValue<String, String>, String>) (o, o2) -> (o == null ? "null" : o.getValue()) + (o2 == null ? "null" : o2.getValue()));
    joinOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return joinOperator;
}