Examples with MISTBiFunction edu.snu.mist.common.functions.MISTBiFunction used on opensource projects

Example 1 with MISTBiFunction

use of edu.snu.mist.common.functions.MISTBiFunction in project mist by snuspl.

the class WindowedStreamTest method testReduceByKeyWindowStream.

/**
 * Test for reduceByKeyWindow operation.
 */
@Test
public void testReduceByKeyWindowStream() throws InjectionException, IOException {
    final MISTBiFunction<Integer, Integer, Integer> reduceFunc = (x, y) -> x + y;
    final ContinuousStream<Map<String, Integer>> reducedWindowStream = timeWindowedStream.reduceByKeyWindow(0, String.class, reduceFunc);
    // Get info
    final Map<String, String> conf = reducedWindowStream.getConfiguration();
    Assert.assertEquals("0", conf.get(ConfKeys.ReduceByKeyOperator.KEY_INDEX.name()));
    Assert.assertEquals(SerializeUtils.serializeToString(reduceFunc), conf.get(ConfKeys.ReduceByKeyOperator.MIST_BI_FUNC.name()));
    // Check windowed -> reduce by key
    checkEdges(queryBuilder.build().getDAG(), 1, timeWindowedStream, reducedWindowStream, new MISTEdge(Direction.LEFT));
}

Example 2 with MISTBiFunction

use of edu.snu.mist.common.functions.MISTBiFunction in project mist by snuspl.

the class PhysicalObjectGenerator method newOperator.

/**
 * Get a new operator.
 * @param conf configuration
 * @param classLoader external class loader
 * @return new operator
 */
@SuppressWarnings("unchecked")
public Operator newOperator(final Map<String, String> conf, final ClassLoader classLoader) throws IOException, ClassNotFoundException {
    final String type = conf.get(ConfKeys.OperatorConf.OP_TYPE.name());
    if (type.equals(ConfValues.OperatorType.MAP.name())) {
        return new MapOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.FILTER.name())) {
        return new FilterOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.FLAT_MAP.name())) {
        return new FlatMapOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.APPLY_STATEFUL.name())) {
        return new ApplyStatefulOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.STATE_TRANSITION.name())) {
        final String initialState = conf.get(ConfKeys.StateTransitionOperator.INITIAL_STATE.name());
        final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = getObject(conf, ConfKeys.StateTransitionOperator.STATE_TABLE.name(), classLoader);
        final Set<String> finalState = getObject(conf, ConfKeys.StateTransitionOperator.FINAL_STATE.name(), classLoader);
        return new StateTransitionOperator(initialState, finalState, stateTable);
    } else if (type.equals(ConfValues.OperatorType.CEP.name())) {
        final List<CepEventPattern> cepEventPatterns = getObject(conf, ConfKeys.CepOperator.CEP_EVENT.name(), classLoader);
        final long windowTime = Long.valueOf(conf.get(ConfKeys.CepOperator.WINDOW_TIME.name()));
        return new CepOperator(cepEventPatterns, windowTime);
    } else if (type.equals(ConfValues.OperatorType.REDUCE_BY_KEY.name())) {
        final int keyFieldNum = Integer.valueOf(conf.get(ConfKeys.ReduceByKeyOperator.KEY_INDEX.name()));
        final MISTBiFunction reduceFunc = getObject(conf, ConfKeys.ReduceByKeyOperator.MIST_BI_FUNC.name(), classLoader);
        return new ReduceByKeyOperator(keyFieldNum, reduceFunc);
    } else if (type.equals(ConfValues.OperatorType.UNION.name())) {
        return new UnionOperator();
    } else if (type.equals(ConfValues.OperatorType.TIME_WINDOW.name())) {
        final int windowSize = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_SIZE.name()));
        final int windowInterval = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
        return new TimeWindowOperator(windowSize, windowInterval);
    } else if (type.equals(ConfValues.OperatorType.COUNT_WINDOW.name())) {
        final int windowSize = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_SIZE.name()));
        final int windowInterval = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
        return new CountWindowOperator(windowSize, windowInterval);
    } else if (type.equals(ConfValues.OperatorType.SESSION_WINDOW.name())) {
        final int windowInterval = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
        return new SessionWindowOperator(windowInterval);
    } else if (type.equals(ConfValues.OperatorType.JOIN.name())) {
        return new JoinOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.AGGREGATE_WINDOW.name())) {
        return new AggregateWindowOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.APPLY_STATEFUL_WINDOW.name())) {
        return new ApplyStatefulWindowOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else {
        throw new RuntimeException("Invalid operator: " + type);
    }
}

Example 3 with MISTBiFunction

use of edu.snu.mist.common.functions.MISTBiFunction in project mist by snuspl.

the class ReduceByKeyOperatorTest method testReduceByKeyOperatorGetState.

/**
 * Test getting state of the ReduceByKeyOperator.
 */
@Test
@SuppressWarnings("unchecked")
public void testReduceByKeyOperatorGetState() throws InterruptedException {
    // Generate the current ReduceByKeyOperator state.
    final List<MistDataEvent> inputStream = ImmutableList.of(createTupleEvent("a", 1, 1L), createTupleEvent("b", 1, 2L), createTupleEvent("c", 1, 3L), createTupleEvent("a", 1, 4L), createTupleEvent("d", 1, 5L), createTupleEvent("a", 1, 6L), createTupleEvent("b", 1, 7L));
    // Generate the expected ReduceByKeyOperator state.
    final Map<String, Integer> expectedOperatorState = new HashMap<>();
    expectedOperatorState.put("a", 3);
    expectedOperatorState.put("b", 2);
    expectedOperatorState.put("c", 1);
    expectedOperatorState.put("d", 1);
    final int keyIndex = 0;
    final MISTBiFunction<Integer, Integer, Integer> wordCountFunc = (oldVal, val) -> oldVal + val;
    final ReduceByKeyOperator<String, Integer> wcOperator = new ReduceByKeyOperator<>(keyIndex, wordCountFunc);
    final List<MistEvent> result = new LinkedList<>();
    wcOperator.setOutputEmitter(new OutputBufferEmitter(result));
    inputStream.stream().forEach(wcOperator::processLeftData);
    // Get the current ReduceByKeyOperator's state.
    final Map<String, Object> operatorStateMap = wcOperator.getStateSnapshot();
    final Map<String, Integer> operatorState = (Map<String, Integer>) operatorStateMap.get("reduceByKeyState");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedOperatorState, operatorState);
}

Example 4 with MISTBiFunction

use of edu.snu.mist.common.functions.MISTBiFunction in project mist by snuspl.

the class ReduceByKeyOperatorTest method testReduceByKeyOperator.

/**
 * Test whether reduceByKeyOperator generates correct outputs.
 * Input: a list of tuples: ("a", 1), ("b", 1), ("c", 1), ("a", 1), ("d", 1), ("a", 1), ("b", 1)
 * Expected outputs: word counts:
 *  {"a": 1},
 *  {"a": 1, "b": 1},
 *  {"a": 1, "b": 1, "c": 1}
 *  {"a": 2, "b": 1, "c": 1}
 *  {"a": 2, "b": 1, "c": 1, "d": 1}
 *  {"a": 3, "b": 1, "c": 1, "d": 1}
 *  {"a": 3, "b": 2, "c": 1, "d": 1}
 */
@Test
public void testReduceByKeyOperator() throws InjectionException {
    // input stream
    final List<MistDataEvent> inputStream = ImmutableList.of(createTupleEvent("a", 1, 1L), createTupleEvent("b", 1, 2L), createTupleEvent("c", 1, 3L), createTupleEvent("a", 1, 4L), createTupleEvent("d", 1, 5L), createTupleEvent("a", 1, 6L), createTupleEvent("b", 1, 7L));
    // expected output
    final Map<String, Integer> o0 = new HashMap<>();
    o0.put("a", 1);
    final Map<String, Integer> o1 = new HashMap<>(o0);
    o1.put("b", 1);
    final Map<String, Integer> o2 = new HashMap<>(o1);
    o2.put("c", 1);
    final Map<String, Integer> o3 = new HashMap<>(o2);
    o3.put("a", 2);
    final Map<String, Integer> o4 = new HashMap<>(o3);
    o4.put("d", 1);
    final Map<String, Integer> o5 = new HashMap<>(o4);
    o5.put("a", 3);
    final Map<String, Integer> o6 = new HashMap<>(o5);
    o6.put("b", 2);
    final List<MistEvent> expectedStream = ImmutableList.of(new MistDataEvent(o0, 1L), new MistDataEvent(o1, 2L), new MistDataEvent(o2, 3L), new MistDataEvent(o3, 4L), new MistDataEvent(o4, 5L), new MistDataEvent(o5, 6L), new MistDataEvent(o6, 7L));
    // Set the key index of tuple
    final int keyIndex = 0;
    // Reduce function for word count
    final MISTBiFunction<Integer, Integer, Integer> wordCountFunc = (oldVal, val) -> oldVal + val;
    final ReduceByKeyOperator<String, Integer> wcOperator = new ReduceByKeyOperator<>(keyIndex, wordCountFunc);
    // output test
    final List<MistEvent> result = new LinkedList<>();
    wcOperator.setOutputEmitter(new OutputBufferEmitter(result));
    inputStream.stream().forEach(wcOperator::processLeftData);
    LOG.info("expected: " + expectedStream);
    LOG.info("result: " + result);
    Assert.assertEquals(expectedStream, result);
}

Example 5 with MISTBiFunction

use of edu.snu.mist.common.functions.MISTBiFunction in project mist by snuspl.

the class ReduceByKeyOperatorTest method testReduceByKeyOperatorSetState.

/**
 * Test setting state of the ReduceByKeyOperator.
 */
@Test
@SuppressWarnings("unchecked")
public void testReduceByKeyOperatorSetState() throws InterruptedException {
    // Generate a new state and set it to the state of a new ReduceByKeyWindowOperator.
    final Map<String, Integer> expectedOperatorState = new HashMap<>();
    expectedOperatorState.put("a", 3);
    expectedOperatorState.put("b", 2);
    expectedOperatorState.put("c", 1);
    expectedOperatorState.put("d", 1);
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("reduceByKeyState", expectedOperatorState);
    final int keyIndex = 0;
    final MISTBiFunction<Integer, Integer, Integer> wordCountFunc = (oldVal, val) -> oldVal + val;
    final ReduceByKeyOperator reduceByKeyOperator = new ReduceByKeyOperator(keyIndex, wordCountFunc);
    reduceByKeyOperator.setState(loadStateMap);
    // Compare the original and the set operator.
    final Map<String, Object> operatorStateMap = reduceByKeyOperator.getStateSnapshot();
    final Map<String, Integer> operatorState = (Map<String, Integer>) operatorStateMap.get("reduceByKeyState");
    Assert.assertEquals(expectedOperatorState, operatorState);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    reduceByKeyOperator.setOutputEmitter(new OutputBufferEmitter(result));
    reduceByKeyOperator.setState(operatorStateMap);
    final List<MistDataEvent> inputStream = ImmutableList.of(createTupleEvent("a", 1, 10L));
    inputStream.stream().forEach(reduceByKeyOperator::processLeftData);
    final List<MistEvent> expected = new LinkedList<>();
    final Map<String, Integer> o1 = new HashMap<>();
    o1.put("a", 4);
    o1.put("b", 2);
    o1.put("c", 1);
    o1.put("d", 1);
    expected.add(new MistDataEvent(o1, 10L));
    Assert.assertEquals(expected, result);
}