Examples with Tuple2 edu.snu.mist.common.types.Tuple2 used on opensource projects

Example 6 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class ContinuousStreamTest method testSessionWindowedStream.

/**
 * Test for creating session-based WindowedStream from ContinuousStream.
 */
@Test
public void testSessionWindowedStream() throws InjectionException {
    final int sessionInterval = 1000;
    /* Creates a test windowed stream with 1 sec session interval */
    final WindowedStream<Tuple2<String, Integer>> sessionWindowedStream = filteredMappedStream.window(new SessionWindowInformation(sessionInterval));
    // Check window info
    final Map<String, String> conf = sessionWindowedStream.getConfiguration();
    Assert.assertEquals(String.valueOf(sessionInterval), conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
    // Check map -> countWindow
    checkEdges(queryBuilder.build().getDAG(), 1, filteredMappedStream, sessionWindowedStream, new MISTEdge(Direction.LEFT));
}

Example 7 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class WindowedStreamTest method testApplyStatefulWindowStream.

/**
 * Test for binding the udf class of applyStatefulWindow operation.
 */
@Test
public void testApplyStatefulWindowStream() throws InjectionException, IOException {
    final ApplyStatefulFunction<Tuple2<String, Integer>, Integer> func = new UDFTestUtils.TestApplyStatefulFunction();
    final ContinuousStream<Integer> applyStatefulWindowStream = timeWindowedStream.applyStatefulWindow(new UDFTestUtils.TestApplyStatefulFunction());
    /* Simulate two data inputs on UDF stream */
    final Map<String, String> conf = applyStatefulWindowStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(func), conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    // Check windowed -> stateful operation applied
    checkEdges(queryBuilder.build().getDAG(), 1, timeWindowedStream, applyStatefulWindowStream, new MISTEdge(Direction.LEFT));
}

Example 8 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class JoinOperatorTest method testAggregateWindowOperator.

/**
 * Tests JoinOperator.
 * It joins a pair of inputs in two streams that has same key.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("Hello", 1), null), 10));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("MIST", 2), null), 20));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 3000L)), 30));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("SNUCMS", 3), null), 40));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 4000L)), 50));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(2, 5000L)), 60));
    final MistDataEvent dataEvent = new MistDataEvent(window, 60L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // predicate that tests whether two input data have same key or not
    final MISTBiPredicate<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinPredicate = (tuple1, tuple2) -> tuple1.get(1).equals(tuple2.get(0));
    final JoinOperator<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinOperator = new JoinOperator<>(joinPredicate);
    // expected pairs
    // {Hello, 1} and {1, 3000L}
    // {Hello, 1} and {1, 4000L}
    // {MIST, 2} and {2, 5000L}
    final List<MistEvent> result = new LinkedList<>();
    joinOperator.setOutputEmitter(new OutputBufferEmitter(result));
    joinOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertTrue(((MistDataEvent) result.get(0)).getValue() instanceof WindowData);
    final WindowData windowData = (WindowData) ((MistDataEvent) result.get(0)).getValue();
    Assert.assertEquals(0L, windowData.getStart());
    Assert.assertEquals(99L, windowData.getEnd());
    final Collection<Tuple2<Tuple2<String, Integer>, Tuple2<Integer, Long>>> dataCollection = windowData.getDataCollection();
    final Iterator iterator = dataCollection.iterator();
    Assert.assertEquals(3, dataCollection.size());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 3000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 4000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("MIST", 2), new Tuple2<>(2, 5000L)), iterator.next());
    Assert.assertEquals(60L, result.get(0).getTimestamp());
    joinOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}

Example 9 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class StateTransitionOperatorTest method testStateTransitionOperatorGetState.

/**
 * Test getting state of StateTransitionOperator.
 */
@Test
public void testStateTransitionOperatorGetState() throws InterruptedException {
    // generate input data event
    final Map<String, Integer> value1 = new HashMap<>();
    value1.put("number", 1);
    final MistDataEvent data1 = new MistDataEvent(value1, 0L);
    final Map<String, Integer> value2 = new HashMap<>();
    value2.put("number", 2);
    final MistDataEvent data2 = new MistDataEvent(value2, 1L);
    // generate a set of final states
    final Set<String> finalSet = new HashSet<>();
    finalSet.add("1");
    finalSet.add("2");
    // generate a state table
    final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = new HashMap<>();
    final Collection<Tuple2<MISTPredicate, String>> list0 = new ArrayList<>();
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 1), "1"));
    final Collection<Tuple2<MISTPredicate, String>> list1 = new ArrayList<>();
    list1.add(new Tuple2<>(new RuleBasedEQPredicate("number", 2), "2"));
    stateTable.put("0", list0);
    stateTable.put("1", list1);
    final StateTransitionOperator stateTransitionOperator = new StateTransitionOperator("0", finalSet, stateTable);
    final List<MistEvent> result = new ArrayList<>();
    stateTransitionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    stateTransitionOperator.processLeftData(data1);
    stateTransitionOperator.processLeftData(data2);
    // Generate the expected state
    final String expectedOperatorState = "2";
    // Get the
    final Map<String, Object> operatorState = stateTransitionOperator.getStateSnapshot();
    final String stateTransitionOperatorState = (String) operatorState.get("stateTransitionOperatorState");
    Assert.assertEquals(expectedOperatorState, stateTransitionOperatorState);
}

Example 10 with Tuple2

use of edu.snu.mist.common.types.Tuple2 in project mist by snuspl.

the class JoinAndApplyStateful method submitQuery.

/**
 * Submit a query joining two sources and apply stateful operation on them.
 * The query reads strings from two source servers, windows them, and
 * prints out the result of apply stateful operation on the window
 * that contains the data which satisfies a predicate.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String source1Socket = injector.getNamedInstance(UnionLeftSourceAddress.class);
    final String source2Socket = injector.getNamedInstance(UnionRightSourceAddress.class);
    final SourceConfiguration localTextSocketSource1Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source1Socket);
    final SourceConfiguration localTextSocketSource2Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source2Socket);
    final MISTBiPredicate<String, String> joinPred = (s1, s2) -> s1.equals(s2);
    final ApplyStatefulFunction<Tuple2<String, String>, String> applyStatefulFunction = new FoldStringTupleFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream sourceStream1 = queryBuilder.socketTextStream(localTextSocketSource1Conf);
    final ContinuousStream sourceStream2 = queryBuilder.socketTextStream(localTextSocketSource2Conf);
    sourceStream1.join(sourceStream2, joinPred, new TimeWindowInformation(5000, 5000)).applyStatefulWindow(applyStatefulFunction).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}