Examples with WindowData edu.snu.mist.common.windows.WindowData used on opensource projects

Example 1 with WindowData

use of edu.snu.mist.common.windows.WindowData in project mist by snuspl.

the class JoinOperatorTest method testAggregateWindowOperator.

/**
 * Tests JoinOperator.
 * It joins a pair of inputs in two streams that has same key.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("Hello", 1), null), 10));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("MIST", 2), null), 20));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 3000L)), 30));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("SNUCMS", 3), null), 40));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 4000L)), 50));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(2, 5000L)), 60));
    final MistDataEvent dataEvent = new MistDataEvent(window, 60L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // predicate that tests whether two input data have same key or not
    final MISTBiPredicate<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinPredicate = (tuple1, tuple2) -> tuple1.get(1).equals(tuple2.get(0));
    final JoinOperator<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinOperator = new JoinOperator<>(joinPredicate);
    // expected pairs
    // {Hello, 1} and {1, 3000L}
    // {Hello, 1} and {1, 4000L}
    // {MIST, 2} and {2, 5000L}
    final List<MistEvent> result = new LinkedList<>();
    joinOperator.setOutputEmitter(new OutputBufferEmitter(result));
    joinOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertTrue(((MistDataEvent) result.get(0)).getValue() instanceof WindowData);
    final WindowData windowData = (WindowData) ((MistDataEvent) result.get(0)).getValue();
    Assert.assertEquals(0L, windowData.getStart());
    Assert.assertEquals(99L, windowData.getEnd());
    final Collection<Tuple2<Tuple2<String, Integer>, Tuple2<Integer, Long>>> dataCollection = windowData.getDataCollection();
    final Iterator iterator = dataCollection.iterator();
    Assert.assertEquals(3, dataCollection.size());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 3000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 4000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("MIST", 2), new Tuple2<>(2, 5000L)), iterator.next());
    Assert.assertEquals(60L, result.get(0).getTimestamp());
    joinOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}

Example 2 with WindowData

use of edu.snu.mist.common.windows.WindowData in project mist by snuspl.

the class SessionWindow method submitQuery.

/**
 * Submit a query.
 * The query reads strings from a source server, puts them into a window,
 * and if there is no incoming data during the interval of the session window,
 * the session will be closed.
 * The query will print out the data in the session, and a new session is created
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    // configurations for source and sink
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    // configurations for windowing and aggregation by session dependent on time
    final int sessionInterval = 5000;
    final MISTFunction<WindowData<String>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", window is started at " + windowData.getStart() + ", window is ended at " + windowData.getEnd() + ".";
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).window(new SessionWindowInformation(sessionInterval)).aggregateWindow(aggregateFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    System.out.println("End of submitQuery");
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 3 with WindowData

use of edu.snu.mist.common.windows.WindowData in project mist by snuspl.

the class WindowedStreamTest method testAggregateWindowStream.

/**
 * Test for aggregateWindow operation.
 */
@Test
public void testAggregateWindowStream() throws InjectionException, IOException, ClassNotFoundException {
    final MISTFunction<WindowData<Tuple2<String, Integer>>, String> func = new WindowAggregateFunction();
    final ContinuousStream<String> aggregateWindowStream = timeWindowedStream.aggregateWindow(func);
    final Map<String, String> conf = aggregateWindowStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(func), conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    // Check windowed -> aggregated
    checkEdges(queryBuilder.build().getDAG(), 1, timeWindowedStream, aggregateWindowStream, new MISTEdge(Direction.LEFT));
}

Example 4 with WindowData

use of edu.snu.mist.common.windows.WindowData in project mist by snuspl.

the class AggregateWindowOperatorTest method testAggregateWindowOperator.

/**
 * Test AggregateWindowOperator.
 * It calculates the maximum value from the collection.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(10, 10));
    window.putData(new MistDataEvent(20, 20));
    window.putData(new MistDataEvent(15, 30));
    window.putData(new MistDataEvent(30, 90));
    final MistDataEvent dataEvent = new MistDataEvent(window, 90L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // functions that dealing with input WindowData
    final MISTFunction<WindowData<Integer>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", " + windowData.getStart() + ", " + windowData.getEnd();
    };
    final AggregateWindowOperator<Integer, String> aggregateWindowOperator = new AggregateWindowOperator<>(aggregateFunc);
    final List<MistEvent> result = new LinkedList<>();
    aggregateWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    aggregateWindowOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertEquals("[10, 20, 15, 30], 0, 99", ((MistDataEvent) result.get(0)).getValue());
    Assert.assertEquals(90L, result.get(0).getTimestamp());
    aggregateWindowOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}

Example 5 with WindowData

use of edu.snu.mist.common.windows.WindowData in project mist by snuspl.

the class WindowAndAggregate method submitQuery.

/**
 * Submit a windowing and aggregating query.
 * The query reads strings from a source server, puts them into window,
 * concatenates all inputs in the window into a single string using toString function, and
 * print out the start and end time of the window.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    // configurations for source and sink
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    // configurations for windowing and aggregation
    final int windowSize = 5000;
    final int windowEmissionInterval = 2500;
    final MISTFunction<WindowData<String>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", window is started at " + windowData.getStart() + ", window is ended at " + windowData.getEnd() + ".";
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).window(new TimeWindowInformation(windowSize, windowEmissionInterval)).aggregateWindow(aggregateFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}