Examples with TimeWindowInformation edu.snu.mist.common.windows.TimeWindowInformation used on opensource projects

Example 1 with TimeWindowInformation

use of edu.snu.mist.common.windows.TimeWindowInformation in project mist by snuspl.

the class WindowedStreamTest method setUp.

@Before
public void setUp() {
    queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    timeWindowedStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).map(s -> new Tuple2<>(s, 1)).window(new TimeWindowInformation(5000, 1000));
}

Example 2 with TimeWindowInformation

use of edu.snu.mist.common.windows.TimeWindowInformation in project mist by snuspl.

the class JoinAndApplyStateful method submitQuery.

/**
 * Submit a query joining two sources and apply stateful operation on them.
 * The query reads strings from two source servers, windows them, and
 * prints out the result of apply stateful operation on the window
 * that contains the data which satisfies a predicate.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String source1Socket = injector.getNamedInstance(UnionLeftSourceAddress.class);
    final String source2Socket = injector.getNamedInstance(UnionRightSourceAddress.class);
    final SourceConfiguration localTextSocketSource1Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source1Socket);
    final SourceConfiguration localTextSocketSource2Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source2Socket);
    final MISTBiPredicate<String, String> joinPred = (s1, s2) -> s1.equals(s2);
    final ApplyStatefulFunction<Tuple2<String, String>, String> applyStatefulFunction = new FoldStringTupleFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream sourceStream1 = queryBuilder.socketTextStream(localTextSocketSource1Conf);
    final ContinuousStream sourceStream2 = queryBuilder.socketTextStream(localTextSocketSource2Conf);
    sourceStream1.join(sourceStream2, joinPred, new TimeWindowInformation(5000, 5000)).applyStatefulWindow(applyStatefulFunction).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 3 with TimeWindowInformation

use of edu.snu.mist.common.windows.TimeWindowInformation in project mist by snuspl.

the class MISTQueryTest method mistComplexQuerySerializeTest.

/**
 * This method tests a serialization of a complex query, containing 9 vertices.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void mistComplexQuerySerializeTest() throws InjectionException, IOException, URISyntaxException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final ContinuousStream<String> sourceStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF, TestParameters.PUNCTUATED_WATERMARK_CONF);
    final ContinuousStream<String> flatMapStream = sourceStream.flatMap(expectedFlatMapFunc);
    final ContinuousStream<String> filterStream = flatMapStream.filter(expectedFilterPredicate);
    final ContinuousStream<Tuple2<String, Integer>> mapStream = filterStream.map(expectedMapFunc);
    final WindowedStream<Tuple2<String, Integer>> windowedStream = mapStream.window(new TimeWindowInformation(expectedWindowSize, expectedWindowEmissionInterval));
    final ContinuousStream<Map<String, Integer>> reduceByKeyStream = windowedStream.reduceByKeyWindow(0, String.class, expectedReduceFunc);
    final MISTStream<String> sinkStream = reduceByKeyStream.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    // Build a query
    final MISTQuery complexQuery = queryBuilder.build();
    final Tuple<List<AvroVertex>, List<Edge>> serializedDAG = complexQuery.getAvroOperatorDag();
    final List<AvroVertex> vertices = serializedDAG.getKey();
    Assert.assertEquals(7, vertices.size());
    Assert.assertEquals(sourceStream.getConfiguration(), vertices.get(0).getConfiguration());
    Assert.assertEquals(flatMapStream.getConfiguration(), vertices.get(1).getConfiguration());
    Assert.assertEquals(filterStream.getConfiguration(), vertices.get(2).getConfiguration());
    Assert.assertEquals(mapStream.getConfiguration(), vertices.get(3).getConfiguration());
    Assert.assertEquals(windowedStream.getConfiguration(), vertices.get(4).getConfiguration());
    Assert.assertEquals(reduceByKeyStream.getConfiguration(), vertices.get(5).getConfiguration());
    Assert.assertEquals(sinkStream.getConfiguration(), vertices.get(6).getConfiguration());
    final List<Edge> edges = serializedDAG.getValue();
    final List<Edge> expectedEdges = Arrays.asList(Edge.newBuilder().setFrom(0).setTo(1).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(1).setTo(2).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(2).setTo(3).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(3).setTo(4).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(4).setTo(5).setDirection(Direction.LEFT).setBranchIndex(0).build(), Edge.newBuilder().setFrom(5).setTo(6).setDirection(Direction.LEFT).setBranchIndex(0).build());
    Assert.assertEquals(new HashSet<>(expectedEdges), new HashSet<>(edges));
}

Example 4 with TimeWindowInformation

use of edu.snu.mist.common.windows.TimeWindowInformation in project mist by snuspl.

the class ContinuousStreamTest method testTimeWindowedStream.

/**
 * Test for creating time-based WindowedStream from ContinuousStream.
 */
@Test
public void testTimeWindowedStream() {
    final WindowedStream<Tuple2<String, Integer>> timeWindowedStream = filteredMappedStream.window(new TimeWindowInformation(windowSize, windowEmissionInterval));
    final Map<String, String> conf = timeWindowedStream.getConfiguration();
    checkSizeBasedWindowInfo(windowSize, windowEmissionInterval, conf);
    // Check map -> timeWindow
    checkEdges(queryBuilder.build().getDAG(), 1, filteredMappedStream, timeWindowedStream, new MISTEdge(Direction.LEFT));
}

Example 5 with TimeWindowInformation

use of edu.snu.mist.common.windows.TimeWindowInformation in project mist by snuspl.

the class WindowAndAggregate method submitQuery.

/**
 * Submit a windowing and aggregating query.
 * The query reads strings from a source server, puts them into window,
 * concatenates all inputs in the window into a single string using toString function, and
 * print out the start and end time of the window.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    // configurations for source and sink
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    // configurations for windowing and aggregation
    final int windowSize = 5000;
    final int windowEmissionInterval = 2500;
    final MISTFunction<WindowData<String>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", window is started at " + windowData.getStart() + ", window is ended at " + windowData.getEnd() + ".";
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).window(new TimeWindowInformation(windowSize, windowEmissionInterval)).aggregateWindow(aggregateFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}