Examples with ApplyStatefulFunction edu.snu.mist.common.functions.ApplyStatefulFunction used on opensource projects

Example 1 with ApplyStatefulFunction

use of edu.snu.mist.common.functions.ApplyStatefulFunction in project mist by snuspl.

the class KMeansClustering method submitQuery.

/**
 * Submit a query doing k-means clustering.
 * The query receives inputs in a form of two-dimensional double point "x1.x2,y1.y2",
 * conducts online k-means clustering, and displays the result of clustering.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    final MISTFunction<List<Cluster>, List<String>> flatMapFunc = // parse clustering result into String list
    (clusterList) -> {
        final List<String> results = new LinkedList<>();
        for (final Cluster cluster : clusterList) {
            String clusterResult = "Cluster id: " + cluster.getId() + ", cluster center: " + cluster.getCenter().toString() + "\n";
            for (final Point point : cluster.getClusteredPoints()) {
                clusterResult += point.toString() + "\n";
            }
            results.add(clusterResult);
        }
        return results;
    };
    final ApplyStatefulFunction<Point, List<Cluster>> applyStatefulFunction = new KMeansFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).map(s -> s.replaceAll(" ", "")).filter(s -> Pattern.matches("-?\\d+\\.\\d+,-?\\d+\\.\\d+", s)).map(s -> {
        final String[] array = s.split(",");
        return new Point(Double.parseDouble(array[0]), Double.parseDouble(array[1]));
    }).applyStateful(applyStatefulFunction).flatMap(flatMapFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 2 with ApplyStatefulFunction

use of edu.snu.mist.common.functions.ApplyStatefulFunction in project mist by snuspl.

the class JoinAndApplyStateful method submitQuery.

/**
 * Submit a query joining two sources and apply stateful operation on them.
 * The query reads strings from two source servers, windows them, and
 * prints out the result of apply stateful operation on the window
 * that contains the data which satisfies a predicate.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String source1Socket = injector.getNamedInstance(UnionLeftSourceAddress.class);
    final String source2Socket = injector.getNamedInstance(UnionRightSourceAddress.class);
    final SourceConfiguration localTextSocketSource1Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source1Socket);
    final SourceConfiguration localTextSocketSource2Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source2Socket);
    final MISTBiPredicate<String, String> joinPred = (s1, s2) -> s1.equals(s2);
    final ApplyStatefulFunction<Tuple2<String, String>, String> applyStatefulFunction = new FoldStringTupleFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream sourceStream1 = queryBuilder.socketTextStream(localTextSocketSource1Conf);
    final ContinuousStream sourceStream2 = queryBuilder.socketTextStream(localTextSocketSource2Conf);
    sourceStream1.join(sourceStream2, joinPred, new TimeWindowInformation(5000, 5000)).applyStatefulWindow(applyStatefulFunction).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 3 with ApplyStatefulFunction

use of edu.snu.mist.common.functions.ApplyStatefulFunction in project mist by snuspl.

the class ApplyStatefulOperatorTest method testApplyStatefulOperator.

/**
 * Test ApplyStatefulOperator.
 * It calculates the maximum value through some inputs.
 */
@Test
public void testApplyStatefulOperator() {
    // input events
    // expected results: 10--20--20--30--watermark--
    final MistDataEvent data10 = new MistDataEvent(10, 0L);
    final MistDataEvent data20 = new MistDataEvent(20, 1L);
    final MistDataEvent data15 = new MistDataEvent(15, 2L);
    final MistDataEvent data30 = new MistDataEvent(30, 3L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(4L);
    // the state managing function finding maximum integer value among received inputs
    final ApplyStatefulFunction applyStatefulFunction = new FindMaxIntFunction();
    final ApplyStatefulOperator<Integer, Integer> applyStatefulOperator = new ApplyStatefulOperator<>(applyStatefulFunction);
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulOperator.setOutputEmitter(new OutputBufferEmitter(result));
    applyStatefulOperator.processLeftData(data10);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(data10, result.get(0));
    applyStatefulOperator.processLeftData(data20);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(data20, result.get(1));
    applyStatefulOperator.processLeftData(data15);
    Assert.assertEquals(3, result.size());
    Assert.assertTrue(result.get(2) instanceof MistDataEvent);
    Assert.assertEquals(20, ((MistDataEvent) result.get(2)).getValue());
    Assert.assertEquals(2L, result.get(2).getTimestamp());
    applyStatefulOperator.processLeftData(data30);
    Assert.assertEquals(4, result.size());
    Assert.assertEquals(data30, result.get(3));
    applyStatefulOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(5, result.size());
    Assert.assertEquals(watermarkEvent, result.get(4));
}

Example 4 with ApplyStatefulFunction

use of edu.snu.mist.common.functions.ApplyStatefulFunction in project mist by snuspl.

the class ApplyStatefulOperatorTest method testApplyStatefulOperatorSetState.

/**
 * Test setting state of the ApplyStatefulOperator.
 */
@Test
public void testApplyStatefulOperatorSetState() throws InterruptedException {
    // Generate a new state and set it to a new ApplyStatefulOperator.
    final ApplyStatefulFunction applyStatefulFunction = new FindMaxIntFunction();
    final int expectedApplyStatefulFunctionState = 5;
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("applyStatefulFunctionState", expectedApplyStatefulFunctionState);
    final ApplyStatefulOperator<Integer, Integer> applyStatefulOperator = new ApplyStatefulOperator<>(applyStatefulFunction);
    applyStatefulOperator.setState(loadStateMap);
    // Get the current ApplyStatefulOperator's state.
    final Map<String, Object> operatorState = applyStatefulOperator.getStateSnapshot();
    final int applyStatefulFunctionState = (Integer) operatorState.get("applyStatefulFunctionState");
    // Compare the original and the set operator.
    Assert.assertEquals(expectedApplyStatefulFunctionState, applyStatefulFunctionState);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulOperator.setOutputEmitter(new OutputBufferEmitter(result));
    final MistDataEvent data10 = new MistDataEvent(10, 0L);
    final MistDataEvent data20 = new MistDataEvent(20, 1L);
    final MistDataEvent data15 = new MistDataEvent(15, 2L);
    applyStatefulOperator.processLeftData(data10);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(10, ((MistDataEvent) result.get(0)).getValue());
    applyStatefulOperator.processLeftData(data20);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(20, ((MistDataEvent) result.get(1)).getValue());
    applyStatefulOperator.processLeftData(data15);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(20, ((MistDataEvent) result.get(2)).getValue());
}

Example 5 with ApplyStatefulFunction

use of edu.snu.mist.common.functions.ApplyStatefulFunction in project mist by snuspl.

the class ApplyStatefulOperatorTest method testApplyStatefulOperatorGetState.

/**
 * Test getting state of the ApplyStatefulOperator.
 */
@Test
public void testApplyStatefulOperatorGetState() throws InterruptedException {
    // Generate the current ApplyStatefulOperator.
    final ApplyStatefulFunction applyStatefulFunction = new FindMaxIntFunction();
    final ApplyStatefulOperator<Integer, Integer> applyStatefulOperator = new ApplyStatefulOperator<>(applyStatefulFunction);
    final MistDataEvent data10 = new MistDataEvent(10, 0L);
    final MistDataEvent data20 = new MistDataEvent(20, 1L);
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulOperator.setOutputEmitter(new OutputBufferEmitter(result));
    applyStatefulOperator.processLeftData(data10);
    applyStatefulOperator.processLeftData(data20);
    // Generate the expected ApplyStatefulOperator's state.
    final int expectedApplyStatefulOperatorState = 20;
    // Get the current ApplyStatefulOperator's state.
    final Map<String, Object> operatorState = applyStatefulOperator.getStateSnapshot();
    final int applyStatefulOperatorState = (int) operatorState.get("applyStatefulFunctionState");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedApplyStatefulOperatorState, applyStatefulOperatorState);
}