Examples with MISTFunction edu.snu.mist.common.functions.MISTFunction used on opensource projects

Example 1 with MISTFunction

use of edu.snu.mist.common.functions.MISTFunction in project mist by snuspl.

the class StatelessOperatorTest method testMapOperation.

/**
 * Test map operation.
 * It converts string to tuple (string, 1).
 */
@Test
public void testMapOperation() throws InjectionException {
    // input stream
    final List<MistDataEvent> inputStream = ImmutableList.of(new MistDataEvent("a", 1L), new MistDataEvent("b", 2L), new MistDataEvent("d", 3L), new MistDataEvent("b", 4L), new MistDataEvent("c", 5L));
    // expected output
    final List<MistEvent> expectedStream = ImmutableList.of(new MistDataEvent(new Tuple<>("a", 1), 1L), new MistDataEvent(new Tuple<>("b", 1), 2L), new MistDataEvent(new Tuple<>("d", 1), 3L), new MistDataEvent(new Tuple<>("b", 1), 4L), new MistDataEvent(new Tuple<>("c", 1), 5L));
    // map function: convert string to tuple
    final MISTFunction<String, Tuple> mapFunc = (mapInput) -> new Tuple<>(mapInput, 1);
    final MapOperator<String, Tuple> mapOperator = new MapOperator<>(mapFunc);
    testStatelessOperator(inputStream, expectedStream, mapOperator);
}

Example 2 with MISTFunction

use of edu.snu.mist.common.functions.MISTFunction in project mist by snuspl.

the class StatelessOperatorTest method testFlatMapOperation.

/**
 * Test flatMap operation.
 * It splits the string by space.
 */
@Test
public void testFlatMapOperation() throws InjectionException {
    // input stream
    final List<MistDataEvent> inputStream = ImmutableList.of(new MistDataEvent("a b c", 1L), new MistDataEvent("b c d", 2L), new MistDataEvent("d e f", 3L));
    // expected output
    final List<MistEvent> expectedStream = ImmutableList.of(new MistDataEvent("a", 1L), new MistDataEvent("b", 1L), new MistDataEvent("c", 1L), new MistDataEvent("b", 2L), new MistDataEvent("c", 2L), new MistDataEvent("d", 2L), new MistDataEvent("d", 3L), new MistDataEvent("e", 3L), new MistDataEvent("f", 3L));
    // map function: splits the string by space.
    final MISTFunction<String, List<String>> flatMapFunc = (mapInput) -> Arrays.asList(mapInput.split(" "));
    final FlatMapOperator<String, String> flatMapOperator = new FlatMapOperator<>(flatMapFunc);
    testStatelessOperator(inputStream, expectedStream, flatMapOperator);
}

Example 3 with MISTFunction

use of edu.snu.mist.common.functions.MISTFunction in project mist by snuspl.

the class KMeansClustering method submitQuery.

/**
 * Submit a query doing k-means clustering.
 * The query receives inputs in a form of two-dimensional double point "x1.x2,y1.y2",
 * conducts online k-means clustering, and displays the result of clustering.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    final MISTFunction<List<Cluster>, List<String>> flatMapFunc = // parse clustering result into String list
    (clusterList) -> {
        final List<String> results = new LinkedList<>();
        for (final Cluster cluster : clusterList) {
            String clusterResult = "Cluster id: " + cluster.getId() + ", cluster center: " + cluster.getCenter().toString() + "\n";
            for (final Point point : cluster.getClusteredPoints()) {
                clusterResult += point.toString() + "\n";
            }
            results.add(clusterResult);
        }
        return results;
    };
    final ApplyStatefulFunction<Point, List<Cluster>> applyStatefulFunction = new KMeansFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).map(s -> s.replaceAll(" ", "")).filter(s -> Pattern.matches("-?\\d+\\.\\d+,-?\\d+\\.\\d+", s)).map(s -> {
        final String[] array = s.split(",");
        return new Point(Double.parseDouble(array[0]), Double.parseDouble(array[1]));
    }).applyStateful(applyStatefulFunction).flatMap(flatMapFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 4 with MISTFunction

use of edu.snu.mist.common.functions.MISTFunction in project mist by snuspl.

the class SessionWindow method submitQuery.

/**
 * Submit a query.
 * The query reads strings from a source server, puts them into a window,
 * and if there is no incoming data during the interval of the session window,
 * the session will be closed.
 * The query will print out the data in the session, and a new session is created
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    // configurations for source and sink
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    // configurations for windowing and aggregation by session dependent on time
    final int sessionInterval = 5000;
    final MISTFunction<WindowData<String>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", window is started at " + windowData.getStart() + ", window is ended at " + windowData.getEnd() + ".";
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).window(new SessionWindowInformation(sessionInterval)).aggregateWindow(aggregateFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    System.out.println("End of submitQuery");
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 5 with MISTFunction

use of edu.snu.mist.common.functions.MISTFunction in project mist by snuspl.

the class PhysicalObjectGenerator method newEventGenerator.

/**
 * Get a new event generator.
 * @param conf configuration
 * @param classLoader external class loader
 * @param <T> event type
 * @return event generator
 */
@SuppressWarnings("unchecked")
public <T> EventGenerator<T> newEventGenerator(final Map<String, String> conf, final ClassLoader classLoader) throws IOException, ClassNotFoundException {
    final String type = conf.get(ConfKeys.Watermark.EVENT_GENERATOR.name());
    final String tefString = conf.get(ConfKeys.SourceConf.TIMESTAMP_EXTRACT_FUNC.name());
    final MISTFunction timestampExtractFunc;
    if (tefString == null) {
        timestampExtractFunc = null;
    } else {
        timestampExtractFunc = SerializeUtils.deserializeFromString(conf.get(ConfKeys.SourceConf.TIMESTAMP_EXTRACT_FUNC.name()), classLoader);
    }
    if (type.equals(ConfValues.EventGeneratorType.PERIODIC_EVENT_GEN.name())) {
        // periodic event generator
        final long period = Long.valueOf(conf.get(ConfKeys.Watermark.PERIODIC_WATERMARK_PERIOD.name()));
        final long delay = Long.valueOf(conf.get(ConfKeys.Watermark.PERIODIC_WATERMARK_DELAY.name()));
        return new PeriodicEventGenerator(timestampExtractFunc, period, checkpointPeriod, delay, watermarkTimeUnit, scheduler);
    } else if (type.equals(ConfValues.EventGeneratorType.PUNCTUATED_EVENT_GEN.name())) {
        // punctuated event generator
        final MISTPredicate watermarkPredicate = SerializeUtils.deserializeFromString(conf.get(ConfKeys.Watermark.WATERMARK_PREDICATE.name()), classLoader);
        final WatermarkTimestampFunction tf = SerializeUtils.deserializeFromString(conf.get(ConfKeys.Watermark.TIMESTAMP_PARSE_OBJECT.name()), classLoader);
        return new PunctuatedEventGenerator(timestampExtractFunc, watermarkPredicate, tf, checkpointPeriod, watermarkTimeUnit, scheduler);
    } else {
        throw new RuntimeException("Invalid event generator: " + type);
    }
}