Examples with SourceConfiguration edu.snu.mist.client.datastreams.configurations.SourceConfiguration used on opensource projects

Example 1 with SourceConfiguration

use of edu.snu.mist.client.datastreams.configurations.SourceConfiguration in project mist by snuspl.

the class SessionWindow method submitQuery.

/**
 * Submit a query.
 * The query reads strings from a source server, puts them into a window,
 * and if there is no incoming data during the interval of the session window,
 * the session will be closed.
 * The query will print out the data in the session, and a new session is created
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    // configurations for source and sink
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    // configurations for windowing and aggregation by session dependent on time
    final int sessionInterval = 5000;
    final MISTFunction<WindowData<String>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", window is started at " + windowData.getStart() + ", window is ended at " + windowData.getEnd() + ".";
    };
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).window(new SessionWindowInformation(sessionInterval)).aggregateWindow(aggregateFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    System.out.println("End of submitQuery");
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 2 with SourceConfiguration

use of edu.snu.mist.client.datastreams.configurations.SourceConfiguration in project mist by snuspl.

the class KMeansClustering method submitQuery.

/**
 * Submit a query doing k-means clustering.
 * The query receives inputs in a form of two-dimensional double point "x1.x2,y1.y2",
 * conducts online k-means clustering, and displays the result of clustering.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    final MISTFunction<List<Cluster>, List<String>> flatMapFunc = // parse clustering result into String list
    (clusterList) -> {
        final List<String> results = new LinkedList<>();
        for (final Cluster cluster : clusterList) {
            String clusterResult = "Cluster id: " + cluster.getId() + ", cluster center: " + cluster.getCenter().toString() + "\n";
            for (final Point point : cluster.getClusteredPoints()) {
                clusterResult += point.toString() + "\n";
            }
            results.add(clusterResult);
        }
        return results;
    };
    final ApplyStatefulFunction<Point, List<Cluster>> applyStatefulFunction = new KMeansFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).map(s -> s.replaceAll(" ", "")).filter(s -> Pattern.matches("-?\\d+\\.\\d+,-?\\d+\\.\\d+", s)).map(s -> {
        final String[] array = s.split(",");
        return new Point(Double.parseDouble(array[0]), Double.parseDouble(array[1]));
    }).applyStateful(applyStatefulFunction).flatMap(flatMapFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 3 with SourceConfiguration

use of edu.snu.mist.client.datastreams.configurations.SourceConfiguration in project mist by snuspl.

the class MqttFilterApplication method submitQuery.

public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String brokerURI = injector.getNamedInstance(TestMQTTBrokerURI.class);
    final String sourceTopic = injector.getNamedInstance(MqttSourceTopic.class);
    final String sinkTopic = injector.getNamedInstance(MqttSinkTopic.class);
    final String filteredString = injector.getNamedInstance(FilteredString.class);
    final String appId = injector.getNamedInstance(ApplicationIdentifier.class);
    final SourceConfiguration mqttSourceConf = new MQTTSourceConfiguration.MQTTSourceConfigurationBuilder().setBrokerURI(brokerURI).setTopic(sourceTopic).build();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(appId).mqttStream(mqttSourceConf).map(new MqttMessageToStringFunc()).filter(new FilterFunc(filteredString)).map(new MapFunc(filteredString)).mqttOutput(brokerURI, sinkTopic);
    final String[] driverSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(MasterAddress.class).split(":");
    final String driverHostname = driverSocket[0];
    final int driverPort = Integer.parseInt(driverSocket[1]);
    try (final MISTExecutionEnvironment executionEnvironment = new MISTDefaultExecutionEnvironmentImpl(driverHostname, driverPort)) {
        return executionEnvironment.submitQuery(queryBuilder.build());
    } catch (final Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}

Example 4 with SourceConfiguration

use of edu.snu.mist.client.datastreams.configurations.SourceConfiguration in project mist by snuspl.

the class JoinAndApplyStateful method submitQuery.

/**
 * Submit a query joining two sources and apply stateful operation on them.
 * The query reads strings from two source servers, windows them, and
 * prints out the result of apply stateful operation on the window
 * that contains the data which satisfies a predicate.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String source1Socket = injector.getNamedInstance(UnionLeftSourceAddress.class);
    final String source2Socket = injector.getNamedInstance(UnionRightSourceAddress.class);
    final SourceConfiguration localTextSocketSource1Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source1Socket);
    final SourceConfiguration localTextSocketSource2Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source2Socket);
    final MISTBiPredicate<String, String> joinPred = (s1, s2) -> s1.equals(s2);
    final ApplyStatefulFunction<Tuple2<String, String>, String> applyStatefulFunction = new FoldStringTupleFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream sourceStream1 = queryBuilder.socketTextStream(localTextSocketSource1Conf);
    final ContinuousStream sourceStream2 = queryBuilder.socketTextStream(localTextSocketSource2Conf);
    sourceStream1.join(sourceStream2, joinPred, new TimeWindowInformation(5000, 5000)).applyStatefulWindow(applyStatefulFunction).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 5 with SourceConfiguration

use of edu.snu.mist.client.datastreams.configurations.SourceConfiguration in project mist by snuspl.

the class KafkaSource method submitQuery.

/**
 * Submit a query fetching data from a kafka source.
 * The query reads strings from a kafka topic and send them to a sink server.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(KafkaSourceAddress.class);
    final SourceConfiguration localKafkaSourceConf = MISTExampleUtils.getLocalKafkaSourceConf("KafkaSource", sourceSocket);
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.kafkaStream(localKafkaSourceConf).map(consumerRecord -> ((ConsumerRecord) consumerRecord).value()).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}