Search in sources :

Example 6 with InjectionException

use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.

the class JoinAndApplyStateful method submitQuery.

/**
 * Submit a query joining two sources and apply stateful operation on them.
 * The query reads strings from two source servers, windows them, and
 * prints out the result of apply stateful operation on the window
 * that contains the data which satisfies a predicate.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String source1Socket = injector.getNamedInstance(UnionLeftSourceAddress.class);
    final String source2Socket = injector.getNamedInstance(UnionRightSourceAddress.class);
    final SourceConfiguration localTextSocketSource1Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source1Socket);
    final SourceConfiguration localTextSocketSource2Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source2Socket);
    final MISTBiPredicate<String, String> joinPred = (s1, s2) -> s1.equals(s2);
    final ApplyStatefulFunction<Tuple2<String, String>, String> applyStatefulFunction = new FoldStringTupleFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream sourceStream1 = queryBuilder.socketTextStream(localTextSocketSource1Conf);
    final ContinuousStream sourceStream2 = queryBuilder.socketTextStream(localTextSocketSource2Conf);
    sourceStream1.join(sourceStream2, joinPred, new TimeWindowInformation(5000, 5000)).applyStatefulWindow(applyStatefulFunction).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}
Also used : TimeWindowInformation(edu.snu.mist.common.windows.TimeWindowInformation) Injector(org.apache.reef.tang.Injector) Tang(org.apache.reef.tang.Tang) SourceConfiguration(edu.snu.mist.client.datastreams.configurations.SourceConfiguration) Tuple2(edu.snu.mist.common.types.Tuple2) URISyntaxException(java.net.URISyntaxException) Collection(java.util.Collection) IOException(java.io.IOException) CommandLine(org.apache.reef.tang.formats.CommandLine) ContinuousStream(edu.snu.mist.client.datastreams.ContinuousStream) APIQueryControlResult(edu.snu.mist.client.APIQueryControlResult) UnionLeftSourceAddress(edu.snu.mist.examples.parameters.UnionLeftSourceAddress) MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) MISTBiPredicate(edu.snu.mist.common.functions.MISTBiPredicate) Configuration(org.apache.reef.tang.Configuration) ApplyStatefulFunction(edu.snu.mist.common.functions.ApplyStatefulFunction) JavaConfigurationBuilder(org.apache.reef.tang.JavaConfigurationBuilder) InjectionException(org.apache.reef.tang.exceptions.InjectionException) LinkedList(java.util.LinkedList) UnionRightSourceAddress(edu.snu.mist.examples.parameters.UnionRightSourceAddress) MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) ContinuousStream(edu.snu.mist.client.datastreams.ContinuousStream) Injector(org.apache.reef.tang.Injector) Tuple2(edu.snu.mist.common.types.Tuple2) SourceConfiguration(edu.snu.mist.client.datastreams.configurations.SourceConfiguration) TimeWindowInformation(edu.snu.mist.common.windows.TimeWindowInformation)

Example 7 with InjectionException

use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.

the class KMeansClustering method submitQuery.

/**
 * Submit a query doing k-means clustering.
 * The query receives inputs in a form of two-dimensional double point "x1.x2,y1.y2",
 * conducts online k-means clustering, and displays the result of clustering.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
    final MISTFunction<List<Cluster>, List<String>> flatMapFunc = // parse clustering result into String list
    (clusterList) -> {
        final List<String> results = new LinkedList<>();
        for (final Cluster cluster : clusterList) {
            String clusterResult = "Cluster id: " + cluster.getId() + ", cluster center: " + cluster.getCenter().toString() + "\n";
            for (final Point point : cluster.getClusteredPoints()) {
                clusterResult += point.toString() + "\n";
            }
            results.add(clusterResult);
        }
        return results;
    };
    final ApplyStatefulFunction<Point, List<Cluster>> applyStatefulFunction = new KMeansFunction();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.socketTextStream(localTextSocketSourceConf).map(s -> s.replaceAll(" ", "")).filter(s -> Pattern.matches("-?\\d+\\.\\d+,-?\\d+\\.\\d+", s)).map(s -> {
        final String[] array = s.split(",");
        return new Point(Double.parseDouble(array[0]), Double.parseDouble(array[1]));
    }).applyStateful(applyStatefulFunction).flatMap(flatMapFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}
Also used : Tang(org.apache.reef.tang.Tang) SourceConfiguration(edu.snu.mist.client.datastreams.configurations.SourceConfiguration) MISTFunction(edu.snu.mist.common.functions.MISTFunction) NettySourceAddress(edu.snu.mist.examples.parameters.NettySourceAddress) URISyntaxException(java.net.URISyntaxException) Collection(java.util.Collection) IOException(java.io.IOException) CommandLine(org.apache.reef.tang.formats.CommandLine) APIQueryControlResult(edu.snu.mist.client.APIQueryControlResult) List(java.util.List) MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) Configuration(org.apache.reef.tang.Configuration) ApplyStatefulFunction(edu.snu.mist.common.functions.ApplyStatefulFunction) JavaConfigurationBuilder(org.apache.reef.tang.JavaConfigurationBuilder) InjectionException(org.apache.reef.tang.exceptions.InjectionException) Pattern(java.util.regex.Pattern) LinkedList(java.util.LinkedList) MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) SourceConfiguration(edu.snu.mist.client.datastreams.configurations.SourceConfiguration) List(java.util.List) LinkedList(java.util.LinkedList)

Example 8 with InjectionException

use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.

the class KafkaSource method submitQuery.

/**
 * Submit a query fetching data from a kafka source.
 * The query reads strings from a kafka topic and send them to a sink server.
 * @return result of the submission
 * @throws IOException
 * @throws InjectionException
 */
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(KafkaSourceAddress.class);
    final SourceConfiguration localKafkaSourceConf = MISTExampleUtils.getLocalKafkaSourceConf("KafkaSource", sourceSocket);
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.kafkaStream(localKafkaSourceConf).map(consumerRecord -> ((ConsumerRecord) consumerRecord).value()).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}
Also used : MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) Tang(org.apache.reef.tang.Tang) SourceConfiguration(edu.snu.mist.client.datastreams.configurations.SourceConfiguration) NettySourceAddress(edu.snu.mist.examples.parameters.NettySourceAddress) URISyntaxException(java.net.URISyntaxException) IOException(java.io.IOException) CommandLine(org.apache.reef.tang.formats.CommandLine) APIQueryControlResult(edu.snu.mist.client.APIQueryControlResult) KafkaSourceAddress(edu.snu.mist.examples.parameters.KafkaSourceAddress) MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) Configuration(org.apache.reef.tang.Configuration) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord) JavaConfigurationBuilder(org.apache.reef.tang.JavaConfigurationBuilder) InjectionException(org.apache.reef.tang.exceptions.InjectionException) SourceConfiguration(edu.snu.mist.client.datastreams.configurations.SourceConfiguration) ConsumerRecord(org.apache.kafka.clients.consumer.ConsumerRecord)

Example 9 with InjectionException

use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.

the class MqttFilterApplication method submitQuery.

public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final Injector injector = Tang.Factory.getTang().newInjector(configuration);
    final String brokerURI = injector.getNamedInstance(TestMQTTBrokerURI.class);
    final String sourceTopic = injector.getNamedInstance(MqttSourceTopic.class);
    final String sinkTopic = injector.getNamedInstance(MqttSinkTopic.class);
    final String filteredString = injector.getNamedInstance(FilteredString.class);
    final String appId = injector.getNamedInstance(ApplicationIdentifier.class);
    final SourceConfiguration mqttSourceConf = new MQTTSourceConfiguration.MQTTSourceConfigurationBuilder().setBrokerURI(brokerURI).setTopic(sourceTopic).build();
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(appId).mqttStream(mqttSourceConf).map(new MqttMessageToStringFunc()).filter(new FilterFunc(filteredString)).map(new MapFunc(filteredString)).mqttOutput(brokerURI, sinkTopic);
    final String[] driverSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(MasterAddress.class).split(":");
    final String driverHostname = driverSocket[0];
    final int driverPort = Integer.parseInt(driverSocket[1]);
    try (final MISTExecutionEnvironment executionEnvironment = new MISTDefaultExecutionEnvironmentImpl(driverHostname, driverPort)) {
        return executionEnvironment.submitQuery(queryBuilder.build());
    } catch (final Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    }
}
Also used : MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) URISyntaxException(java.net.URISyntaxException) IOException(java.io.IOException) InjectionException(org.apache.reef.tang.exceptions.InjectionException) Injector(org.apache.reef.tang.Injector) MQTTSourceConfiguration(edu.snu.mist.client.datastreams.configurations.MQTTSourceConfiguration) SourceConfiguration(edu.snu.mist.client.datastreams.configurations.SourceConfiguration) MISTDefaultExecutionEnvironmentImpl(edu.snu.mist.client.MISTDefaultExecutionEnvironmentImpl) MISTExecutionEnvironment(edu.snu.mist.client.MISTExecutionEnvironment) MQTTSourceConfiguration(edu.snu.mist.client.datastreams.configurations.MQTTSourceConfiguration)

Example 10 with InjectionException

use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}
Also used : Injector(org.apache.reef.tang.Injector) java.util(java.util) NettyTextSink(edu.snu.mist.core.sinks.NettyTextSink) Tuple2(edu.snu.mist.common.types.Tuple2) URISyntaxException(java.net.URISyntaxException) MISTEdge(edu.snu.mist.common.graph.MISTEdge) AvroDag(edu.snu.mist.formats.avro.AvroDag) MISTQuery(edu.snu.mist.client.MISTQuery) ReduceByKeyOperator(edu.snu.mist.core.operators.ReduceByKeyOperator) ServerSocket(java.net.ServerSocket) DAG(edu.snu.mist.common.graph.DAG) MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) After(org.junit.After) MapOperator(edu.snu.mist.core.operators.MapOperator) Before(org.junit.Before) Tang(org.apache.reef.tang.Tang) Tuple(org.apache.reef.io.Tuple) Test(org.junit.Test) IOException(java.io.IOException) TestParameters(edu.snu.mist.core.utils.TestParameters) TaskHostname(edu.snu.mist.core.parameters.TaskHostname) FilterOperator(edu.snu.mist.core.operators.FilterOperator) Edge(edu.snu.mist.formats.avro.Edge) FlatMapOperator(edu.snu.mist.core.operators.FlatMapOperator) JavaConfigurationBuilder(org.apache.reef.tang.JavaConfigurationBuilder) AvroVertex(edu.snu.mist.formats.avro.AvroVertex) InjectionException(org.apache.reef.tang.exceptions.InjectionException) Assert(org.junit.Assert) FlatMapOperator(edu.snu.mist.core.operators.FlatMapOperator) NettyTextSink(edu.snu.mist.core.sinks.NettyTextSink) AvroDag(edu.snu.mist.formats.avro.AvroDag) MapOperator(edu.snu.mist.core.operators.MapOperator) FlatMapOperator(edu.snu.mist.core.operators.FlatMapOperator) Injector(org.apache.reef.tang.Injector) JavaConfigurationBuilder(org.apache.reef.tang.JavaConfigurationBuilder) MISTEdge(edu.snu.mist.common.graph.MISTEdge) MISTQueryBuilder(edu.snu.mist.client.MISTQueryBuilder) FilterOperator(edu.snu.mist.core.operators.FilterOperator) ReduceByKeyOperator(edu.snu.mist.core.operators.ReduceByKeyOperator) MISTQuery(edu.snu.mist.client.MISTQuery) Tuple(org.apache.reef.io.Tuple) Test(org.junit.Test)

Aggregations

InjectionException (org.apache.reef.tang.exceptions.InjectionException)24 IOException (java.io.IOException)17 MISTQueryBuilder (edu.snu.mist.client.MISTQueryBuilder)16 URISyntaxException (java.net.URISyntaxException)14 Configuration (org.apache.reef.tang.Configuration)14 Tang (org.apache.reef.tang.Tang)13 JavaConfigurationBuilder (org.apache.reef.tang.JavaConfigurationBuilder)12 APIQueryControlResult (edu.snu.mist.client.APIQueryControlResult)11 SourceConfiguration (edu.snu.mist.client.datastreams.configurations.SourceConfiguration)11 CommandLine (org.apache.reef.tang.formats.CommandLine)11 Tuple2 (edu.snu.mist.common.types.Tuple2)9 MISTFunction (edu.snu.mist.common.functions.MISTFunction)8 NettySourceAddress (edu.snu.mist.examples.parameters.NettySourceAddress)8 Injector (org.apache.reef.tang.Injector)8 Assert (org.junit.Assert)8 Test (org.junit.Test)8 LinkedList (java.util.LinkedList)7 List (java.util.List)7 MISTBiFunction (edu.snu.mist.common.functions.MISTBiFunction)5 Tuple (org.apache.reef.io.Tuple)5