use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.
the class JoinAndApplyStateful method submitQuery.
/**
* Submit a query joining two sources and apply stateful operation on them.
* The query reads strings from two source servers, windows them, and
* prints out the result of apply stateful operation on the window
* that contains the data which satisfies a predicate.
* @return result of the submission
* @throws IOException
* @throws InjectionException
*/
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
final Injector injector = Tang.Factory.getTang().newInjector(configuration);
final String source1Socket = injector.getNamedInstance(UnionLeftSourceAddress.class);
final String source2Socket = injector.getNamedInstance(UnionRightSourceAddress.class);
final SourceConfiguration localTextSocketSource1Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source1Socket);
final SourceConfiguration localTextSocketSource2Conf = MISTExampleUtils.getLocalTextSocketSourceConf(source2Socket);
final MISTBiPredicate<String, String> joinPred = (s1, s2) -> s1.equals(s2);
final ApplyStatefulFunction<Tuple2<String, String>, String> applyStatefulFunction = new FoldStringTupleFunction();
final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
final ContinuousStream sourceStream1 = queryBuilder.socketTextStream(localTextSocketSource1Conf);
final ContinuousStream sourceStream2 = queryBuilder.socketTextStream(localTextSocketSource2Conf);
sourceStream1.join(sourceStream2, joinPred, new TimeWindowInformation(5000, 5000)).applyStatefulWindow(applyStatefulFunction).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
return MISTExampleUtils.submit(queryBuilder, configuration);
}
use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.
the class KMeansClustering method submitQuery.
/**
* Submit a query doing k-means clustering.
* The query receives inputs in a form of two-dimensional double point "x1.x2,y1.y2",
* conducts online k-means clustering, and displays the result of clustering.
* @return result of the submission
* @throws IOException
* @throws InjectionException
*/
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
final SourceConfiguration localTextSocketSourceConf = MISTExampleUtils.getLocalTextSocketSourceConf(sourceSocket);
final MISTFunction<List<Cluster>, List<String>> flatMapFunc = // parse clustering result into String list
(clusterList) -> {
final List<String> results = new LinkedList<>();
for (final Cluster cluster : clusterList) {
String clusterResult = "Cluster id: " + cluster.getId() + ", cluster center: " + cluster.getCenter().toString() + "\n";
for (final Point point : cluster.getClusteredPoints()) {
clusterResult += point.toString() + "\n";
}
results.add(clusterResult);
}
return results;
};
final ApplyStatefulFunction<Point, List<Cluster>> applyStatefulFunction = new KMeansFunction();
final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
queryBuilder.socketTextStream(localTextSocketSourceConf).map(s -> s.replaceAll(" ", "")).filter(s -> Pattern.matches("-?\\d+\\.\\d+,-?\\d+\\.\\d+", s)).map(s -> {
final String[] array = s.split(",");
return new Point(Double.parseDouble(array[0]), Double.parseDouble(array[1]));
}).applyStateful(applyStatefulFunction).flatMap(flatMapFunc).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
return MISTExampleUtils.submit(queryBuilder, configuration);
}
use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.
the class KafkaSource method submitQuery.
/**
* Submit a query fetching data from a kafka source.
* The query reads strings from a kafka topic and send them to a sink server.
* @return result of the submission
* @throws IOException
* @throws InjectionException
*/
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(KafkaSourceAddress.class);
final SourceConfiguration localKafkaSourceConf = MISTExampleUtils.getLocalKafkaSourceConf("KafkaSource", sourceSocket);
final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
queryBuilder.kafkaStream(localKafkaSourceConf).map(consumerRecord -> ((ConsumerRecord) consumerRecord).value()).textSocketOutput(MISTExampleUtils.SINK_HOSTNAME, MISTExampleUtils.SINK_PORT);
return MISTExampleUtils.submit(queryBuilder, configuration);
}
use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.
the class MqttFilterApplication method submitQuery.
public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
final Injector injector = Tang.Factory.getTang().newInjector(configuration);
final String brokerURI = injector.getNamedInstance(TestMQTTBrokerURI.class);
final String sourceTopic = injector.getNamedInstance(MqttSourceTopic.class);
final String sinkTopic = injector.getNamedInstance(MqttSinkTopic.class);
final String filteredString = injector.getNamedInstance(FilteredString.class);
final String appId = injector.getNamedInstance(ApplicationIdentifier.class);
final SourceConfiguration mqttSourceConf = new MQTTSourceConfiguration.MQTTSourceConfigurationBuilder().setBrokerURI(brokerURI).setTopic(sourceTopic).build();
final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
queryBuilder.setApplicationId(appId).mqttStream(mqttSourceConf).map(new MqttMessageToStringFunc()).filter(new FilterFunc(filteredString)).map(new MapFunc(filteredString)).mqttOutput(brokerURI, sinkTopic);
final String[] driverSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(MasterAddress.class).split(":");
final String driverHostname = driverSocket[0];
final int driverPort = Integer.parseInt(driverSocket[1]);
try (final MISTExecutionEnvironment executionEnvironment = new MISTDefaultExecutionEnvironmentImpl(driverHostname, driverPort)) {
return executionEnvironment.submitQuery(queryBuilder.build());
} catch (final Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
use of org.apache.reef.tang.exceptions.InjectionException in project mist by snuspl.
the class DefaultDagGeneratorImplTest method testPlanGenerator.
/**
* Round-trip test of de-serializing AvroOperatorChainDag.
* @throws org.apache.reef.tang.exceptions.InjectionException
*/
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
// Generate a query
final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
final MISTQuery query = queryBuilder.build();
// Generate avro operator chain dag
final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
// Test execution dag
final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
final Set<ExecutionVertex> sources = dag.getRootVertices();
Assert.assertEquals(1, sources.size());
Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
final PhysicalSource source = (PhysicalSource) sources.iterator().next();
final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
Assert.assertEquals(1, nextOps.size());
final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}
Aggregations