Examples with NettyTextSink edu.snu.mist.core.sinks.NettyTextSink used on opensource projects

Example 1 with NettyTextSink

use of edu.snu.mist.core.sinks.NettyTextSink in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}

Example 2 with NettyTextSink

use of edu.snu.mist.core.sinks.NettyTextSink in project mist by snuspl.

the class PhysicalObjectGeneratorTest method testSuccessOfSink.

@Test
public void testSuccessOfSink() throws IOException, InjectionException {
    final int port = 13667;
    final ServerSocket socket = new ServerSocket(port);
    final Map<String, String> conf = new HashMap<String, String>() {

        {
            put(ConfKeys.SinkConf.SINK_TYPE.name(), ConfValues.SinkType.NETTY.name());
            put(ConfKeys.NettySink.SINK_ADDRESS.name(), "localhost");
            put(ConfKeys.NettySink.SINK_PORT.name(), String.valueOf(port));
        }
    };
    final Sink sink = generator.newSink(conf, classLoader);
    Assert.assertTrue(sink instanceof NettyTextSink);
    socket.close();
}

Example 3 with NettyTextSink

use of edu.snu.mist.core.sinks.NettyTextSink in project mist by snuspl.

the class PhysicalObjectGenerator method newSink.

/**
 * Get a new sink.
 * @param conf configuration
 * @param classLoader external class loader
 * @return new sink
 */
@SuppressWarnings("unchecked")
public <T> Sink<T> newSink(final Map<String, String> conf, final ClassLoader classLoader) throws IOException {
    final String type = conf.get(ConfKeys.SinkConf.SINK_TYPE.name());
    if (type.equals(ConfValues.SinkType.NETTY.name())) {
        final String serverAddress = conf.get(ConfKeys.NettySink.SINK_ADDRESS.name());
        final int serverPort = Integer.valueOf(conf.get(ConfKeys.NettySink.SINK_PORT.name()));
        return (Sink<T>) new NettyTextSink(serverAddress, serverPort, nettySharedResource, identifierFactory);
    } else if (type.equals(ConfValues.SinkType.MQTT.name())) {
        final String brokerURI = conf.get(ConfKeys.MqttSink.MQTT_SINK_BROKER_URI.name());
        final String topic = conf.get(ConfKeys.MqttSink.MQTT_SINK_TOPIC.name());
        try {
            return (Sink<T>) new MqttSink(brokerURI, topic, mqttSharedResource);
        } catch (final MqttException e) {
            e.printStackTrace();
            throw new IOException(e);
        }
    } else {
        throw new RuntimeException("Invalid sink type: " + type);
    }
}