Examples with MISTEdge edu.snu.mist.common.graph.MISTEdge used on opensource projects

Example 11 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamTest method testTextSocketSinkImpl.

/**
 * Test for SinkImpl.
 */
@Test
public void testTextSocketSinkImpl() throws InjectionException {
    final MISTStream<String> sink = filteredMappedStream.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final Map<String, String> conf = sink.getConfiguration();
    Assert.assertEquals(TestParameters.HOST, conf.get(ConfKeys.NettySink.SINK_ADDRESS.name()));
    Assert.assertEquals(String.valueOf(TestParameters.SINK_PORT), conf.get(ConfKeys.NettySink.SINK_PORT.name()));
    // Check src -> sink
    final DAG<MISTStream, MISTEdge> dag = queryBuilder.build().getDAG();
    final Map<MISTStream, MISTEdge> neighbors = dag.getEdges(filteredMappedStream);
    Assert.assertEquals(1, neighbors.size());
    Assert.assertEquals(new MISTEdge(Direction.LEFT), neighbors.get(sink));
}

Example 12 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamTest method testSessionWindowedStream.

/**
 * Test for creating session-based WindowedStream from ContinuousStream.
 */
@Test
public void testSessionWindowedStream() throws InjectionException {
    final int sessionInterval = 1000;
    /* Creates a test windowed stream with 1 sec session interval */
    final WindowedStream<Tuple2<String, Integer>> sessionWindowedStream = filteredMappedStream.window(new SessionWindowInformation(sessionInterval));
    // Check window info
    final Map<String, String> conf = sessionWindowedStream.getConfiguration();
    Assert.assertEquals(String.valueOf(sessionInterval), conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
    // Check map -> countWindow
    checkEdges(queryBuilder.build().getDAG(), 1, filteredMappedStream, sessionWindowedStream, new MISTEdge(Direction.LEFT));
}

Example 13 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class WindowedStreamTest method testReduceByKeyWindowStream.

/**
 * Test for reduceByKeyWindow operation.
 */
@Test
public void testReduceByKeyWindowStream() throws InjectionException, IOException {
    final MISTBiFunction<Integer, Integer, Integer> reduceFunc = (x, y) -> x + y;
    final ContinuousStream<Map<String, Integer>> reducedWindowStream = timeWindowedStream.reduceByKeyWindow(0, String.class, reduceFunc);
    // Get info
    final Map<String, String> conf = reducedWindowStream.getConfiguration();
    Assert.assertEquals("0", conf.get(ConfKeys.ReduceByKeyOperator.KEY_INDEX.name()));
    Assert.assertEquals(SerializeUtils.serializeToString(reduceFunc), conf.get(ConfKeys.ReduceByKeyOperator.MIST_BI_FUNC.name()));
    // Check windowed -> reduce by key
    checkEdges(queryBuilder.build().getDAG(), 1, timeWindowedStream, reducedWindowStream, new MISTEdge(Direction.LEFT));
}

Example 14 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class WindowedStreamTest method testApplyStatefulWindowStream.

/**
 * Test for binding the udf class of applyStatefulWindow operation.
 */
@Test
public void testApplyStatefulWindowStream() throws InjectionException, IOException {
    final ApplyStatefulFunction<Tuple2<String, Integer>, Integer> func = new UDFTestUtils.TestApplyStatefulFunction();
    final ContinuousStream<Integer> applyStatefulWindowStream = timeWindowedStream.applyStatefulWindow(new UDFTestUtils.TestApplyStatefulFunction());
    /* Simulate two data inputs on UDF stream */
    final Map<String, String> conf = applyStatefulWindowStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(func), conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    // Check windowed -> stateful operation applied
    checkEdges(queryBuilder.build().getDAG(), 1, timeWindowedStream, applyStatefulWindowStream, new MISTEdge(Direction.LEFT));
}

Example 15 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class DefaultDagGeneratorImplTest method testPlanGenerator.

/**
 * Round-trip test of de-serializing AvroOperatorChainDag.
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void testPlanGenerator() throws InjectionException, IOException, URISyntaxException, ClassNotFoundException {
    // Generate a query
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF).flatMap(s -> Arrays.asList(s.split(" "))).filter(s -> s.startsWith("A")).map(s -> new Tuple2<>(s, 1)).reduceByKey(0, String.class, (Integer x, Integer y) -> x + y).textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    // Generate avro operator chain dag
    final Tuple<List<AvroVertex>, List<Edge>> serializedDag = query.getAvroOperatorDag();
    final AvroDag.Builder avroDagBuilder = AvroDag.newBuilder();
    final AvroDag avroChainedDag = avroDagBuilder.setAppId(TestParameters.SUPER_GROUP_ID).setQueryId(TestParameters.QUERY_ID).setJarPaths(new ArrayList<>()).setAvroVertices(serializedDag.getKey()).setEdges(serializedDag.getValue()).build();
    final JavaConfigurationBuilder jcb = Tang.Factory.getTang().newConfigurationBuilder();
    jcb.bindNamedParameter(TaskHostname.class, "127.0.0.1");
    final Injector injector = Tang.Factory.getTang().newInjector(jcb.build());
    final ConfigDagGenerator configDagGenerator = injector.getInstance(ConfigDagGenerator.class);
    final DagGenerator dagGenerator = injector.getInstance(DagGenerator.class);
    final Tuple<String, AvroDag> tuple = new Tuple<>("query-test", avroChainedDag);
    final DAG<ConfigVertex, MISTEdge> configDag = configDagGenerator.generate(tuple.getValue());
    final ExecutionDag executionDag = dagGenerator.generate(configDag, new LinkedList<>());
    // Test execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    final Set<ExecutionVertex> sources = dag.getRootVertices();
    Assert.assertEquals(1, sources.size());
    Assert.assertTrue(sources.iterator().next() instanceof PhysicalSource);
    final PhysicalSource source = (PhysicalSource) sources.iterator().next();
    final Map<ExecutionVertex, MISTEdge> nextOps = dag.getEdges(source);
    Assert.assertEquals(1, nextOps.size());
    final PhysicalOperator flatMapOp = (PhysicalOperator) nextOps.entrySet().iterator().next().getKey();
    final PhysicalOperator filterOp = (PhysicalOperator) dag.getEdges(flatMapOp).entrySet().iterator().next().getKey();
    final PhysicalOperator mapOp = (PhysicalOperator) dag.getEdges(filterOp).entrySet().iterator().next().getKey();
    final PhysicalOperator reduceByKeyOp = (PhysicalOperator) dag.getEdges(mapOp).entrySet().iterator().next().getKey();
    final PhysicalSink sink = (PhysicalSink) dag.getEdges(reduceByKeyOp).entrySet().iterator().next().getKey();
    Assert.assertTrue(flatMapOp.getOperator() instanceof FlatMapOperator);
    Assert.assertTrue(filterOp.getOperator() instanceof FilterOperator);
    Assert.assertTrue(mapOp.getOperator() instanceof MapOperator);
    Assert.assertTrue(reduceByKeyOp.getOperator() instanceof ReduceByKeyOperator);
    Assert.assertTrue(sink.getSink() instanceof NettyTextSink);
}