Examples with MISTEdge edu.snu.mist.common.graph.MISTEdge used on opensource projects

Example 1 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class LogicalDagOptimizerTest method testConditionalBranch.

/**
 * Test conditional branch.
 * logical dag:
 *                            -> op2 -> sink1
 *             -> cbr1 (idx 0)-> op3 -> sink2
 * src1 -> op1 -> cbr2 (idx 0)-------->
 *             -> cbr3 (idx 0)-> op4 -> union -> sink3
 *
 * should be converted to the expected optimized dag:
 *                      (idx 1)-> op2 -> sink1
 *                      (idx 1)-> op3 -> sink2
 * src1 -> op1 -> cbrOp (idx 2)-------->
 *                      (idx 3)-> op4 -> union -> sink3
 */
@Test
public void testConditionalBranch() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op1 = src1.filter((x) -> true);
    final ContinuousStream<String> cbr1 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr2 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr3 = op1.routeIf((x) -> true);
    final ContinuousStream<String> op2 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op3 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op4 = cbr3.filter((x) -> true);
    final ContinuousStream<String> union = op4.union(cbr2);
    final MISTStream<String> sink1 = op2.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op3.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink3 = union.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final LogicalDagOptimizer logicalDagOptimizer = new LogicalDagOptimizer(dag);
    final DAG<MISTStream, MISTEdge> optimizedDAG = logicalDagOptimizer.getOptimizedDAG();
    // Check src1 -> op1
    final Map<MISTStream, MISTEdge> e1 = optimizedDAG.getEdges(src1);
    final Map<MISTStream, MISTEdge> result1 = new HashMap<>();
    result1.put(op1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check op1 -> cbrOp
    final Map<MISTStream, MISTEdge> e2 = optimizedDAG.getEdges(op1);
    Assert.assertEquals(1, e2.size());
    // a new vertex cbrOp would be created during the optimization
    final MISTStream cbrOp = e2.entrySet().iterator().next().getKey();
    Assert.assertTrue(cbrOp instanceof ContinuousStreamImpl);
    // (idx1)-> op2
    // (idx1)-> op3
    // Check cbrOp (idx2)--------> union
    // (idx3)-> op4
    final Map<MISTStream, MISTEdge> e3 = optimizedDAG.getEdges(cbrOp);
    final Map<MISTStream, MISTEdge> result3 = new HashMap<>();
    result3.put(op2, new MISTEdge(Direction.LEFT, 1));
    result3.put(op3, new MISTEdge(Direction.LEFT, 1));
    result3.put(union, new MISTEdge(Direction.RIGHT, 2));
    result3.put(op4, new MISTEdge(Direction.LEFT, 3));
    Assert.assertEquals(e3, result3);
    // Check op4 -> union
    final Map<MISTStream, MISTEdge> e4 = optimizedDAG.getEdges(op4);
    final Map<MISTStream, MISTEdge> result4 = new HashMap<>();
    result4.put(union, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check op2 -> sink1
    final Map<MISTStream, MISTEdge> e5 = optimizedDAG.getEdges(op2);
    final Map<MISTStream, MISTEdge> result5 = new HashMap<>();
    result5.put(sink1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
    // Check op3 -> sink2
    final Map<MISTStream, MISTEdge> e6 = optimizedDAG.getEdges(op3);
    final Map<MISTStream, MISTEdge> result6 = new HashMap<>();
    result6.put(sink2, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check union -> sink3
    final Map<MISTStream, MISTEdge> e7 = optimizedDAG.getEdges(union);
    final Map<MISTStream, MISTEdge> result7 = new HashMap<>();
    result7.put(sink3, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e7, result7);
}

Example 2 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class OperatorChainDagGeneratorTest method testMergingQueryPartitioning.

/**
 * Test merge chaining.
 * logical dag:
 * src1 -> op11 -> op12 ->
 * src2 ---------> op21 -> op13 -> op14 -> sink1
 * src3 -----------------> op31 ->
 *
 * should be converted to the expected operator chain dag:
 * src1 -> [op11 -> op12] ->
 * src2 ---------> [op21] -> [op13] -> [op14] -> sink1
 * src3 -------------------> [op31] ->
 */
@Test
public void testMergingQueryPartitioning() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> src2 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> src3 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op11 = src1.filter((x) -> true);
    final ContinuousStream<String> op12 = op11.filter((x) -> true);
    final ContinuousStream<String> op21 = src2.filter((x) -> true);
    final ContinuousStream<String> op13 = op12.union(op21);
    final ContinuousStream<String> op31 = src3.filter((x) -> true);
    final ContinuousStream<String> op14 = op13.union(op31);
    final MISTStream<String> sink1 = op14.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final OperatorChainDagGenerator chainDagGenerator = new OperatorChainDagGenerator(dag);
    final DAG<List<MISTStream>, MISTEdge> operatorChainDag = chainDagGenerator.generateOperatorChainDAG();
    // Expected outputs
    final List<MISTStream> src1List = Arrays.asList(src1);
    final List<MISTStream> src2List = Arrays.asList(src2);
    final List<MISTStream> src3List = Arrays.asList(src3);
    final List<MISTStream> op11op12 = Arrays.asList(op11, op12);
    final List<MISTStream> op21List = Arrays.asList(op21);
    final List<MISTStream> op13List = Arrays.asList(op13);
    final List<MISTStream> op31List = Arrays.asList(op31);
    final List<MISTStream> op14List = Arrays.asList(op14);
    final List<MISTStream> sink1List = Arrays.asList(sink1);
    // Check src1 -> [op11->op12] edge
    final Map<List<MISTStream>, MISTEdge> e1 = operatorChainDag.getEdges(src1List);
    final Map<List<MISTStream>, MISTEdge> result1 = new HashMap<>();
    result1.put(op11op12, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check src2 -> [op21] edge
    final Map<List<MISTStream>, MISTEdge> e2 = operatorChainDag.getEdges(src2List);
    final Map<List<MISTStream>, MISTEdge> result2 = new HashMap<>();
    result2.put(op21List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e2, result2);
    // Check src3 -> [op31] edge
    final Map<List<MISTStream>, MISTEdge> e3 = operatorChainDag.getEdges(src3List);
    final Map<List<MISTStream>, MISTEdge> result3 = new HashMap<>();
    result3.put(op31List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e3, result3);
    // Check [op11->op12] -> [op13] edge
    final Map<List<MISTStream>, MISTEdge> e4 = operatorChainDag.getEdges(op11op12);
    final Map<List<MISTStream>, MISTEdge> result4 = new HashMap<>();
    result4.put(op13List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check [op21] -> [op13] edge
    final Map<List<MISTStream>, MISTEdge> e5 = operatorChainDag.getEdges(op21List);
    final Map<List<MISTStream>, MISTEdge> result5 = new HashMap<>();
    result5.put(op13List, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(e5, result5);
    // Check [op13] -> [op14] edge
    final Map<List<MISTStream>, MISTEdge> e6 = operatorChainDag.getEdges(op13List);
    final Map<List<MISTStream>, MISTEdge> result6 = new HashMap<>();
    result6.put(op14List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check [op31] -> [op14] edge
    final Map<List<MISTStream>, MISTEdge> e7 = operatorChainDag.getEdges(op31List);
    final Map<List<MISTStream>, MISTEdge> result7 = new HashMap<>();
    result7.put(op14List, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(e7, result7);
    // Check [op14] -> [sink1] edge
    final Map<List<MISTStream>, MISTEdge> e8 = operatorChainDag.getEdges(op14List);
    final Map<List<MISTStream>, MISTEdge> result8 = new HashMap<>();
    result8.put(sink1List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e8, result8);
}

Example 3 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class OperatorChainDagGeneratorTest method testBranchTest.

/**
 * Test branch chaining.
 * logical dag:
 *                      -> op14 -> sink2
 * src1 -> op11 -> op12 -> op13 -> sink1
 *                      -> op15 -> sink3
 * should be converted to the expected chained dag:
 *                        -> [op14] -> sink2
 * src1 -> [op11 -> op12] -> [op13] -> sink1
 *                        -> [op15] -> sink3
 */
@Test
public void testBranchTest() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op11 = src1.filter((x) -> true);
    final ContinuousStream<String> op12 = op11.filter((x) -> true);
    final ContinuousStream<String> op13 = op12.filter((x) -> true);
    final ContinuousStream<String> op14 = op12.filter((x) -> true);
    final ContinuousStream<String> op15 = op12.filter((x) -> true);
    final MISTStream<String> sink1 = op13.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op14.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink3 = op15.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final OperatorChainDagGenerator chainDagGenerator = new OperatorChainDagGenerator(dag);
    final DAG<List<MISTStream>, MISTEdge> chainPlan = chainDagGenerator.generateOperatorChainDAG();
    // Expected outputs
    final List<MISTStream> src1List = Arrays.asList(src1);
    final List<MISTStream> op11op12 = Arrays.asList(op11, op12);
    final List<MISTStream> op13List = Arrays.asList(op13);
    final List<MISTStream> op14List = Arrays.asList(op14);
    final List<MISTStream> op15List = Arrays.asList(op15);
    final List<MISTStream> sink1List = Arrays.asList(sink1);
    final List<MISTStream> sink2List = Arrays.asList(sink2);
    final List<MISTStream> sink3List = Arrays.asList(sink3);
    // Check src1 -> [op11->op12] edge
    final Map<List<MISTStream>, MISTEdge> e1 = chainPlan.getEdges(src1List);
    final Map<List<MISTStream>, MISTEdge> result1 = new HashMap<>();
    result1.put(op11op12, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check [op11->op12] -> [op13] edges
    // -> [op14]
    // -> [op15]
    final Map<List<MISTStream>, MISTEdge> e2 = chainPlan.getEdges(op11op12);
    final Map<List<MISTStream>, MISTEdge> result2 = new HashMap<>();
    result2.put(op13List, new MISTEdge(Direction.LEFT));
    result2.put(op14List, new MISTEdge(Direction.LEFT));
    result2.put(op15List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e2, result2);
    // Check [op14] -> [sink2] edge
    final Map<List<MISTStream>, MISTEdge> e3 = chainPlan.getEdges(op14List);
    final Map<List<MISTStream>, MISTEdge> result3 = new HashMap<>();
    result3.put(sink2List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e3, result3);
    // Check [op13] -> [sink1] edge
    final Map<List<MISTStream>, MISTEdge> e4 = chainPlan.getEdges(op13List);
    final Map<List<MISTStream>, MISTEdge> result4 = new HashMap<>();
    result4.put(sink1List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check [op15] -> [sink3] edge
    final Map<List<MISTStream>, MISTEdge> e5 = chainPlan.getEdges(op15List);
    final Map<List<MISTStream>, MISTEdge> result5 = new HashMap<>();
    result5.put(sink3List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
}

Example 4 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamTest method testBasicOperatorStream.

/**
 * Test for basic stateless OperatorStreams.
 */
@Test
public void testBasicOperatorStream() throws IOException {
    final Map<String, String> filteredConf = filteredMappedStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(defaultMap), filteredConf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    // Check src -> filiter
    final Map<MISTStream, MISTEdge> neighbors = dag.getEdges(sourceStream);
    Assert.assertEquals(1, neighbors.size());
    final MISTEdge edgeInfo = neighbors.get(filteredStream);
    Assert.assertEquals(Direction.LEFT, edgeInfo.getDirection());
    Assert.assertEquals(0, edgeInfo.getIndex());
    // Check filter -> map
    final Map<MISTStream, MISTEdge> neighbors2 = dag.getEdges(filteredStream);
    Assert.assertEquals(1, neighbors2.size());
    final MISTEdge edgeInfo2 = neighbors2.get(filteredMappedStream);
    Assert.assertEquals(Direction.LEFT, edgeInfo2.getDirection());
    Assert.assertEquals(0, edgeInfo2.getIndex());
}

Example 5 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ContinuousStreamTest method testJoinOperatorStream.

/**
 * Test for join operation.
 */
@Test
public void testJoinOperatorStream() throws InjectionException, IOException, ClassNotFoundException {
    final ContinuousStream<String> firstInputStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> secondInputStream = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final MISTBiPredicate<String, String> joinBiPred = (string1, string2) -> string1.equals(string2);
    final WindowedStream<Tuple2<String, String>> joinedStream = firstInputStream.join(secondInputStream, joinBiPred, new CountWindowInformation(5, 3));
    final Map<String, String> conf = joinedStream.getConfiguration();
    Assert.assertEquals(SerializeUtils.serializeToString(joinBiPred), conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
    // Check first input -> mapped
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final MISTStream firstMappedInputStream = getNextOperatorStream(dag, 1, firstInputStream, new MISTEdge(Direction.LEFT));
    // Check second input -> mapped
    final MISTStream secondMappedInputStream = getNextOperatorStream(dag, 1, secondInputStream, new MISTEdge(Direction.LEFT));
    // Check two mapped input -> unified
    final MISTStream firstUnifiedStream = getNextOperatorStream(dag, 1, firstMappedInputStream, new MISTEdge(Direction.LEFT));
    final MISTStream secondUnifiedStream = getNextOperatorStream(dag, 1, secondMappedInputStream, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(firstUnifiedStream, secondUnifiedStream);
    // Check unified stream -> windowed
    final MISTStream windowedStream = getNextOperatorStream(dag, 1, firstUnifiedStream, new MISTEdge(Direction.LEFT));
    // Check windowed stream -> joined
    checkEdges(dag, 1, windowedStream, joinedStream, new MISTEdge(Direction.LEFT));
}