Examples with MISTStream edu.snu.mist.client.datastreams.MISTStream used on opensource projects

Example 1 with MISTStream

use of edu.snu.mist.client.datastreams.MISTStream in project mist by snuspl.

the class LogicalDagOptimizerTest method testConditionalBranch.

/**
 * Test conditional branch.
 * logical dag:
 *                            -> op2 -> sink1
 *             -> cbr1 (idx 0)-> op3 -> sink2
 * src1 -> op1 -> cbr2 (idx 0)-------->
 *             -> cbr3 (idx 0)-> op4 -> union -> sink3
 *
 * should be converted to the expected optimized dag:
 *                      (idx 1)-> op2 -> sink1
 *                      (idx 1)-> op3 -> sink2
 * src1 -> op1 -> cbrOp (idx 2)-------->
 *                      (idx 3)-> op4 -> union -> sink3
 */
@Test
public void testConditionalBranch() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op1 = src1.filter((x) -> true);
    final ContinuousStream<String> cbr1 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr2 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr3 = op1.routeIf((x) -> true);
    final ContinuousStream<String> op2 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op3 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op4 = cbr3.filter((x) -> true);
    final ContinuousStream<String> union = op4.union(cbr2);
    final MISTStream<String> sink1 = op2.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op3.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink3 = union.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final LogicalDagOptimizer logicalDagOptimizer = new LogicalDagOptimizer(dag);
    final DAG<MISTStream, MISTEdge> optimizedDAG = logicalDagOptimizer.getOptimizedDAG();
    // Check src1 -> op1
    final Map<MISTStream, MISTEdge> e1 = optimizedDAG.getEdges(src1);
    final Map<MISTStream, MISTEdge> result1 = new HashMap<>();
    result1.put(op1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check op1 -> cbrOp
    final Map<MISTStream, MISTEdge> e2 = optimizedDAG.getEdges(op1);
    Assert.assertEquals(1, e2.size());
    // a new vertex cbrOp would be created during the optimization
    final MISTStream cbrOp = e2.entrySet().iterator().next().getKey();
    Assert.assertTrue(cbrOp instanceof ContinuousStreamImpl);
    // (idx1)-> op2
    // (idx1)-> op3
    // Check cbrOp (idx2)--------> union
    // (idx3)-> op4
    final Map<MISTStream, MISTEdge> e3 = optimizedDAG.getEdges(cbrOp);
    final Map<MISTStream, MISTEdge> result3 = new HashMap<>();
    result3.put(op2, new MISTEdge(Direction.LEFT, 1));
    result3.put(op3, new MISTEdge(Direction.LEFT, 1));
    result3.put(union, new MISTEdge(Direction.RIGHT, 2));
    result3.put(op4, new MISTEdge(Direction.LEFT, 3));
    Assert.assertEquals(e3, result3);
    // Check op4 -> union
    final Map<MISTStream, MISTEdge> e4 = optimizedDAG.getEdges(op4);
    final Map<MISTStream, MISTEdge> result4 = new HashMap<>();
    result4.put(union, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check op2 -> sink1
    final Map<MISTStream, MISTEdge> e5 = optimizedDAG.getEdges(op2);
    final Map<MISTStream, MISTEdge> result5 = new HashMap<>();
    result5.put(sink1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
    // Check op3 -> sink2
    final Map<MISTStream, MISTEdge> e6 = optimizedDAG.getEdges(op3);
    final Map<MISTStream, MISTEdge> result6 = new HashMap<>();
    result6.put(sink2, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check union -> sink3
    final Map<MISTStream, MISTEdge> e7 = optimizedDAG.getEdges(union);
    final Map<MISTStream, MISTEdge> result7 = new HashMap<>();
    result7.put(sink3, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e7, result7);
}

Example 2 with MISTStream

use of edu.snu.mist.client.datastreams.MISTStream in project mist by snuspl.

the class OperatorChainDagGeneratorTest method testMergingQueryPartitioning.

/**
 * Test merge chaining.
 * logical dag:
 * src1 -> op11 -> op12 ->
 * src2 ---------> op21 -> op13 -> op14 -> sink1
 * src3 -----------------> op31 ->
 *
 * should be converted to the expected operator chain dag:
 * src1 -> [op11 -> op12] ->
 * src2 ---------> [op21] -> [op13] -> [op14] -> sink1
 * src3 -------------------> [op31] ->
 */
@Test
public void testMergingQueryPartitioning() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> src2 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> src3 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op11 = src1.filter((x) -> true);
    final ContinuousStream<String> op12 = op11.filter((x) -> true);
    final ContinuousStream<String> op21 = src2.filter((x) -> true);
    final ContinuousStream<String> op13 = op12.union(op21);
    final ContinuousStream<String> op31 = src3.filter((x) -> true);
    final ContinuousStream<String> op14 = op13.union(op31);
    final MISTStream<String> sink1 = op14.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final OperatorChainDagGenerator chainDagGenerator = new OperatorChainDagGenerator(dag);
    final DAG<List<MISTStream>, MISTEdge> operatorChainDag = chainDagGenerator.generateOperatorChainDAG();
    // Expected outputs
    final List<MISTStream> src1List = Arrays.asList(src1);
    final List<MISTStream> src2List = Arrays.asList(src2);
    final List<MISTStream> src3List = Arrays.asList(src3);
    final List<MISTStream> op11op12 = Arrays.asList(op11, op12);
    final List<MISTStream> op21List = Arrays.asList(op21);
    final List<MISTStream> op13List = Arrays.asList(op13);
    final List<MISTStream> op31List = Arrays.asList(op31);
    final List<MISTStream> op14List = Arrays.asList(op14);
    final List<MISTStream> sink1List = Arrays.asList(sink1);
    // Check src1 -> [op11->op12] edge
    final Map<List<MISTStream>, MISTEdge> e1 = operatorChainDag.getEdges(src1List);
    final Map<List<MISTStream>, MISTEdge> result1 = new HashMap<>();
    result1.put(op11op12, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check src2 -> [op21] edge
    final Map<List<MISTStream>, MISTEdge> e2 = operatorChainDag.getEdges(src2List);
    final Map<List<MISTStream>, MISTEdge> result2 = new HashMap<>();
    result2.put(op21List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e2, result2);
    // Check src3 -> [op31] edge
    final Map<List<MISTStream>, MISTEdge> e3 = operatorChainDag.getEdges(src3List);
    final Map<List<MISTStream>, MISTEdge> result3 = new HashMap<>();
    result3.put(op31List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e3, result3);
    // Check [op11->op12] -> [op13] edge
    final Map<List<MISTStream>, MISTEdge> e4 = operatorChainDag.getEdges(op11op12);
    final Map<List<MISTStream>, MISTEdge> result4 = new HashMap<>();
    result4.put(op13List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check [op21] -> [op13] edge
    final Map<List<MISTStream>, MISTEdge> e5 = operatorChainDag.getEdges(op21List);
    final Map<List<MISTStream>, MISTEdge> result5 = new HashMap<>();
    result5.put(op13List, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(e5, result5);
    // Check [op13] -> [op14] edge
    final Map<List<MISTStream>, MISTEdge> e6 = operatorChainDag.getEdges(op13List);
    final Map<List<MISTStream>, MISTEdge> result6 = new HashMap<>();
    result6.put(op14List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check [op31] -> [op14] edge
    final Map<List<MISTStream>, MISTEdge> e7 = operatorChainDag.getEdges(op31List);
    final Map<List<MISTStream>, MISTEdge> result7 = new HashMap<>();
    result7.put(op14List, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(e7, result7);
    // Check [op14] -> [sink1] edge
    final Map<List<MISTStream>, MISTEdge> e8 = operatorChainDag.getEdges(op14List);
    final Map<List<MISTStream>, MISTEdge> result8 = new HashMap<>();
    result8.put(sink1List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e8, result8);
}

Example 3 with MISTStream

use of edu.snu.mist.client.datastreams.MISTStream in project mist by snuspl.

the class OperatorChainDagGeneratorTest method testBranchTest.

/**
 * Test branch chaining.
 * logical dag:
 *                      -> op14 -> sink2
 * src1 -> op11 -> op12 -> op13 -> sink1
 *                      -> op15 -> sink3
 * should be converted to the expected chained dag:
 *                        -> [op14] -> sink2
 * src1 -> [op11 -> op12] -> [op13] -> sink1
 *                        -> [op15] -> sink3
 */
@Test
public void testBranchTest() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op11 = src1.filter((x) -> true);
    final ContinuousStream<String> op12 = op11.filter((x) -> true);
    final ContinuousStream<String> op13 = op12.filter((x) -> true);
    final ContinuousStream<String> op14 = op12.filter((x) -> true);
    final ContinuousStream<String> op15 = op12.filter((x) -> true);
    final MISTStream<String> sink1 = op13.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op14.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink3 = op15.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final OperatorChainDagGenerator chainDagGenerator = new OperatorChainDagGenerator(dag);
    final DAG<List<MISTStream>, MISTEdge> chainPlan = chainDagGenerator.generateOperatorChainDAG();
    // Expected outputs
    final List<MISTStream> src1List = Arrays.asList(src1);
    final List<MISTStream> op11op12 = Arrays.asList(op11, op12);
    final List<MISTStream> op13List = Arrays.asList(op13);
    final List<MISTStream> op14List = Arrays.asList(op14);
    final List<MISTStream> op15List = Arrays.asList(op15);
    final List<MISTStream> sink1List = Arrays.asList(sink1);
    final List<MISTStream> sink2List = Arrays.asList(sink2);
    final List<MISTStream> sink3List = Arrays.asList(sink3);
    // Check src1 -> [op11->op12] edge
    final Map<List<MISTStream>, MISTEdge> e1 = chainPlan.getEdges(src1List);
    final Map<List<MISTStream>, MISTEdge> result1 = new HashMap<>();
    result1.put(op11op12, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check [op11->op12] -> [op13] edges
    // -> [op14]
    // -> [op15]
    final Map<List<MISTStream>, MISTEdge> e2 = chainPlan.getEdges(op11op12);
    final Map<List<MISTStream>, MISTEdge> result2 = new HashMap<>();
    result2.put(op13List, new MISTEdge(Direction.LEFT));
    result2.put(op14List, new MISTEdge(Direction.LEFT));
    result2.put(op15List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e2, result2);
    // Check [op14] -> [sink2] edge
    final Map<List<MISTStream>, MISTEdge> e3 = chainPlan.getEdges(op14List);
    final Map<List<MISTStream>, MISTEdge> result3 = new HashMap<>();
    result3.put(sink2List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e3, result3);
    // Check [op13] -> [sink1] edge
    final Map<List<MISTStream>, MISTEdge> e4 = chainPlan.getEdges(op13List);
    final Map<List<MISTStream>, MISTEdge> result4 = new HashMap<>();
    result4.put(sink1List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check [op15] -> [sink3] edge
    final Map<List<MISTStream>, MISTEdge> e5 = chainPlan.getEdges(op15List);
    final Map<List<MISTStream>, MISTEdge> result5 = new HashMap<>();
    result5.put(sink3List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
}

Example 4 with MISTStream

use of edu.snu.mist.client.datastreams.MISTStream in project mist by snuspl.

the class OperatorChainDagGenerator method chainingInDfsOrder.

/**
 * Chain the operators and sinks recursively (DFS order) according to the mechanism.
 * @param operatorChain current chain
 * @param currVertex  current vertex
 * @param chainDag operator chain dag
 * @param vertexChainMap vertex and chain mapping
 */
private void chainingInDfsOrder(final OperatorChain operatorChain, final MISTStream currVertex, final DAG<OperatorChain, MISTEdge> chainDag, final Map<MISTStream, OperatorChain> vertexChainMap) {
    if (vertexChainMap.containsKey(currVertex)) {
        return;
    }
    vertexChainMap.put(currVertex, operatorChain);
    operatorChain.chain.add(currVertex);
    final Map<MISTStream, MISTEdge> edges = optimizedDag.getEdges(currVertex);
    for (final Map.Entry<MISTStream, MISTEdge> entry : edges.entrySet()) {
        final MISTStream nextVertex = entry.getKey();
        final MISTEdge edge = entry.getValue();
        if (optimizedDag.getInDegree(nextVertex) > 1 || edges.size() > 1) {
            // The current vertex is 2) branching (have multiple next ops)
            // or the next vertex is 3) merging operator (have multiple incoming edges)
            // so try to create a new OperatorChain for the next operator.
            final OperatorChain nextChain = getOperatorChain(nextVertex, vertexChainMap, chainDag);
            chainDag.addEdge(operatorChain, nextChain, edge);
            chainingInDfsOrder(nextChain, nextVertex, chainDag, vertexChainMap);
        } else if (optimizedDag.getEdges(nextVertex).size() == 0) {
            // The next vertex is Sink. End of the chaining
            final OperatorChain nextChain = getOperatorChain(nextVertex, vertexChainMap, chainDag);
            chainDag.addEdge(operatorChain, nextChain, edge);
            chainingInDfsOrder(nextChain, nextVertex, chainDag, vertexChainMap);
        } else {
            // 1) The next vertex is sequentially following the current vertex
            // so add the next operator to the current OperatorChain
            chainingInDfsOrder(operatorChain, nextVertex, chainDag, vertexChainMap);
        }
    }
}

Example 5 with MISTStream

use of edu.snu.mist.client.datastreams.MISTStream in project mist by snuspl.

the class OperatorChainDagGeneratorTest method testComplexQueryPartitioning.

/**
 * Test complex chaining (branch and merge exist).
 * The logical DAG:
 * src1 -> op11 -> op12 -> union -> op14 -> op15 -> sink1
 * src2 -> op21 -> op22 ->       -> op23 ->      -> sink2.
 *
 * should be converted to the expected operator chain DAG:
 * src1 -> [op11 -> op12] -> [op13] -> [op14 -> op15] -> sink1
 * src2 -> [op21 -> op22] ->        -> [op23] -> sink2.
 */
// @Test
public void testComplexQueryPartitioning() throws InjectionException {
    // Build a operator chain dag
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> src2 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op11 = src1.filter((x) -> true);
    final ContinuousStream<String> op21 = src2.filter((x) -> true);
    final ContinuousStream<String> op12 = op11.filter((x) -> true);
    final ContinuousStream<String> op22 = op21.filter((x) -> true);
    final ContinuousStream<String> union = op12.union(op22);
    final ContinuousStream<String> op14 = union.filter((x) -> true);
    final ContinuousStream<String> op23 = union.filter((x) -> true);
    final ContinuousStream<String> op15 = op14.filter((x) -> true);
    final MISTStream<String> sink1 = op15.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op23.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final OperatorChainDagGenerator chainDagGenerator = new OperatorChainDagGenerator(dag);
    final DAG<List<MISTStream>, MISTEdge> operatorChainDag = chainDagGenerator.generateOperatorChainDAG();
    final List<MISTStream> src1List = Arrays.asList(src1);
    final List<MISTStream> src2List = Arrays.asList(src2);
    final List<MISTStream> op11op12 = Arrays.asList(op11, op12);
    final List<MISTStream> op21op22 = Arrays.asList(op21, op22);
    final List<MISTStream> unionList = Arrays.asList(union);
    final List<MISTStream> op14op15 = Arrays.asList(op14, op15);
    final List<MISTStream> op23List = Arrays.asList(op23);
    final List<MISTStream> sink1List = Arrays.asList(sink1);
    final List<MISTStream> sink2List = Arrays.asList(sink2);
    // Check src1 -> [op11->op12] edge
    final Map<List<MISTStream>, MISTEdge> e1 = operatorChainDag.getEdges(src1List);
    final Map<List<MISTStream>, MISTEdge> result1 = new HashMap<>();
    result1.put(op11op12, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check src2 -> [op21->op22] edge
    final Map<List<MISTStream>, MISTEdge> e2 = operatorChainDag.getEdges(src2List);
    final Map<List<MISTStream>, MISTEdge> result2 = new HashMap<>();
    result2.put(op21op22, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e2, result2);
    // Check [op11->op12] -> [union] edge
    final Map<List<MISTStream>, MISTEdge> e3 = operatorChainDag.getEdges(op11op12);
    final Map<List<MISTStream>, MISTEdge> result3 = new HashMap<>();
    result3.put(unionList, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e3, result3);
    // Check [op21->op22] -> [union] edge
    final Map<List<MISTStream>, MISTEdge> e4 = operatorChainDag.getEdges(op21op22);
    final Map<List<MISTStream>, MISTEdge> result4 = new HashMap<>();
    result4.put(unionList, new MISTEdge(Direction.RIGHT));
    Assert.assertEquals(e4, result4);
    // Check [union] -> [op14->op15] edge
    // Check [union] -> [op23] edge
    final Map<List<MISTStream>, MISTEdge> e5 = operatorChainDag.getEdges(unionList);
    final Map<List<MISTStream>, MISTEdge> result5 = new HashMap<>();
    result5.put(op14op15, new MISTEdge(Direction.LEFT));
    result5.put(op23List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
    // Check [op14->op15] -> sink1 edge
    final Map<List<MISTStream>, MISTEdge> e6 = operatorChainDag.getEdges(op14op15);
    final Map<List<MISTStream>, MISTEdge> result6 = new HashMap<>();
    result6.put(sink1List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check [op23] -> sink2 edge
    final Map<List<MISTStream>, MISTEdge> e7 = operatorChainDag.getEdges(op23List);
    final Map<List<MISTStream>, MISTEdge> result7 = new HashMap<>();
    result7.put(sink2List, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e7, result7);
}