Examples with ContinuousStreamImpl edu.snu.mist.client.datastreams.ContinuousStreamImpl used on opensource projects

Example 1 with ContinuousStreamImpl

use of edu.snu.mist.client.datastreams.ContinuousStreamImpl in project mist by snuspl.

the class LogicalDagOptimizerTest method testConditionalBranch.

/**
 * Test conditional branch.
 * logical dag:
 *                            -> op2 -> sink1
 *             -> cbr1 (idx 0)-> op3 -> sink2
 * src1 -> op1 -> cbr2 (idx 0)-------->
 *             -> cbr3 (idx 0)-> op4 -> union -> sink3
 *
 * should be converted to the expected optimized dag:
 *                      (idx 1)-> op2 -> sink1
 *                      (idx 1)-> op3 -> sink2
 * src1 -> op1 -> cbrOp (idx 2)-------->
 *                      (idx 3)-> op4 -> union -> sink3
 */
@Test
public void testConditionalBranch() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op1 = src1.filter((x) -> true);
    final ContinuousStream<String> cbr1 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr2 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr3 = op1.routeIf((x) -> true);
    final ContinuousStream<String> op2 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op3 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op4 = cbr3.filter((x) -> true);
    final ContinuousStream<String> union = op4.union(cbr2);
    final MISTStream<String> sink1 = op2.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op3.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink3 = union.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final LogicalDagOptimizer logicalDagOptimizer = new LogicalDagOptimizer(dag);
    final DAG<MISTStream, MISTEdge> optimizedDAG = logicalDagOptimizer.getOptimizedDAG();
    // Check src1 -> op1
    final Map<MISTStream, MISTEdge> e1 = optimizedDAG.getEdges(src1);
    final Map<MISTStream, MISTEdge> result1 = new HashMap<>();
    result1.put(op1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check op1 -> cbrOp
    final Map<MISTStream, MISTEdge> e2 = optimizedDAG.getEdges(op1);
    Assert.assertEquals(1, e2.size());
    // a new vertex cbrOp would be created during the optimization
    final MISTStream cbrOp = e2.entrySet().iterator().next().getKey();
    Assert.assertTrue(cbrOp instanceof ContinuousStreamImpl);
    // (idx1)-> op2
    // (idx1)-> op3
    // Check cbrOp (idx2)--------> union
    // (idx3)-> op4
    final Map<MISTStream, MISTEdge> e3 = optimizedDAG.getEdges(cbrOp);
    final Map<MISTStream, MISTEdge> result3 = new HashMap<>();
    result3.put(op2, new MISTEdge(Direction.LEFT, 1));
    result3.put(op3, new MISTEdge(Direction.LEFT, 1));
    result3.put(union, new MISTEdge(Direction.RIGHT, 2));
    result3.put(op4, new MISTEdge(Direction.LEFT, 3));
    Assert.assertEquals(e3, result3);
    // Check op4 -> union
    final Map<MISTStream, MISTEdge> e4 = optimizedDAG.getEdges(op4);
    final Map<MISTStream, MISTEdge> result4 = new HashMap<>();
    result4.put(union, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check op2 -> sink1
    final Map<MISTStream, MISTEdge> e5 = optimizedDAG.getEdges(op2);
    final Map<MISTStream, MISTEdge> result5 = new HashMap<>();
    result5.put(sink1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
    // Check op3 -> sink2
    final Map<MISTStream, MISTEdge> e6 = optimizedDAG.getEdges(op3);
    final Map<MISTStream, MISTEdge> result6 = new HashMap<>();
    result6.put(sink2, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check union -> sink3
    final Map<MISTStream, MISTEdge> e7 = optimizedDAG.getEdges(union);
    final Map<MISTStream, MISTEdge> result7 = new HashMap<>();
    result7.put(sink3, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e7, result7);
}

Example 2 with ContinuousStreamImpl

use of edu.snu.mist.client.datastreams.ContinuousStreamImpl in project mist by snuspl.

the class LogicalDagOptimizer method optimizeSubDag.

/**
 * Obtimize the operators and sinks recursively (DFS order) according to the mechanism.
 * @param currVertex  current vertex
 * @param visited visited vertices
 */
private void optimizeSubDag(final MISTStream currVertex, final Set<MISTStream> visited) {
    if (!visited.contains(currVertex)) {
        visited.add(currVertex);
        final Map<MISTStream, MISTEdge> edges = dag.getEdges(currVertex);
        // checking whether there is any conditionally branching edge diverged from current vertex
        if (!(currVertex instanceof ContinuousStreamImpl) || ((ContinuousStreamImpl) currVertex).getCondBranchCount() == 0) {
            // current vertex is not a continuous stream or this edge is an ordinary (non-branch) edge
            for (final MISTStream nextVertex : edges.keySet()) {
                optimizeSubDag(nextVertex, visited);
            }
        } else {
            // current vertex has some conditionally branching edges
            final Map<Integer, ContinuousStreamImpl> branchStreams = new HashMap<>();
            // gather the branching streams
            for (final MISTStream nextVertex : edges.keySet()) {
                if (nextVertex instanceof ContinuousStreamImpl) {
                    final ContinuousStreamImpl contNextVertex = (ContinuousStreamImpl) nextVertex;
                    if (contNextVertex.getBranchIndex() > 0) {
                        // this edge is a conditionally branching edge
                        branchStreams.put(contNextVertex.getBranchIndex(), contNextVertex);
                    }
                }
                optimizeSubDag(nextVertex, visited);
            }
            // gather condition udfs from each branch stream
            final List<String> udfs = new ArrayList<>(branchStreams.size());
            for (int i = 1; i <= branchStreams.size(); i++) {
                final ContinuousStreamImpl branchStream = branchStreams.get(i);
                final Map<String, String> conf = branchStream.getConfiguration();
                udfs.add(conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
            }
            // create a new conditional branch vertex to unify these branch streams
            final Map<String, String> opConf = new HashMap<>();
            try {
                opConf.put(ConfKeys.ConditionalBranchOperator.UDF_LIST_STRING.name(), SerializeUtils.serializeToString((Serializable) udfs));
            } catch (final IOException e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
            final ContinuousStreamImpl unifiedBranchStream = new ContinuousStreamImpl(dag, opConf);
            dag.addVertex(unifiedBranchStream);
            dag.addEdge(currVertex, unifiedBranchStream, new MISTEdge(Direction.LEFT));
            // merging all the branching vertices from the current vertex into a single conditional branch vertex
            for (final ContinuousStreamImpl branchStream : branchStreams.values()) {
                final List<MISTStream> branchDownStreams = new ArrayList<>();
                for (final Map.Entry<MISTStream, MISTEdge> edgeFromBranch : dag.getEdges(branchStream).entrySet()) {
                    final MISTStream branchDownStream = edgeFromBranch.getKey();
                    branchDownStreams.add(branchDownStream);
                    dag.addEdge(unifiedBranchStream, branchDownStream, new MISTEdge(edgeFromBranch.getValue().getDirection(), branchStream.getBranchIndex()));
                }
                // to prevent the concurrent map modification, remove the edges from downStream separately
                for (final MISTStream branchDownStream : branchDownStreams) {
                    dag.removeEdge(branchStream, branchDownStream);
                }
                dag.removeEdge(currVertex, branchStream);
                dag.removeVertex(branchStream);
            }
        }
    }
}