Examples with MISTEdge edu.snu.mist.common.graph.MISTEdge used on opensource projects

Example 16 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class QueryManagerTest method constructExecutionDag.

/**
 * Construct execution dag.
 * Creates operators and adds source, dag vertices, edges and sinks to dag.
 */
private void constructExecutionDag(final ExecutionDag executionDag, final PhysicalSource src, final PhysicalSink sink1, final PhysicalSink sink2) {
    // Create operators
    // src -> [flatMap -> filter -> toTupleMap -> reduceByKey] -> [toStringMap]   -> sink1
    // -> [totalCountMap] -> sink2
    final PhysicalOperator flatMap = new DefaultPhysicalOperatorImpl("flatMap", null, new FlatMapOperator<>(flatMapFunc));
    final PhysicalOperator filter = new DefaultPhysicalOperatorImpl("filter", null, new FilterOperator<>(filterFunc));
    final PhysicalOperator toTupleMap = new DefaultPhysicalOperatorImpl("toTupleMap", null, new MapOperator<>(toTupleMapFunc));
    final PhysicalOperator reduceByKey = new DefaultPhysicalOperatorImpl("reduceByKey", null, new ReduceByKeyOperator<>(0, reduceByKeyFunc));
    final PhysicalOperator toStringMap = new DefaultPhysicalOperatorImpl("toStringMap", null, new MapOperator<>(toStringMapFunc));
    final PhysicalOperator totalCountMap = new DefaultPhysicalOperatorImpl("totalCountMap", null, new MapOperator<>(totalCountMapFunc));
    // Build the execution dag
    final DAG<ExecutionVertex, MISTEdge> dag = executionDag.getDag();
    // Add Source
    dag.addVertex(src);
    // Add dag vertices and edges
    dag.addVertex(flatMap);
    dag.addVertex(filter);
    dag.addVertex(toTupleMap);
    dag.addVertex(reduceByKey);
    dag.addVertex(toStringMap);
    dag.addVertex(totalCountMap);
    dag.addEdge(src, flatMap, new MISTEdge(Direction.LEFT));
    dag.addEdge(flatMap, filter, new MISTEdge(Direction.LEFT));
    dag.addEdge(filter, toTupleMap, new MISTEdge(Direction.LEFT));
    dag.addEdge(toTupleMap, reduceByKey, new MISTEdge(Direction.LEFT));
    dag.addEdge(reduceByKey, toStringMap, new MISTEdge(Direction.LEFT));
    dag.addEdge(reduceByKey, totalCountMap, new MISTEdge(Direction.LEFT));
    // Add Sink
    dag.addVertex(sink1);
    dag.addEdge(toStringMap, sink1, new MISTEdge(Direction.LEFT));
    dag.addVertex(sink2);
    dag.addEdge(totalCountMap, sink2, new MISTEdge(Direction.LEFT));
}

Example 17 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class MergeAwareQueryRemoverTest method generateSimpleDag.

/**
 * Generate a simple query that has the following structure: src -> operator chain -> sink.
 * @param source source
 * @param physicalOperator operator chain
 * @param sink sink
 * @return dag
 */
private Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> generateSimpleDag(final TestSource source, final PhysicalOperator physicalOperator, final PhysicalSink<String> sink, final ConfigVertex srcVertex, final ConfigVertex ocVertex, final ConfigVertex sinkVertex) throws IOException {
    // Create DAG
    final DAG<ConfigVertex, MISTEdge> dag = new AdjacentListDAG<>();
    dag.addVertex(srcVertex);
    dag.addVertex(ocVertex);
    dag.addVertex(sinkVertex);
    dag.addEdge(srcVertex, ocVertex, new MISTEdge(Direction.LEFT));
    dag.addEdge(ocVertex, sinkVertex, new MISTEdge(Direction.LEFT));
    final DAG<ExecutionVertex, MISTEdge> newDag = new AdjacentListDAG<>();
    newDag.addVertex(source);
    newDag.addVertex(physicalOperator);
    newDag.addVertex(sink);
    newDag.addEdge(source, physicalOperator, new MISTEdge(Direction.LEFT));
    newDag.addEdge(physicalOperator, sink, new MISTEdge(Direction.LEFT));
    final ExecutionDag executionDag = new ExecutionDag(newDag);
    return new Tuple<>(dag, executionDag);
}

Example 18 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ImmediateQueryMergingStarterTest method mergingSameSourceAndSameOperatorQueriesOneGroupTest.

/**
 * Case 3: Merging two dags that have same source and operator chain.
 * @throws InjectionException
 */
@Test
public void mergingSameSourceAndSameOperatorQueriesOneGroupTest() throws InjectionException, IOException, ClassNotFoundException {
    // Create a query 1:
    // src1 -> oc1 -> sink1
    final List<String> result1 = new LinkedList<>();
    final Map<String, String> sourceConf = idAndConfGenerator.generateConf();
    final Map<String, String> operatorConf = idAndConfGenerator.generateConf();
    final TestSource src1 = generateSource(sourceConf);
    final Map<String, String> sinkConf1 = idAndConfGenerator.generateConf();
    final PhysicalOperator physicalOp1 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink1 = generateSink(sinkConf1, result1);
    // Config vertices
    final ConfigVertex srcVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, operatorConf);
    final ConfigVertex sinkVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf1);
    final List<String> paths1 = mock(List.class);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple1 = generateSimpleDag(src1, physicalOp1, sink1, srcVertex1, ocVertex1, sinkVertex1);
    // Create a query 2:
    // src2 -> oc2 -> sink2
    // The configuration of src2 and operatorChain2 is same as that of src1 and operatorChain2.
    final List<String> result2 = new LinkedList<>();
    final Map<String, String> sinkConf2 = idAndConfGenerator.generateConf();
    final TestSource src2 = generateSource(sourceConf);
    final PhysicalOperator physicalOp2 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink2 = generateSink(sinkConf2, result2);
    final List<String> paths2 = mock(List.class);
    // Config vertices
    final ConfigVertex srcVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, operatorConf);
    final ConfigVertex sinkVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf2);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple2 = generateSimpleDag(src2, physicalOp2, sink2, srcVertex2, ocVertex2, sinkVertex2);
    // Execute two queries
    final Query query1 = mock(Query.class);
    final Query query2 = mock(Query.class);
    final String query1Id = "q1";
    final String query2Id = "q2";
    queryStarter.start(query1Id, query1, dagTuple1.getKey(), paths1);
    queryStarter.start(query2Id, query2, dagTuple2.getKey(), paths2);
    // Generate events for the merged query and check if the dag is executed correctly
    final String data = "Hello";
    src1.send(data);
    Assert.assertEquals(1, src1.getSourceOutputEmitter().numberOfEvents());
    // This is not created because the source is the same!
    Assert.assertEquals(null, src2.getSourceOutputEmitter());
    Assert.assertEquals(1, src1.getSourceOutputEmitter().processAllEvent());
    Assert.assertEquals(Arrays.asList(data), result1);
    Assert.assertEquals(Arrays.asList(data), result2);
    // Src2 and 1 are the same, so the output emitter of src2 should be null
    try {
        src2.send(data);
        Assert.fail("OutputEmitter should be null");
    } catch (final NullPointerException e) {
    // do nothing
    }
    // Check queryIdConfigDagMap
    Assert.assertEquals(dagTuple1.getKey(), queryIdConfigDagMap.get(query1Id));
    Assert.assertEquals(dagTuple2.getKey(), queryIdConfigDagMap.get(query2Id));
    // Check execution dags
    final Collection<ExecutionDag> expectedDags = new HashSet<>();
    final DAG<ExecutionVertex, MISTEdge> mergedDag = dagTuple1.getValue().getDag();
    mergedDag.addVertex(sink2);
    mergedDag.addEdge(physicalOp1, sink2, new MISTEdge(Direction.LEFT));
    expectedDags.add(srcAndDagMap.get(sourceConf));
    Assert.assertEquals(expectedDags, executionDags.values());
    // Check srcAndDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, srcAndDagMap.get(sourceConf).getDag()));
    // Check executionVertexDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(src1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(physicalOp1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink1).getDag()));
    // They are merged, so src2, oc2 and sink2 should be included in dag1
    Assert.assertNull(executionVertexDagMap.get(src2));
    Assert.assertNull(executionVertexDagMap.get(physicalOp2));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink2).getDag()));
    // Check configExecutionVertexMap
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex1));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex1));
    Assert.assertEquals(sink1, configExecutionVertexMap.get(sinkVertex1));
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex2));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex2));
    Assert.assertEquals(sink2, configExecutionVertexMap.get(sinkVertex2));
    // Check reference count
    Assert.assertEquals(2, (int) executionVertexCountMap.get(src1));
    Assert.assertEquals(2, (int) executionVertexCountMap.get(physicalOp1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink1));
    Assert.assertNull(executionVertexCountMap.get(src2));
    Assert.assertNull(executionVertexCountMap.get(physicalOp2));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink2));
}

Example 19 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class OperatorChainDagGenerator method chainingInDfsOrder.

/**
 * Chain the operators and sinks recursively (DFS order) according to the mechanism.
 * @param operatorChain current chain
 * @param currVertex  current vertex
 * @param chainDag operator chain dag
 * @param vertexChainMap vertex and chain mapping
 */
private void chainingInDfsOrder(final OperatorChain operatorChain, final MISTStream currVertex, final DAG<OperatorChain, MISTEdge> chainDag, final Map<MISTStream, OperatorChain> vertexChainMap) {
    if (vertexChainMap.containsKey(currVertex)) {
        return;
    }
    vertexChainMap.put(currVertex, operatorChain);
    operatorChain.chain.add(currVertex);
    final Map<MISTStream, MISTEdge> edges = optimizedDag.getEdges(currVertex);
    for (final Map.Entry<MISTStream, MISTEdge> entry : edges.entrySet()) {
        final MISTStream nextVertex = entry.getKey();
        final MISTEdge edge = entry.getValue();
        if (optimizedDag.getInDegree(nextVertex) > 1 || edges.size() > 1) {
            // The current vertex is 2) branching (have multiple next ops)
            // or the next vertex is 3) merging operator (have multiple incoming edges)
            // so try to create a new OperatorChain for the next operator.
            final OperatorChain nextChain = getOperatorChain(nextVertex, vertexChainMap, chainDag);
            chainDag.addEdge(operatorChain, nextChain, edge);
            chainingInDfsOrder(nextChain, nextVertex, chainDag, vertexChainMap);
        } else if (optimizedDag.getEdges(nextVertex).size() == 0) {
            // The next vertex is Sink. End of the chaining
            final OperatorChain nextChain = getOperatorChain(nextVertex, vertexChainMap, chainDag);
            chainDag.addEdge(operatorChain, nextChain, edge);
            chainingInDfsOrder(nextChain, nextVertex, chainDag, vertexChainMap);
        } else {
            // 1) The next vertex is sequentially following the current vertex
            // so add the next operator to the current OperatorChain
            chainingInDfsOrder(operatorChain, nextVertex, chainDag, vertexChainMap);
        }
    }
}

Example 20 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class LogicalDagOptimizerTest method testConditionalBranch.

/**
 * Test conditional branch.
 * logical dag:
 *                            -> op2 -> sink1
 *             -> cbr1 (idx 0)-> op3 -> sink2
 * src1 -> op1 -> cbr2 (idx 0)-------->
 *             -> cbr3 (idx 0)-> op4 -> union -> sink3
 *
 * should be converted to the expected optimized dag:
 *                      (idx 1)-> op2 -> sink1
 *                      (idx 1)-> op3 -> sink2
 * src1 -> op1 -> cbrOp (idx 2)-------->
 *                      (idx 3)-> op4 -> union -> sink3
 */
@Test
public void testConditionalBranch() throws InjectionException {
    final MISTQueryBuilder queryBuilder = new MISTQueryBuilder();
    queryBuilder.setApplicationId(TestParameters.SUPER_GROUP_ID);
    final ContinuousStream<String> src1 = queryBuilder.socketTextStream(TestParameters.LOCAL_TEXT_SOCKET_SOURCE_CONF);
    final ContinuousStream<String> op1 = src1.filter((x) -> true);
    final ContinuousStream<String> cbr1 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr2 = op1.routeIf((x) -> true);
    final ContinuousStream<String> cbr3 = op1.routeIf((x) -> true);
    final ContinuousStream<String> op2 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op3 = cbr1.filter((x) -> true);
    final ContinuousStream<String> op4 = cbr3.filter((x) -> true);
    final ContinuousStream<String> union = op4.union(cbr2);
    final MISTStream<String> sink1 = op2.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink2 = op3.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTStream<String> sink3 = union.textSocketOutput(TestParameters.HOST, TestParameters.SINK_PORT);
    final MISTQuery query = queryBuilder.build();
    final DAG<MISTStream, MISTEdge> dag = query.getDAG();
    final LogicalDagOptimizer logicalDagOptimizer = new LogicalDagOptimizer(dag);
    final DAG<MISTStream, MISTEdge> optimizedDAG = logicalDagOptimizer.getOptimizedDAG();
    // Check src1 -> op1
    final Map<MISTStream, MISTEdge> e1 = optimizedDAG.getEdges(src1);
    final Map<MISTStream, MISTEdge> result1 = new HashMap<>();
    result1.put(op1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e1, result1);
    // Check op1 -> cbrOp
    final Map<MISTStream, MISTEdge> e2 = optimizedDAG.getEdges(op1);
    Assert.assertEquals(1, e2.size());
    // a new vertex cbrOp would be created during the optimization
    final MISTStream cbrOp = e2.entrySet().iterator().next().getKey();
    Assert.assertTrue(cbrOp instanceof ContinuousStreamImpl);
    // (idx1)-> op2
    // (idx1)-> op3
    // Check cbrOp (idx2)--------> union
    // (idx3)-> op4
    final Map<MISTStream, MISTEdge> e3 = optimizedDAG.getEdges(cbrOp);
    final Map<MISTStream, MISTEdge> result3 = new HashMap<>();
    result3.put(op2, new MISTEdge(Direction.LEFT, 1));
    result3.put(op3, new MISTEdge(Direction.LEFT, 1));
    result3.put(union, new MISTEdge(Direction.RIGHT, 2));
    result3.put(op4, new MISTEdge(Direction.LEFT, 3));
    Assert.assertEquals(e3, result3);
    // Check op4 -> union
    final Map<MISTStream, MISTEdge> e4 = optimizedDAG.getEdges(op4);
    final Map<MISTStream, MISTEdge> result4 = new HashMap<>();
    result4.put(union, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e4, result4);
    // Check op2 -> sink1
    final Map<MISTStream, MISTEdge> e5 = optimizedDAG.getEdges(op2);
    final Map<MISTStream, MISTEdge> result5 = new HashMap<>();
    result5.put(sink1, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e5, result5);
    // Check op3 -> sink2
    final Map<MISTStream, MISTEdge> e6 = optimizedDAG.getEdges(op3);
    final Map<MISTStream, MISTEdge> result6 = new HashMap<>();
    result6.put(sink2, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e6, result6);
    // Check union -> sink3
    final Map<MISTStream, MISTEdge> e7 = optimizedDAG.getEdges(union);
    final Map<MISTStream, MISTEdge> result7 = new HashMap<>();
    result7.put(sink3, new MISTEdge(Direction.LEFT));
    Assert.assertEquals(e7, result7);
}