Examples with DAG edu.snu.mist.common.graph.DAG used on opensource projects

Example 6 with DAG

use of edu.snu.mist.common.graph.DAG in project mist by snuspl.

the class ImmediateQueryMergingStarterTest method mergingSameSourceButDifferentOperatorQueriesOneGroupTest.

/**
 * Case 4: Merging two dags that have same source but different operator chain.
 * @throws InjectionException
 */
@Test
public void mergingSameSourceButDifferentOperatorQueriesOneGroupTest() throws InjectionException, IOException, ClassNotFoundException {
    // Create a query 1:
    // src1 -> oc1 -> sink1
    final List<String> result1 = new LinkedList<>();
    final Map<String, String> sourceConf = idAndConfGenerator.generateConf();
    final Map<String, String> ocConf1 = idAndConfGenerator.generateConf();
    final Map<String, String> sinkConf1 = idAndConfGenerator.generateConf();
    final List<String> paths1 = mock(List.class);
    final TestSource src1 = generateSource(sourceConf);
    final PhysicalOperator physicalOp1 = generateFilterOperator(ocConf1, (s) -> true);
    final PhysicalSink<String> sink1 = generateSink(sinkConf1, result1);
    // Config vertices
    final ConfigVertex srcVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, ocConf1);
    final ConfigVertex sinkVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf1);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple1 = generateSimpleDag(src1, physicalOp1, sink1, srcVertex1, ocVertex1, sinkVertex1);
    // Create a query 2:
    // src2 -> oc2 -> sink2
    // The configuration of src2 and operatorChain2 is same as that of src1 and operatorChain2.
    final List<String> result2 = new LinkedList<>();
    final Map<String, String> ocConf2 = idAndConfGenerator.generateConf();
    final Map<String, String> sinkConf2 = idAndConfGenerator.generateConf();
    final TestSource src2 = generateSource(sourceConf);
    final PhysicalOperator physicalOp2 = generateFilterOperator(ocConf2, (s) -> true);
    final PhysicalSink<String> sink2 = generateSink(sinkConf2, result2);
    final List<String> paths2 = mock(List.class);
    // Config vertices
    final ConfigVertex srcVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, ocConf2);
    final ConfigVertex sinkVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf2);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple2 = generateSimpleDag(src2, physicalOp2, sink2, srcVertex2, ocVertex2, sinkVertex2);
    // Execute two queries
    // The query 1 and 2 should be merged and the following dag should be created:
    // src1 -> oc1 -> sink1
    // -> oc2 -> sink2
    final String query1Id = "q1";
    final String query2Id = "q2";
    final Query query1 = mock(Query.class);
    final Query query2 = mock(Query.class);
    queryStarter.start(query1Id, query1, dagTuple1.getKey(), paths1);
    queryStarter.start(query2Id, query2, dagTuple2.getKey(), paths2);
    // Generate events for the merged query and check if the dag is executed correctly
    final String data1 = "Hello";
    src1.send(data1);
    Assert.assertEquals(1, src1.getSourceOutputEmitter().numberOfEvents());
    Assert.assertEquals(null, src2.getSourceOutputEmitter());
    Assert.assertEquals(1, src1.getSourceOutputEmitter().processAllEvent());
    Assert.assertEquals(Arrays.asList(data1), result1);
    Assert.assertEquals(Arrays.asList(data1), result2);
    final String data2 = "World";
    src1.send(data2);
    Assert.assertEquals(1, src1.getSourceOutputEmitter().numberOfEvents());
    Assert.assertEquals(1, src1.getSourceOutputEmitter().processAllEvent());
    Assert.assertEquals(Arrays.asList(data1, data2), result1);
    Assert.assertEquals(Arrays.asList(data1, data2), result2);
    // Src2 and 1 are the same, so the output emitter of src2 should be null
    try {
        src2.send(data1);
        Assert.fail("OutputEmitter should be null");
    } catch (final NullPointerException e) {
    // do nothing
    }
    // Check execution dags
    final Collection<ExecutionDag> expectedDags = new HashSet<>();
    final DAG<ExecutionVertex, MISTEdge> mergedDag = dagTuple1.getValue().getDag();
    mergedDag.addVertex(physicalOp2);
    mergedDag.addVertex(sink2);
    mergedDag.addEdge(src1, physicalOp2, new MISTEdge(Direction.LEFT));
    mergedDag.addEdge(physicalOp2, sink2, new MISTEdge(Direction.LEFT));
    expectedDags.add(srcAndDagMap.get(sourceConf));
    Assert.assertEquals(expectedDags, executionDags.values());
    // Check queryIdConfigDagMap
    Assert.assertEquals(dagTuple1.getKey(), queryIdConfigDagMap.get(query1Id));
    Assert.assertEquals(dagTuple2.getKey(), queryIdConfigDagMap.get(query2Id));
    // Check srcAndDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, srcAndDagMap.get(sourceConf).getDag()));
    // Check executionVertexDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(src1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(physicalOp1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink1).getDag()));
    // They are merged, so src2, oc2 and sink2 should be included in dag1
    Assert.assertEquals(null, executionVertexDagMap.get(src2));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(physicalOp2).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink2).getDag()));
    // Check configExecutionVertexMap
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex1));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex1));
    Assert.assertEquals(sink1, configExecutionVertexMap.get(sinkVertex1));
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex2));
    Assert.assertEquals(physicalOp2, configExecutionVertexMap.get(ocVertex2));
    Assert.assertEquals(sink2, configExecutionVertexMap.get(sinkVertex2));
    // Check reference count
    Assert.assertEquals(2, (int) executionVertexCountMap.get(src1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(physicalOp1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink1));
    Assert.assertEquals(null, executionVertexCountMap.get(src2));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(physicalOp2));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink2));
}

Example 7 with DAG

use of edu.snu.mist.common.graph.DAG in project mist by snuspl.

the class MergeAwareQueryRemoverTest method removeQueryFromMergedExecutionQueriesTest.

/**
 * Case 3: Remove a query where two queries are merged.
 * Query1: src1 -> oc1 -> sink1
 * Query2: src2 -> oc2 -> sink2
 * Merged: src1 -> oc1 -> sink1
 *                     -> sink2
 * After removing query2:
 *         src1 -> oc1 -> sink1
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void removeQueryFromMergedExecutionQueriesTest() throws InjectionException, IOException, ClassNotFoundException {
    // Create a query 1:
    // src1 -> oc1 -> sink1
    final List<String> result1 = new LinkedList<>();
    final Map<String, String> sourceConf = idAndConfGenerator.generateConf();
    final Map<String, String> operatorConf = idAndConfGenerator.generateConf();
    final TestSource src1 = generateSource(sourceConf);
    final Map<String, String> sinkConf1 = idAndConfGenerator.generateConf();
    final PhysicalOperator physicalOp1 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink1 = generateSink(sinkConf1, result1);
    // Config vertices
    final ConfigVertex srcVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, operatorConf);
    final ConfigVertex sinkVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf1);
    final List<String> paths1 = mock(List.class);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple1 = generateSimpleDag(src1, physicalOp1, sink1, srcVertex1, ocVertex1, sinkVertex1);
    // Create a query 2:
    // src2 -> oc2 -> sink2
    // The configuration of src2 and operatorChain2 is same as that of src1 and operatorChain2.
    final List<String> result2 = new LinkedList<>();
    final Map<String, String> sinkConf2 = idAndConfGenerator.generateConf();
    final TestSource src2 = generateSource(sourceConf);
    final PhysicalOperator physicalOp2 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink2 = generateSink(sinkConf2, result2);
    final List<String> paths2 = mock(List.class);
    // Config vertices
    final ConfigVertex srcVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, operatorConf);
    final ConfigVertex sinkVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf2);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple2 = generateSimpleDag(src2, physicalOp2, sink2, srcVertex2, ocVertex2, sinkVertex2);
    final String query1Id = "q1";
    final String query2Id = "q2";
    // Merged dag
    final ExecutionDag mergedExecutionDag = dagTuple1.getValue();
    final DAG<ExecutionVertex, MISTEdge> dag = mergedExecutionDag.getDag();
    dag.addVertex(sink2);
    dag.addEdge(physicalOp1, sink2, new MISTEdge(Direction.LEFT));
    // Add execution dag to srcAndDagMap
    srcAndDagMap.put(sourceConf, mergedExecutionDag);
    // Add execution dag to queryIdConfigDagMap
    queryIdConfigDagMap.put(query1Id, dagTuple1.getKey());
    queryIdConfigDagMap.put(query2Id, dagTuple2.getKey());
    // ConfigExecutionVertexMap
    configExecutionVertexMap.put(srcVertex1, src1);
    configExecutionVertexMap.put(ocVertex1, physicalOp1);
    configExecutionVertexMap.put(sinkVertex1, sink1);
    configExecutionVertexMap.put(srcVertex2, src1);
    configExecutionVertexMap.put(ocVertex2, physicalOp1);
    configExecutionVertexMap.put(sinkVertex2, sink2);
    // ExecutionDags
    executionDags.add(mergedExecutionDag);
    // ExecutionVertexCountMap
    executionVertexCountMap.put(src1, 2);
    executionVertexCountMap.put(physicalOp1, 2);
    executionVertexCountMap.put(sink1, 1);
    executionVertexCountMap.put(sink2, 1);
    // ExecutionVertexDagMap
    executionVertexDagMap.put(src1, mergedExecutionDag);
    executionVertexDagMap.put(physicalOp1, mergedExecutionDag);
    executionVertexDagMap.put(sink1, mergedExecutionDag);
    executionVertexDagMap.put(sink2, mergedExecutionDag);
    // Remove query2
    queryRemover.deleteQuery(query2Id);
    // Check srcAndDagMap
    Assert.assertEquals(1, srcAndDagMap.size());
    Assert.assertEquals(dagTuple1.getValue(), srcAndDagMap.get(src1.getConfiguration()));
    // Check queryIdConfigDagMap
    Assert.assertNull(queryIdConfigDagMap.get(query2Id));
    Assert.assertEquals(dagTuple1.getKey(), queryIdConfigDagMap.get(query1Id));
    // Check configExecutionVertexMap
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex1));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex1));
    Assert.assertEquals(sink1, configExecutionVertexMap.get(sinkVertex1));
    Assert.assertNull(configExecutionVertexMap.get(srcVertex2));
    Assert.assertNull(configExecutionVertexMap.get(ocVertex2));
    Assert.assertNull(configExecutionVertexMap.get(sinkVertex2));
    // Check ExecutionDags
    Assert.assertEquals(1, executionDags.values().size());
    // Check ExecutionVertexCountMap
    Assert.assertEquals(1, (int) executionVertexCountMap.get(src1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(physicalOp1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink1));
    Assert.assertNull(executionVertexCountMap.get(src2));
    Assert.assertNull(executionVertexCountMap.get(physicalOp2));
    Assert.assertNull(executionVertexCountMap.get(sink2));
    // Check ExecutionVertexDagMap
    final DAG<ExecutionVertex, MISTEdge> dag1 = dagTuple1.getValue().getDag();
    Assert.assertTrue(GraphUtils.compareTwoDag(dag1, executionVertexDagMap.get(src1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(dag1, executionVertexDagMap.get(physicalOp1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(dag1, executionVertexDagMap.get(sink1).getDag()));
    Assert.assertNull(executionVertexDagMap.get(src2));
    Assert.assertNull(executionVertexDagMap.get(physicalOp2));
    Assert.assertNull(executionVertexDagMap.get(sink2));
    // Check if the execution dag is changed correctly
    final Map<ExecutionVertex, MISTEdge> oc1Edges = mergedExecutionDag.getDag().getEdges(physicalOp1);
    Assert.assertEquals(1, oc1Edges.size());
    Assert.assertEquals(sink1, oc1Edges.keySet().iterator().next());
}