Examples with MISTEdge edu.snu.mist.common.graph.MISTEdge used on opensource projects

Example 36 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class GroupAwareQueryManagerImpl method createQueryWithCheckpoint.

@Override
public QueryControlResult createQueryWithCheckpoint(final AvroDag avroDag, final QueryCheckpoint checkpointedState) {
    final QueryControlResult queryControlResult = new QueryControlResult();
    final String queryId = avroDag.getQueryId();
    queryControlResult.setQueryId(queryId);
    try {
        // Create the submitted query
        // Update app information
        final String appId = avroDag.getAppId();
        if (LOG.isLoggable(Level.FINE)) {
            LOG.log(Level.FINE, "Create Query [aid: {0}, qid: {2}]", new Object[] { appId, queryId });
        }
        if (!applicationMap.containsKey(appId)) {
            createApplication(appId, avroDag.getJarPaths());
        }
        final ApplicationInfo applicationInfo = applicationMap.get(appId);
        if (applicationInfo.getGroups().size() == 0) {
            synchronized (applicationInfo) {
                if (applicationInfo.getGroups().size() == 0) {
                    createGroup(applicationInfo);
                    // Waiting for group information being added
                    while (applicationInfo.getGroups().isEmpty()) {
                        Thread.sleep(100);
                    }
                }
            }
        }
        final DAG<ConfigVertex, MISTEdge> configDag;
        if (checkpointedState == null) {
            configDag = configDagGenerator.generate(avroDag);
        } else {
            configDag = configDagGenerator.generateWithCheckpointedStates(avroDag, checkpointedState);
        }
        final Query query = createAndStartQuery(queryId, applicationInfo, configDag);
        // Waiting for the query is assigned to a group
        while (query.getGroup() == null) {
            Thread.sleep(100);
        }
        // Store the query to the disk.
        checkpointManager.storeQuery(query.getGroup(), avroDag);
        queryControlResult.setIsSuccess(true);
        queryControlResult.setMsg(ResultMessage.submitSuccess(queryId));
        return queryControlResult;
    } catch (final Exception e) {
        e.printStackTrace();
        // [MIST-345] We need to release all of the information that is required for the query when it fails.
        LOG.log(Level.SEVERE, "An exception occurred while starting {0} query: {1}", new Object[] { queryId, e.toString() });
        queryControlResult.setIsSuccess(false);
        queryControlResult.setMsg(e.getMessage());
        return queryControlResult;
    }
}

Example 37 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class QueryManagerTest method testSubmitComplexQueryHelper.

@SuppressWarnings("unchecked")
private void testSubmitComplexQueryHelper(final Configuration conf) throws Exception {
    final String queryId = "testQuery";
    final List<String> inputs = Arrays.asList("mist is a cloud of tiny water droplets suspended in the atmosphere", "a mist rose out of the river", "the peaks were shrouded in mist");
    // Expected results
    final List<Map<String, Integer>> intermediateResult = getIntermediateResult(inputs);
    final List<String> expectedSink1Output = intermediateResult.stream().map(input -> input.toString()).collect(Collectors.toList());
    final List<Integer> expectedSink2Output = intermediateResult.stream().map(totalCountMapFunc).collect(Collectors.toList());
    // Number of expected outputs
    final CountDownLatch countDownAllOutputs = new CountDownLatch(intermediateResult.size() * 2);
    // Create the execution DAG of the query
    final ExecutionDag executionDag = new ExecutionDag(new AdjacentListDAG<>());
    // Create source
    final TestDataGenerator dataGenerator = new TestDataGenerator(inputs);
    final EventGenerator eventGenerator = new PunctuatedEventGenerator(null, input -> false, null, 0, null, null);
    final PhysicalSource src = new PhysicalSourceImpl("testSource", new HashMap<>(), dataGenerator, eventGenerator);
    final Injector injector = Tang.Factory.getTang().newInjector(conf);
    // Create sinks
    final List<String> sink1Result = new LinkedList<>();
    final List<Integer> sink2Result = new LinkedList<>();
    final PhysicalSink sink1 = new PhysicalSinkImpl<>("sink1", null, new TestWithCountDownSink<String>(sink1Result, countDownAllOutputs));
    final PhysicalSink sink2 = new PhysicalSinkImpl<>("sink2", null, new TestWithCountDownSink<Integer>(sink2Result, countDownAllOutputs));
    // Fake operator chain dag of QueryManager
    final AvroDag fakeAvroDag = new AvroDag();
    fakeAvroDag.setQueryId(queryId);
    // fakeAvroDag.setSuperGroupId("testGroup");
    // Construct execution dag
    constructExecutionDag(executionDag, src, sink1, sink2);
    // Create mock DagGenerator. It returns the above  execution dag
    final ConfigDagGenerator configDagGenerator = mock(ConfigDagGenerator.class);
    final DAG<ConfigVertex, MISTEdge> configDag = mock(DAG.class);
    when(configDagGenerator.generate(fakeAvroDag)).thenReturn(configDag);
    final DagGenerator dagGenerator = mock(DagGenerator.class);
    // when(dagGenerator.generate(configDag, tuple.getValue().getJarFilePaths())).thenReturn(executionDag);
    // Build QueryManager
    final QueryManager queryManager = queryManagerBuild(fakeAvroDag, configDagGenerator, dagGenerator, injector);
    queryManager.create(fakeAvroDag);
    // Wait until all of the outputs are generated
    countDownAllOutputs.await();
    // Check the outputs
    Assert.assertEquals(expectedSink1Output, sink1Result);
    Assert.assertEquals(expectedSink2Output, sink2Result);
    src.close();
    queryManager.close();
    // Delete plan directory and plans
    deletePlans(injector);
}

Example 38 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class MergeAwareQueryRemoverTest method removeQueryFromMergedExecutionQueriesTest.

/**
 * Case 3: Remove a query where two queries are merged.
 * Query1: src1 -> oc1 -> sink1
 * Query2: src2 -> oc2 -> sink2
 * Merged: src1 -> oc1 -> sink1
 *                     -> sink2
 * After removing query2:
 *         src1 -> oc1 -> sink1
 * @throws org.apache.reef.tang.exceptions.InjectionException
 */
@Test
public void removeQueryFromMergedExecutionQueriesTest() throws InjectionException, IOException, ClassNotFoundException {
    // Create a query 1:
    // src1 -> oc1 -> sink1
    final List<String> result1 = new LinkedList<>();
    final Map<String, String> sourceConf = idAndConfGenerator.generateConf();
    final Map<String, String> operatorConf = idAndConfGenerator.generateConf();
    final TestSource src1 = generateSource(sourceConf);
    final Map<String, String> sinkConf1 = idAndConfGenerator.generateConf();
    final PhysicalOperator physicalOp1 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink1 = generateSink(sinkConf1, result1);
    // Config vertices
    final ConfigVertex srcVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, operatorConf);
    final ConfigVertex sinkVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf1);
    final List<String> paths1 = mock(List.class);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple1 = generateSimpleDag(src1, physicalOp1, sink1, srcVertex1, ocVertex1, sinkVertex1);
    // Create a query 2:
    // src2 -> oc2 -> sink2
    // The configuration of src2 and operatorChain2 is same as that of src1 and operatorChain2.
    final List<String> result2 = new LinkedList<>();
    final Map<String, String> sinkConf2 = idAndConfGenerator.generateConf();
    final TestSource src2 = generateSource(sourceConf);
    final PhysicalOperator physicalOp2 = generateFilterOperator(operatorConf, (s) -> true);
    final PhysicalSink<String> sink2 = generateSink(sinkConf2, result2);
    final List<String> paths2 = mock(List.class);
    // Config vertices
    final ConfigVertex srcVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, operatorConf);
    final ConfigVertex sinkVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf2);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple2 = generateSimpleDag(src2, physicalOp2, sink2, srcVertex2, ocVertex2, sinkVertex2);
    final String query1Id = "q1";
    final String query2Id = "q2";
    // Merged dag
    final ExecutionDag mergedExecutionDag = dagTuple1.getValue();
    final DAG<ExecutionVertex, MISTEdge> dag = mergedExecutionDag.getDag();
    dag.addVertex(sink2);
    dag.addEdge(physicalOp1, sink2, new MISTEdge(Direction.LEFT));
    // Add execution dag to srcAndDagMap
    srcAndDagMap.put(sourceConf, mergedExecutionDag);
    // Add execution dag to queryIdConfigDagMap
    queryIdConfigDagMap.put(query1Id, dagTuple1.getKey());
    queryIdConfigDagMap.put(query2Id, dagTuple2.getKey());
    // ConfigExecutionVertexMap
    configExecutionVertexMap.put(srcVertex1, src1);
    configExecutionVertexMap.put(ocVertex1, physicalOp1);
    configExecutionVertexMap.put(sinkVertex1, sink1);
    configExecutionVertexMap.put(srcVertex2, src1);
    configExecutionVertexMap.put(ocVertex2, physicalOp1);
    configExecutionVertexMap.put(sinkVertex2, sink2);
    // ExecutionDags
    executionDags.add(mergedExecutionDag);
    // ExecutionVertexCountMap
    executionVertexCountMap.put(src1, 2);
    executionVertexCountMap.put(physicalOp1, 2);
    executionVertexCountMap.put(sink1, 1);
    executionVertexCountMap.put(sink2, 1);
    // ExecutionVertexDagMap
    executionVertexDagMap.put(src1, mergedExecutionDag);
    executionVertexDagMap.put(physicalOp1, mergedExecutionDag);
    executionVertexDagMap.put(sink1, mergedExecutionDag);
    executionVertexDagMap.put(sink2, mergedExecutionDag);
    // Remove query2
    queryRemover.deleteQuery(query2Id);
    // Check srcAndDagMap
    Assert.assertEquals(1, srcAndDagMap.size());
    Assert.assertEquals(dagTuple1.getValue(), srcAndDagMap.get(src1.getConfiguration()));
    // Check queryIdConfigDagMap
    Assert.assertNull(queryIdConfigDagMap.get(query2Id));
    Assert.assertEquals(dagTuple1.getKey(), queryIdConfigDagMap.get(query1Id));
    // Check configExecutionVertexMap
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex1));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex1));
    Assert.assertEquals(sink1, configExecutionVertexMap.get(sinkVertex1));
    Assert.assertNull(configExecutionVertexMap.get(srcVertex2));
    Assert.assertNull(configExecutionVertexMap.get(ocVertex2));
    Assert.assertNull(configExecutionVertexMap.get(sinkVertex2));
    // Check ExecutionDags
    Assert.assertEquals(1, executionDags.values().size());
    // Check ExecutionVertexCountMap
    Assert.assertEquals(1, (int) executionVertexCountMap.get(src1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(physicalOp1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink1));
    Assert.assertNull(executionVertexCountMap.get(src2));
    Assert.assertNull(executionVertexCountMap.get(physicalOp2));
    Assert.assertNull(executionVertexCountMap.get(sink2));
    // Check ExecutionVertexDagMap
    final DAG<ExecutionVertex, MISTEdge> dag1 = dagTuple1.getValue().getDag();
    Assert.assertTrue(GraphUtils.compareTwoDag(dag1, executionVertexDagMap.get(src1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(dag1, executionVertexDagMap.get(physicalOp1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(dag1, executionVertexDagMap.get(sink1).getDag()));
    Assert.assertNull(executionVertexDagMap.get(src2));
    Assert.assertNull(executionVertexDagMap.get(physicalOp2));
    Assert.assertNull(executionVertexDagMap.get(sink2));
    // Check if the execution dag is changed correctly
    final Map<ExecutionVertex, MISTEdge> oc1Edges = mergedExecutionDag.getDag().getEdges(physicalOp1);
    Assert.assertEquals(1, oc1Edges.size());
    Assert.assertEquals(sink1, oc1Edges.keySet().iterator().next());
}

Example 39 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ImmediateQueryMergingStarterTest method generateSimpleDag.

/**
 * Generate a simple query that has the following structure: src -> operator -> sink.
 * @param source source
 * @param physicalOperator operator
 * @param sink sink
 * @return dag
 */
private Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> generateSimpleDag(final TestSource source, final PhysicalOperator physicalOperator, final PhysicalSink<String> sink, final ConfigVertex srcVertex, final ConfigVertex ocVertex, final ConfigVertex sinkVertex) throws IOException, ClassNotFoundException {
    // Create DAG
    final DAG<ConfigVertex, MISTEdge> dag = new AdjacentListConcurrentMapDAG<>();
    dag.addVertex(srcVertex);
    dag.addVertex(ocVertex);
    dag.addVertex(sinkVertex);
    dag.addEdge(srcVertex, ocVertex, new MISTEdge(Direction.LEFT));
    dag.addEdge(ocVertex, sinkVertex, new MISTEdge(Direction.LEFT));
    final DAG<ExecutionVertex, MISTEdge> exDag = new AdjacentListConcurrentMapDAG<>();
    exDag.addVertex(source);
    exDag.addVertex(physicalOperator);
    exDag.addVertex(sink);
    exDag.addEdge(source, physicalOperator, new MISTEdge(Direction.LEFT));
    exDag.addEdge(physicalOperator, sink, new MISTEdge(Direction.LEFT));
    when(executionVertexGenerator.generate(eq(srcVertex), any(URL[].class), any(ClassLoader.class))).thenReturn(source);
    when(executionVertexGenerator.generate(eq(ocVertex), any(URL[].class), any(ClassLoader.class))).thenReturn(physicalOperator);
    when(executionVertexGenerator.generate(eq(sinkVertex), any(URL[].class), any(ClassLoader.class))).thenReturn(sink);
    final ExecutionDag executionDag = new ExecutionDag(exDag);
    return new Tuple<>(dag, executionDag);
}

Example 40 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ImmediateQueryMergingStarterTest method mergingSameSourceButDifferentOperatorQueriesOneGroupTest.

/**
 * Case 4: Merging two dags that have same source but different operator chain.
 * @throws InjectionException
 */
@Test
public void mergingSameSourceButDifferentOperatorQueriesOneGroupTest() throws InjectionException, IOException, ClassNotFoundException {
    // Create a query 1:
    // src1 -> oc1 -> sink1
    final List<String> result1 = new LinkedList<>();
    final Map<String, String> sourceConf = idAndConfGenerator.generateConf();
    final Map<String, String> ocConf1 = idAndConfGenerator.generateConf();
    final Map<String, String> sinkConf1 = idAndConfGenerator.generateConf();
    final List<String> paths1 = mock(List.class);
    final TestSource src1 = generateSource(sourceConf);
    final PhysicalOperator physicalOp1 = generateFilterOperator(ocConf1, (s) -> true);
    final PhysicalSink<String> sink1 = generateSink(sinkConf1, result1);
    // Config vertices
    final ConfigVertex srcVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, ocConf1);
    final ConfigVertex sinkVertex1 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf1);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple1 = generateSimpleDag(src1, physicalOp1, sink1, srcVertex1, ocVertex1, sinkVertex1);
    // Create a query 2:
    // src2 -> oc2 -> sink2
    // The configuration of src2 and operatorChain2 is same as that of src1 and operatorChain2.
    final List<String> result2 = new LinkedList<>();
    final Map<String, String> ocConf2 = idAndConfGenerator.generateConf();
    final Map<String, String> sinkConf2 = idAndConfGenerator.generateConf();
    final TestSource src2 = generateSource(sourceConf);
    final PhysicalOperator physicalOp2 = generateFilterOperator(ocConf2, (s) -> true);
    final PhysicalSink<String> sink2 = generateSink(sinkConf2, result2);
    final List<String> paths2 = mock(List.class);
    // Config vertices
    final ConfigVertex srcVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SOURCE, sourceConf);
    final ConfigVertex ocVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.OPERATOR, ocConf2);
    final ConfigVertex sinkVertex2 = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), ExecutionVertex.Type.SINK, sinkConf2);
    // Create dag
    final Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> dagTuple2 = generateSimpleDag(src2, physicalOp2, sink2, srcVertex2, ocVertex2, sinkVertex2);
    // Execute two queries
    // The query 1 and 2 should be merged and the following dag should be created:
    // src1 -> oc1 -> sink1
    // -> oc2 -> sink2
    final String query1Id = "q1";
    final String query2Id = "q2";
    final Query query1 = mock(Query.class);
    final Query query2 = mock(Query.class);
    queryStarter.start(query1Id, query1, dagTuple1.getKey(), paths1);
    queryStarter.start(query2Id, query2, dagTuple2.getKey(), paths2);
    // Generate events for the merged query and check if the dag is executed correctly
    final String data1 = "Hello";
    src1.send(data1);
    Assert.assertEquals(1, src1.getSourceOutputEmitter().numberOfEvents());
    Assert.assertEquals(null, src2.getSourceOutputEmitter());
    Assert.assertEquals(1, src1.getSourceOutputEmitter().processAllEvent());
    Assert.assertEquals(Arrays.asList(data1), result1);
    Assert.assertEquals(Arrays.asList(data1), result2);
    final String data2 = "World";
    src1.send(data2);
    Assert.assertEquals(1, src1.getSourceOutputEmitter().numberOfEvents());
    Assert.assertEquals(1, src1.getSourceOutputEmitter().processAllEvent());
    Assert.assertEquals(Arrays.asList(data1, data2), result1);
    Assert.assertEquals(Arrays.asList(data1, data2), result2);
    // Src2 and 1 are the same, so the output emitter of src2 should be null
    try {
        src2.send(data1);
        Assert.fail("OutputEmitter should be null");
    } catch (final NullPointerException e) {
    // do nothing
    }
    // Check execution dags
    final Collection<ExecutionDag> expectedDags = new HashSet<>();
    final DAG<ExecutionVertex, MISTEdge> mergedDag = dagTuple1.getValue().getDag();
    mergedDag.addVertex(physicalOp2);
    mergedDag.addVertex(sink2);
    mergedDag.addEdge(src1, physicalOp2, new MISTEdge(Direction.LEFT));
    mergedDag.addEdge(physicalOp2, sink2, new MISTEdge(Direction.LEFT));
    expectedDags.add(srcAndDagMap.get(sourceConf));
    Assert.assertEquals(expectedDags, executionDags.values());
    // Check queryIdConfigDagMap
    Assert.assertEquals(dagTuple1.getKey(), queryIdConfigDagMap.get(query1Id));
    Assert.assertEquals(dagTuple2.getKey(), queryIdConfigDagMap.get(query2Id));
    // Check srcAndDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, srcAndDagMap.get(sourceConf).getDag()));
    // Check executionVertexDagMap
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(src1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(physicalOp1).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink1).getDag()));
    // They are merged, so src2, oc2 and sink2 should be included in dag1
    Assert.assertEquals(null, executionVertexDagMap.get(src2));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(physicalOp2).getDag()));
    Assert.assertTrue(GraphUtils.compareTwoDag(mergedDag, executionVertexDagMap.get(sink2).getDag()));
    // Check configExecutionVertexMap
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex1));
    Assert.assertEquals(physicalOp1, configExecutionVertexMap.get(ocVertex1));
    Assert.assertEquals(sink1, configExecutionVertexMap.get(sinkVertex1));
    Assert.assertEquals(src1, configExecutionVertexMap.get(srcVertex2));
    Assert.assertEquals(physicalOp2, configExecutionVertexMap.get(ocVertex2));
    Assert.assertEquals(sink2, configExecutionVertexMap.get(sinkVertex2));
    // Check reference count
    Assert.assertEquals(2, (int) executionVertexCountMap.get(src1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(physicalOp1));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink1));
    Assert.assertEquals(null, executionVertexCountMap.get(src2));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(physicalOp2));
    Assert.assertEquals(1, (int) executionVertexCountMap.get(sink2));
}