Examples with AdjacentListDAG edu.snu.mist.common.graph.AdjacentListDAG used on opensource projects

Example 1 with AdjacentListDAG

use of edu.snu.mist.common.graph.AdjacentListDAG in project mist by snuspl.

the class DefaultConfigDagGeneratorImpl method generateWithCheckpointedStates.

@Override
public DAG<ConfigVertex, MISTEdge> generateWithCheckpointedStates(final AvroDag avroDag, final QueryCheckpoint checkpointedState) {
    final List<ConfigVertex> deserializedVertices = new ArrayList<>(avroDag.getAvroVertices().size());
    final DAG<ConfigVertex, MISTEdge> configDag = new AdjacentListDAG<>();
    final List<AvroVertex> avroVertices = avroDag.getAvroVertices();
    if (checkpointedState == null) {
        // Fetch configurations from avro vertex dag
        for (final AvroVertex avroVertex : avroVertices) {
            final ExecutionVertex.Type type = getVertexType(avroVertex);
            // Create a config vertex
            final ConfigVertex configVertex = new ConfigVertex(avroVertex.getVertexId(), type, avroVertex.getConfiguration());
            deserializedVertices.add(configVertex);
            configDag.addVertex(configVertex);
        }
    } else {
        // There are checkpointed states -> generate vertices with internal states.
        final int numVertices = avroVertices.size();
        final Map<String, StateWithTimestamp> queryState = checkpointedState.getQueryState();
        // This can be guaranteed because Java List semantic always guarantees the order among elements.
        for (final AvroVertex avroVertex : avroVertices) {
            final StateWithTimestamp vertexStateWithTimestamp = queryState.get(avroVertex.getVertexId());
            final ExecutionVertex.Type type = getVertexType(avroVertex);
            // Create a config vertex with checkpointed states.
            final ConfigVertex configVertex = new ConfigVertex(Long.toString(configVertexId.getAndIncrement()), type, avroVertex.getConfiguration(), vertexStateWithTimestamp.getVertexState(), vertexStateWithTimestamp.getCheckpointTimestamp());
            deserializedVertices.add(configVertex);
            configDag.addVertex(configVertex);
        }
    }
    // Add edge info to the config dag
    for (final Edge edge : avroDag.getEdges()) {
        final int srcIndex = edge.getFrom();
        final int dstIndex = edge.getTo();
        // Add edge to the config dag
        final ConfigVertex deserializedSrcVertex = deserializedVertices.get(srcIndex);
        final ConfigVertex deserializedDstVertex = deserializedVertices.get(dstIndex);
        configDag.addEdge(deserializedSrcVertex, deserializedDstVertex, new MISTEdge(edge.getDirection(), edge.getBranchIndex()));
    }
    return configDag;
}

Example 2 with AdjacentListDAG

use of edu.snu.mist.common.graph.AdjacentListDAG in project mist by snuspl.

the class MergeAwareQueryRemoverTest method generateSimpleDag.

/**
 * Generate a simple query that has the following structure: src -> operator chain -> sink.
 * @param source source
 * @param physicalOperator operator chain
 * @param sink sink
 * @return dag
 */
private Tuple<DAG<ConfigVertex, MISTEdge>, ExecutionDag> generateSimpleDag(final TestSource source, final PhysicalOperator physicalOperator, final PhysicalSink<String> sink, final ConfigVertex srcVertex, final ConfigVertex ocVertex, final ConfigVertex sinkVertex) throws IOException {
    // Create DAG
    final DAG<ConfigVertex, MISTEdge> dag = new AdjacentListDAG<>();
    dag.addVertex(srcVertex);
    dag.addVertex(ocVertex);
    dag.addVertex(sinkVertex);
    dag.addEdge(srcVertex, ocVertex, new MISTEdge(Direction.LEFT));
    dag.addEdge(ocVertex, sinkVertex, new MISTEdge(Direction.LEFT));
    final DAG<ExecutionVertex, MISTEdge> newDag = new AdjacentListDAG<>();
    newDag.addVertex(source);
    newDag.addVertex(physicalOperator);
    newDag.addVertex(sink);
    newDag.addEdge(source, physicalOperator, new MISTEdge(Direction.LEFT));
    newDag.addEdge(physicalOperator, sink, new MISTEdge(Direction.LEFT));
    final ExecutionDag executionDag = new ExecutionDag(newDag);
    return new Tuple<>(dag, executionDag);
}

Example 3 with AdjacentListDAG

use of edu.snu.mist.common.graph.AdjacentListDAG in project mist by snuspl.

the class OperatorChainDagGenerator method generateOperatorChainDAG.

/**
 * Generate OperatorChain DAG according to the logic described above.
 * @return the OperatorChain DAG
 * The chain is represented as a list and AvroVertexSerializable can be serialized by avro
 */
public DAG<List<MISTStream>, MISTEdge> generateOperatorChainDAG() {
    final DAG<OperatorChain, MISTEdge> chainDag = new AdjacentListDAG<>();
    final Map<MISTStream, OperatorChain> vertexChainMap = new HashMap<>();
    // from the root operators which are following sources.
    for (final MISTStream source : optimizedDag.getRootVertices()) {
        final Map<MISTStream, MISTEdge> rootEdges = optimizedDag.getEdges(source);
        // This chaining group is a wrapper for List, for equality check
        final OperatorChain srcChain = new OperatorChain();
        // Partition Source
        srcChain.chain.add(source);
        chainDag.addVertex(srcChain);
        vertexChainMap.put(source, srcChain);
        for (final Map.Entry<MISTStream, MISTEdge> entry : rootEdges.entrySet()) {
            final MISTStream nextVertex = entry.getKey();
            final MISTEdge edge = entry.getValue();
            final OperatorChain nextChain = getOperatorChain(nextVertex, vertexChainMap, chainDag);
            chainDag.addEdge(srcChain, nextChain, edge);
            chainingInDfsOrder(nextChain, nextVertex, chainDag, vertexChainMap);
        }
    }
    // Convert to List<AvroVertexSerializable> for AvroOperatorChainDag
    final DAG<List<MISTStream>, MISTEdge> result = new AdjacentListDAG<>();
    final Queue<OperatorChain> queue = new LinkedList<>();
    final Iterator<OperatorChain> iterator = GraphUtils.topologicalSort(chainDag);
    while (iterator.hasNext()) {
        final OperatorChain queryPartition = iterator.next();
        queue.add(queryPartition);
        result.addVertex(queryPartition.chain);
    }
    for (final OperatorChain operatorChain : queue) {
        final Map<OperatorChain, MISTEdge> edges = chainDag.getEdges(operatorChain);
        for (final Map.Entry<OperatorChain, MISTEdge> edge : edges.entrySet()) {
            result.addEdge(operatorChain.chain, edge.getKey().chain, edge.getValue());
        }
    }
    return result;
}

Example 4 with AdjacentListDAG

use of edu.snu.mist.common.graph.AdjacentListDAG in project mist by snuspl.

the class QueryManagerTest method testSubmitComplexQueryHelper.

@SuppressWarnings("unchecked")
private void testSubmitComplexQueryHelper(final Configuration conf) throws Exception {
    final String queryId = "testQuery";
    final List<String> inputs = Arrays.asList("mist is a cloud of tiny water droplets suspended in the atmosphere", "a mist rose out of the river", "the peaks were shrouded in mist");
    // Expected results
    final List<Map<String, Integer>> intermediateResult = getIntermediateResult(inputs);
    final List<String> expectedSink1Output = intermediateResult.stream().map(input -> input.toString()).collect(Collectors.toList());
    final List<Integer> expectedSink2Output = intermediateResult.stream().map(totalCountMapFunc).collect(Collectors.toList());
    // Number of expected outputs
    final CountDownLatch countDownAllOutputs = new CountDownLatch(intermediateResult.size() * 2);
    // Create the execution DAG of the query
    final ExecutionDag executionDag = new ExecutionDag(new AdjacentListDAG<>());
    // Create source
    final TestDataGenerator dataGenerator = new TestDataGenerator(inputs);
    final EventGenerator eventGenerator = new PunctuatedEventGenerator(null, input -> false, null, 0, null, null);
    final PhysicalSource src = new PhysicalSourceImpl("testSource", new HashMap<>(), dataGenerator, eventGenerator);
    final Injector injector = Tang.Factory.getTang().newInjector(conf);
    // Create sinks
    final List<String> sink1Result = new LinkedList<>();
    final List<Integer> sink2Result = new LinkedList<>();
    final PhysicalSink sink1 = new PhysicalSinkImpl<>("sink1", null, new TestWithCountDownSink<String>(sink1Result, countDownAllOutputs));
    final PhysicalSink sink2 = new PhysicalSinkImpl<>("sink2", null, new TestWithCountDownSink<Integer>(sink2Result, countDownAllOutputs));
    // Fake operator chain dag of QueryManager
    final AvroDag fakeAvroDag = new AvroDag();
    fakeAvroDag.setQueryId(queryId);
    // fakeAvroDag.setSuperGroupId("testGroup");
    // Construct execution dag
    constructExecutionDag(executionDag, src, sink1, sink2);
    // Create mock DagGenerator. It returns the above  execution dag
    final ConfigDagGenerator configDagGenerator = mock(ConfigDagGenerator.class);
    final DAG<ConfigVertex, MISTEdge> configDag = mock(DAG.class);
    when(configDagGenerator.generate(fakeAvroDag)).thenReturn(configDag);
    final DagGenerator dagGenerator = mock(DagGenerator.class);
    // when(dagGenerator.generate(configDag, tuple.getValue().getJarFilePaths())).thenReturn(executionDag);
    // Build QueryManager
    final QueryManager queryManager = queryManagerBuild(fakeAvroDag, configDagGenerator, dagGenerator, injector);
    queryManager.create(fakeAvroDag);
    // Wait until all of the outputs are generated
    countDownAllOutputs.await();
    // Check the outputs
    Assert.assertEquals(expectedSink1Output, sink1Result);
    Assert.assertEquals(expectedSink2Output, sink2Result);
    src.close();
    queryManager.close();
    // Delete plan directory and plans
    deletePlans(injector);
}