Examples with MISTEdge edu.snu.mist.common.graph.MISTEdge used on opensource projects

Example 26 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class MISTStreamImpl method transformToSingleInputContinuousStream.

/**
 * Transform the upstream to a new continuous stream
 * by applying the operation corresponding to the given configuration.
 * @param opConf configuration
 * @param upStream upstream
 * @param <OUT> output type
 * @return continuous stream
 */
protected <OUT> ContinuousStream<OUT> transformToSingleInputContinuousStream(final Map<String, String> opConf, final MISTStream upStream) {
    final ContinuousStream<OUT> downStream = new ContinuousStreamImpl<>(dag, opConf);
    dag.addVertex(downStream);
    dag.addEdge(upStream, downStream, new MISTEdge(Direction.LEFT));
    return downStream;
}

Example 27 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ImmediateQueryMergingStarter method generate.

/**
 * This generates a new execution dag from the configuration dag.
 */
private ExecutionDag generate(final DAG<ConfigVertex, MISTEdge> configDag, final URL[] urls, final ClassLoader classLoader) throws IOException, ClassNotFoundException {
    // For execution dag
    final ExecutionDag executionDag = new ExecutionDag(new AdjacentListConcurrentMapDAG<>());
    final Map<ConfigVertex, ExecutionVertex> created = new HashMap<>(configDag.numberOfVertices());
    for (final ConfigVertex source : configDag.getRootVertices()) {
        final ExecutionVertex currExecutionVertex = executionVertexGenerator.generate(source, urls, classLoader);
        created.put(source, currExecutionVertex);
        configExecutionVertexMap.put(source, currExecutionVertex);
        executionVertexCountMap.put(currExecutionVertex, 1);
        executionVertexDagMap.put(currExecutionVertex, executionDag);
        executionDag.getDag().addVertex(currExecutionVertex);
        // do dfs creation
        for (final Map.Entry<ConfigVertex, MISTEdge> edges : configDag.getEdges(source).entrySet()) {
            final ConfigVertex childVertex = edges.getKey();
            final MISTEdge edge = edges.getValue();
            dfsCreation(currExecutionVertex, edge, childVertex, created, configDag, executionDag, urls, classLoader);
        }
    }
    return executionDag;
}

Example 28 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class ImmediateQueryMergingStarter method dfsCreation.

/**
 * Create the execution dag in dfs order.
 */
private void dfsCreation(final ExecutionVertex parent, final MISTEdge parentEdge, final ConfigVertex currVertex, final Map<ConfigVertex, ExecutionVertex> created, final DAG<ConfigVertex, MISTEdge> configDag, final ExecutionDag executionDag, final URL[] urls, final ClassLoader classLoader) throws IOException, ClassNotFoundException {
    final ExecutionVertex currExecutionVertex;
    if (created.get(currVertex) == null) {
        currExecutionVertex = executionVertexGenerator.generate(currVertex, urls, classLoader);
        created.put(currVertex, currExecutionVertex);
        executionVertexCountMap.put(currExecutionVertex, 1);
        executionVertexDagMap.put(currExecutionVertex, executionDag);
        executionDag.getDag().addVertex(currExecutionVertex);
        // do dfs creation
        for (final Map.Entry<ConfigVertex, MISTEdge> edges : configDag.getEdges(currVertex).entrySet()) {
            final ConfigVertex childVertex = edges.getKey();
            final MISTEdge edge = edges.getValue();
            dfsCreation(currExecutionVertex, edge, childVertex, created, configDag, executionDag, urls, classLoader);
        }
    } else {
        currExecutionVertex = created.get(currVertex);
    }
    configExecutionVertexMap.put(currVertex, currExecutionVertex);
    executionDag.getDag().addEdge(parent, currExecutionVertex, parentEdge);
}

Example 29 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class LogicalDagOptimizer method getOptimizedDAG.

/**
 * Apply optimization techniques to the logical DAG.
 * @return the optimized DAG
 */
public DAG<MISTStream, MISTEdge> getOptimizedDAG() {
    // check visited vertices
    final Set<MISTStream> visited = new HashSet<>();
    // from the root operators which are following sources.
    for (final MISTStream source : dag.getRootVertices()) {
        final Map<MISTStream, MISTEdge> rootEdges = dag.getEdges(source);
        visited.add(source);
        for (final MISTStream nextVertex : rootEdges.keySet()) {
            optimizeSubDag(nextVertex, visited);
        }
    }
    return dag;
}

Example 30 with MISTEdge

use of edu.snu.mist.common.graph.MISTEdge in project mist by snuspl.

the class LogicalDagOptimizer method optimizeSubDag.

/**
 * Obtimize the operators and sinks recursively (DFS order) according to the mechanism.
 * @param currVertex  current vertex
 * @param visited visited vertices
 */
private void optimizeSubDag(final MISTStream currVertex, final Set<MISTStream> visited) {
    if (!visited.contains(currVertex)) {
        visited.add(currVertex);
        final Map<MISTStream, MISTEdge> edges = dag.getEdges(currVertex);
        // checking whether there is any conditionally branching edge diverged from current vertex
        if (!(currVertex instanceof ContinuousStreamImpl) || ((ContinuousStreamImpl) currVertex).getCondBranchCount() == 0) {
            // current vertex is not a continuous stream or this edge is an ordinary (non-branch) edge
            for (final MISTStream nextVertex : edges.keySet()) {
                optimizeSubDag(nextVertex, visited);
            }
        } else {
            // current vertex has some conditionally branching edges
            final Map<Integer, ContinuousStreamImpl> branchStreams = new HashMap<>();
            // gather the branching streams
            for (final MISTStream nextVertex : edges.keySet()) {
                if (nextVertex instanceof ContinuousStreamImpl) {
                    final ContinuousStreamImpl contNextVertex = (ContinuousStreamImpl) nextVertex;
                    if (contNextVertex.getBranchIndex() > 0) {
                        // this edge is a conditionally branching edge
                        branchStreams.put(contNextVertex.getBranchIndex(), contNextVertex);
                    }
                }
                optimizeSubDag(nextVertex, visited);
            }
            // gather condition udfs from each branch stream
            final List<String> udfs = new ArrayList<>(branchStreams.size());
            for (int i = 1; i <= branchStreams.size(); i++) {
                final ContinuousStreamImpl branchStream = branchStreams.get(i);
                final Map<String, String> conf = branchStream.getConfiguration();
                udfs.add(conf.get(ConfKeys.OperatorConf.UDF_STRING.name()));
            }
            // create a new conditional branch vertex to unify these branch streams
            final Map<String, String> opConf = new HashMap<>();
            try {
                opConf.put(ConfKeys.ConditionalBranchOperator.UDF_LIST_STRING.name(), SerializeUtils.serializeToString((Serializable) udfs));
            } catch (final IOException e) {
                e.printStackTrace();
                throw new RuntimeException(e);
            }
            final ContinuousStreamImpl unifiedBranchStream = new ContinuousStreamImpl(dag, opConf);
            dag.addVertex(unifiedBranchStream);
            dag.addEdge(currVertex, unifiedBranchStream, new MISTEdge(Direction.LEFT));
            // merging all the branching vertices from the current vertex into a single conditional branch vertex
            for (final ContinuousStreamImpl branchStream : branchStreams.values()) {
                final List<MISTStream> branchDownStreams = new ArrayList<>();
                for (final Map.Entry<MISTStream, MISTEdge> edgeFromBranch : dag.getEdges(branchStream).entrySet()) {
                    final MISTStream branchDownStream = edgeFromBranch.getKey();
                    branchDownStreams.add(branchDownStream);
                    dag.addEdge(unifiedBranchStream, branchDownStream, new MISTEdge(edgeFromBranch.getValue().getDirection(), branchStream.getBranchIndex()));
                }
                // to prevent the concurrent map modification, remove the edges from downStream separately
                for (final MISTStream branchDownStream : branchDownStreams) {
                    dag.removeEdge(branchStream, branchDownStream);
                }
                dag.removeEdge(currVertex, branchStream);
                dag.removeVertex(branchStream);
            }
        }
    }
}