Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 66 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method computeVertexDescendants.

private BitSet computeVertexDescendants(BitSet verticesVisited, Vertex v) {
    int vertexIndex = v.getVertexId().getId();
    BitSet descendants = vertexDescendants.get(vertexIndex);
    if (!verticesVisited.get(vertexIndex)) {
        for (Vertex child : v.getOutputVertices().keySet()) {
            descendants.set(child.getVertexId().getId());
            BitSet childDescendants = computeVertexDescendants(verticesVisited, child);
            descendants.or(childDescendants);
        }
        verticesVisited.set(vertexIndex);
    }
    return descendants;
}

Example 67 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method commitOrFinish.

// either commit when all vertices are completed or just finish if there's no committer
private synchronized DAGState commitOrFinish() {
    // commit all other outputs
    // we come here for successful dag completion and when outputs need to be
    // committed at the end for all or none visibility
    Map<OutputKey, CallableEvent> commitEvents = new HashMap<OutputKey, CallableEvent>();
    // commit all shared outputs
    for (final VertexGroupInfo groupInfo : vertexGroups.values()) {
        if (!groupInfo.outputs.isEmpty()) {
            groupInfo.commitStarted = true;
            final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
            try {
                TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), v.getVertexId());
                for (final String outputName : groupInfo.outputs) {
                    final OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
                    CommitCallback groupCommitCallback = new CommitCallback(outputKey);
                    CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {

                        @Override
                        public Void call() throws Exception {
                            OutputCommitter committer = v.getOutputCommitters().get(outputName);
                            LOG.info("Committing output: " + outputKey);
                            commitOutput(committer);
                            return null;
                        }
                    };
                    commitEvents.put(outputKey, groupCommitCallableEvent);
                }
            } finally {
                appContext.getHadoopShim().clearHadoopCallerContext();
            }
        }
    }
    for (final Vertex vertex : vertices.values()) {
        if (vertex.getOutputCommitters() == null) {
            LOG.info("No output committers for vertex: " + vertex.getLogIdentifier());
            continue;
        }
        Map<String, OutputCommitter> outputCommitters = new HashMap<String, OutputCommitter>(vertex.getOutputCommitters());
        Set<String> sharedOutputs = vertex.getSharedOutputs();
        // remove shared outputs
        if (sharedOutputs != null) {
            Iterator<Map.Entry<String, OutputCommitter>> iter = outputCommitters.entrySet().iterator();
            while (iter.hasNext()) {
                if (sharedOutputs.contains(iter.next().getKey())) {
                    iter.remove();
                }
            }
        }
        if (outputCommitters.isEmpty()) {
            LOG.info("No exclusive output committers for vertex: " + vertex.getLogIdentifier());
            continue;
        }
        try {
            TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), vertex.getVertexId());
            for (final Map.Entry<String, OutputCommitter> entry : outputCommitters.entrySet()) {
                if (vertex.getState() != VertexState.SUCCEEDED) {
                    throw new TezUncheckedException("Vertex: " + vertex.getLogIdentifier() + " not in SUCCEEDED state. State= " + vertex.getState());
                }
                OutputKey outputKey = new OutputKey(entry.getKey(), vertex.getName(), false);
                CommitCallback commitCallback = new CommitCallback(outputKey);
                CallableEvent commitCallableEvent = new CallableEvent(commitCallback) {

                    @Override
                    public Void call() throws Exception {
                        LOG.info("Committing output: " + entry.getKey() + " for vertex: " + vertex.getLogIdentifier() + ", outputName: " + entry.getKey());
                        commitOutput(entry.getValue());
                        return null;
                    }
                };
                commitEvents.put(outputKey, commitCallableEvent);
            }
        } finally {
            appContext.getHadoopShim().clearHadoopCallerContext();
        }
    }
    if (!commitEvents.isEmpty()) {
        try {
            LOG.info("Start writing dag commit event, " + getID());
            appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(), new DAGCommitStartedEvent(getID(), clock.getTime())));
        } catch (IOException e) {
            LOG.error("Failed to send commit event to history/recovery handler", e);
            trySetTerminationCause(DAGTerminationCause.RECOVERY_FAILURE);
            return finished(DAGState.FAILED);
        }
        for (Map.Entry<OutputKey, CallableEvent> entry : commitEvents.entrySet()) {
            ListenableFuture<Void> commitFuture = appContext.getExecService().submit(entry.getValue());
            Futures.addCallback(commitFuture, entry.getValue().getCallback());
            commitFutures.put(entry.getKey(), commitFuture);
        }
    }
    if (commitFutures.isEmpty()) {
        // no commit needs to be done
        return finished(DAGState.SUCCEEDED);
    } else {
        return DAGState.COMMITTING;
    }
}

Example 68 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method constructFinalFullcounters.

@Private
public void constructFinalFullcounters() {
    this.fullCounters = new TezCounters();
    this.fullCounters.incrAllCounters(dagCounters);
    for (Vertex v : this.vertices.values()) {
        this.fullCounters.incrAllCounters(v.getAllCounters());
    }
}

Example 69 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method parseVertexEdges.

// hooks up this VertexImpl to input and output EdgeProperties
private static void parseVertexEdges(DAGImpl dag, Map<String, EdgePlan> edgePlans, Vertex vertex) {
    VertexPlan vertexPlan = vertex.getVertexPlan();
    Map<Vertex, Edge> inVertices = new HashMap<Vertex, Edge>();
    Map<Vertex, Edge> outVertices = new HashMap<Vertex, Edge>();
    for (String inEdgeId : vertexPlan.getInEdgeIdList()) {
        EdgePlan edgePlan = edgePlans.get(inEdgeId);
        Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName());
        Edge edge = dag.edges.get(inEdgeId);
        edge.setSourceVertex(inVertex);
        edge.setDestinationVertex(vertex);
        inVertices.put(inVertex, edge);
    }
    for (String outEdgeId : vertexPlan.getOutEdgeIdList()) {
        EdgePlan edgePlan = edgePlans.get(outEdgeId);
        Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName());
        Edge edge = dag.edges.get(outEdgeId);
        edge.setSourceVertex(vertex);
        edge.setDestinationVertex(outVertex);
        outVertices.put(outVertex, edge);
    }
    vertex.setInputVertices(inVertices);
    vertex.setOutputVertices(outVertices);
}

Example 70 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method assignDAGScheduler.

private static void assignDAGScheduler(DAGImpl dag) throws TezException {
    String dagSchedulerClassName = dag.dagConf.get(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS_DEFAULT);
    LOG.info("Using DAG Scheduler: " + dagSchedulerClassName);
    dag.dagScheduler = ReflectionUtils.createClazzInstance(dagSchedulerClassName, new Class<?>[] { DAG.class, EventHandler.class }, new Object[] { dag, dag.eventHandler });
    for (Vertex v : dag.vertices.values()) {
        dag.dagScheduler.addVertexConcurrencyLimit(v.getVertexId(), v.getMaxTaskConcurrency());
    }
}