Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 6 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method getDAGStatus.

// monitoring apis
@Override
public DAGStatusBuilder getDAGStatus(Set<StatusGetOpts> statusOptions) {
    DAGStatusBuilder status = new DAGStatusBuilder();
    int totalTaskCount = 0;
    int totalSucceededTaskCount = 0;
    int totalRunningTaskCount = 0;
    int totalFailedTaskCount = 0;
    int totalKilledTaskCount = 0;
    int totalFailedTaskAttemptCount = 0;
    int totalKilledTaskAttemptCount = 0;
    int totalRejectedTaskAttemptCount = 0;
    readLock.lock();
    try {
        for (Map.Entry<String, Vertex> entry : vertexMap.entrySet()) {
            ProgressBuilder progress = entry.getValue().getVertexProgress();
            status.addVertexProgress(entry.getKey(), progress);
            totalTaskCount += progress.getTotalTaskCount();
            totalSucceededTaskCount += progress.getSucceededTaskCount();
            totalRunningTaskCount += progress.getRunningTaskCount();
            totalFailedTaskCount += progress.getFailedTaskCount();
            totalKilledTaskCount += progress.getKilledTaskCount();
            totalFailedTaskAttemptCount += progress.getFailedTaskAttemptCount();
            totalKilledTaskAttemptCount += progress.getKilledTaskAttemptCount();
            totalRejectedTaskAttemptCount += progress.getRejectedTaskAttemptCount();
        }
        ProgressBuilder dagProgress = new ProgressBuilder();
        dagProgress.setTotalTaskCount(totalTaskCount);
        dagProgress.setSucceededTaskCount(totalSucceededTaskCount);
        dagProgress.setRunningTaskCount(totalRunningTaskCount);
        dagProgress.setFailedTaskCount(totalFailedTaskCount);
        dagProgress.setKilledTaskCount(totalKilledTaskCount);
        dagProgress.setFailedTaskAttemptCount(totalFailedTaskAttemptCount);
        dagProgress.setKilledTaskAttemptCount(totalKilledTaskAttemptCount);
        dagProgress.setRejectedTaskAttemptCount(totalRejectedTaskAttemptCount);
        status.setState(getState());
        status.setDiagnostics(diagnostics);
        status.setDAGProgress(dagProgress);
        if (statusOptions.contains(StatusGetOpts.GET_COUNTERS)) {
            status.setDAGCounters(getAllCounters());
        }
        return status;
    } finally {
        readLock.unlock();
    }
}

Example 7 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method vertexSucceeded.

private boolean vertexSucceeded(Vertex vertex) {
    numSuccessfulVertices++;
    boolean recoveryFailed = false;
    if (!commitAllOutputsOnSuccess && isCommittable()) {
        // committing successful outputs immediately. check for shared outputs
        List<VertexGroupInfo> groupsList = vertexGroupInfo.get(vertex.getName());
        if (groupsList != null) {
            List<VertexGroupInfo> commitList = Lists.newArrayListWithCapacity(groupsList.size());
            for (VertexGroupInfo groupInfo : groupsList) {
                groupInfo.successfulMembers++;
                if (groupInfo.groupMembers.size() == groupInfo.successfulMembers && !groupInfo.outputs.isEmpty()) {
                    // group has outputs and all vertex members are done
                    LOG.info("All members of group: " + groupInfo.groupName + " are succeeded. Commiting outputs");
                    commitList.add(groupInfo);
                }
            }
            for (VertexGroupInfo groupInfo : commitList) {
                if (recoveryData != null && recoveryData.isVertexGroupCommitted(groupInfo.groupName)) {
                    LOG.info("VertexGroup was already committed as per recovery" + " data, groupName=" + groupInfo.groupName);
                    for (String vertexName : groupInfo.groupMembers) {
                        VertexRecoveryData vertexRecoveryData = recoveryData.getVertexRecoveryData(getVertex(vertexName).getVertexId());
                        Preconditions.checkArgument(vertexRecoveryData != null, "Vertex Group has been committed" + ", but no VertexRecoveryData found for its vertex " + vertexName);
                        VertexFinishedEvent vertexFinishedEvent = vertexRecoveryData.getVertexFinishedEvent();
                        Preconditions.checkArgument(vertexFinishedEvent != null, "Vertex Group has been committed" + ", but no VertexFinishedEvent found in its vertex " + vertexName);
                        Preconditions.checkArgument(vertexFinishedEvent.getState() == VertexState.SUCCEEDED, "Vertex Group has been committed, but unexpected vertex state of its vertex " + vertexName + ", vertexstate=" + vertexFinishedEvent.getState());
                    }
                    continue;
                }
                groupInfo.commitStarted = true;
                final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
                try {
                    Collection<TezVertexID> vertexIds = getVertexIds(groupInfo.groupMembers);
                    appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(), new VertexGroupCommitStartedEvent(dagId, groupInfo.groupName, vertexIds, clock.getTime())));
                } catch (IOException e) {
                    LOG.error("Failed to send commit recovery event to handler", e);
                    recoveryFailed = true;
                }
                if (!recoveryFailed) {
                    for (final String outputName : groupInfo.outputs) {
                        OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
                        CommitCallback groupCommitCallback = new CommitCallback(outputKey);
                        CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {

                            public Void call() throws Exception {
                                OutputCommitter committer = v.getOutputCommitters().get(outputName);
                                LOG.info("Committing output: " + outputName);
                                commitOutput(committer);
                                return null;
                            }
                        };
                        ListenableFuture<Void> groupCommitFuture = appContext.getExecService().submit(groupCommitCallableEvent);
                        Futures.addCallback(groupCommitFuture, groupCommitCallableEvent.getCallback());
                        commitFutures.put(outputKey, groupCommitFuture);
                    }
                }
            }
        }
    }
    if (recoveryFailed) {
        LOG.info("Recovery failure occurred during commit");
        enactKill(DAGTerminationCause.RECOVERY_FAILURE, VertexTerminationCause.COMMIT_FAILURE);
    }
    return !recoveryFailed;
}

Example 8 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method computeVertexDescendants.

private void computeVertexDescendants() {
    vertexDescendants = new ArrayList<>(numVertices);
    for (int i = 0; i < numVertices; ++i) {
        vertexDescendants.add(new BitSet(numVertices));
    }
    BitSet verticesVisited = new BitSet(numVertices);
    for (Vertex v : vertices.values()) {
        computeVertexDescendants(verticesVisited, v);
    }
}

Example 9 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method getDAGProgress.

private ProgressBuilder getDAGProgress() {
    int totalTaskCount = 0;
    int totalSucceededTaskCount = 0;
    int totalRunningTaskCount = 0;
    int totalFailedTaskCount = 0;
    int totalKilledTaskCount = 0;
    int totalFailedTaskAttemptCount = 0;
    int totalKilledTaskAttemptCount = 0;
    int totalRejectedTaskAttemptCount = 0;
    readLock.lock();
    try {
        for (Map.Entry<String, Vertex> entry : vertexMap.entrySet()) {
            ProgressBuilder progress = entry.getValue().getVertexProgress();
            totalTaskCount += progress.getTotalTaskCount();
            totalSucceededTaskCount += progress.getSucceededTaskCount();
            totalRunningTaskCount += progress.getRunningTaskCount();
            totalFailedTaskCount += progress.getFailedTaskCount();
            totalKilledTaskCount += progress.getKilledTaskCount();
            totalFailedTaskAttemptCount += progress.getFailedTaskAttemptCount();
            totalKilledTaskAttemptCount += progress.getKilledTaskAttemptCount();
            totalRejectedTaskAttemptCount += progress.getRejectedTaskAttemptCount();
        }
        ProgressBuilder dagProgress = new ProgressBuilder();
        dagProgress.setTotalTaskCount(totalTaskCount);
        dagProgress.setSucceededTaskCount(totalSucceededTaskCount);
        dagProgress.setRunningTaskCount(totalRunningTaskCount);
        dagProgress.setFailedTaskCount(totalFailedTaskCount);
        dagProgress.setKilledTaskCount(totalKilledTaskCount);
        dagProgress.setFailedTaskAttemptCount(totalFailedTaskAttemptCount);
        dagProgress.setKilledTaskAttemptCount(totalKilledTaskAttemptCount);
        dagProgress.setRejectedTaskAttemptCount(totalRejectedTaskAttemptCount);
        return dagProgress;
    } finally {
        readLock.unlock();
    }
}

Example 10 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGSchedulerNaturalOrderControlled method trySchedulingVertex.

/* Process the specified vertex, and add it to the cache of scheduled vertices if it can be scheduled */
private boolean trySchedulingVertex(Vertex vertex) {
    boolean canSchedule = true;
    if (vertexScheduledTasks.get(vertex.getName()) == null) {
        // 0 task vertices handled elsewhere.
        if (LOG.isDebugEnabled()) {
            LOG.debug("No schedule requests for vertex: " + vertex.getLogIdentifier() + ", Not scheduling");
        }
        canSchedule = false;
    } else {
        Map<Vertex, Edge> inputVertexEdgeMap = vertex.getInputVertices();
        if (inputVertexEdgeMap == null || inputVertexEdgeMap.isEmpty()) {
            // Nothing to wait for. Go ahead and scheduled.
            if (LOG.isDebugEnabled()) {
                LOG.debug("No sources for vertex: " + vertex.getLogIdentifier() + ", Scheduling now");
            }
        } else {
            // Check if all sources are scheduled.
            for (Vertex srcVertex : inputVertexEdgeMap.keySet()) {
                if (scheduledTasksForwarded(srcVertex)) {
                    // Nothing to wait for. Go ahead and check the next source.
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Trying to schedule: " + vertex.getLogIdentifier() + ", All tasks forwarded for srcVertex: " + srcVertex.getLogIdentifier() + ", count: " + srcVertex.getTotalTasks());
                    }
                } else {
                    // Special case for vertices with 0 tasks. 0 check is sufficient since parallelism cannot increase.
                    if (srcVertex.getTotalTasks() == 0) {
                        LOG.info("Vertex: " + srcVertex.getLogIdentifier() + " has 0 tasks. Marking as scheduled");
                        scheduledVertices.add(srcVertex.getName());
                        taskAttemptSeen(srcVertex.getName(), null);
                    } else {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Not all sources schedule requests complete while trying to schedule: " + vertex.getLogIdentifier() + ", For source vertex: " + srcVertex.getLogIdentifier() + ", Forwarded requests: " + (vertexScheduledTasks.get(srcVertex.getName()) == null ? "null(0)" : vertexScheduledTasks.get(srcVertex.getName()).cardinality()) + " out of " + srcVertex.getTotalTasks());
                        }
                        canSchedule = false;
                        break;
                    }
                }
            }
        }
    }
    if (canSchedule) {
        scheduledVertices.add(vertex.getName());
    }
    return canSchedule;
}