Examples with DAGHistoryEvent org.apache.tez.dag.history.DAGHistoryEvent used on opensource projects

Example 6 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class DAGImpl method vertexSucceeded.

private boolean vertexSucceeded(Vertex vertex) {
    numSuccessfulVertices++;
    boolean recoveryFailed = false;
    if (!commitAllOutputsOnSuccess && isCommittable()) {
        // committing successful outputs immediately. check for shared outputs
        List<VertexGroupInfo> groupsList = vertexGroupInfo.get(vertex.getName());
        if (groupsList != null) {
            List<VertexGroupInfo> commitList = Lists.newArrayListWithCapacity(groupsList.size());
            for (VertexGroupInfo groupInfo : groupsList) {
                groupInfo.successfulMembers++;
                if (groupInfo.groupMembers.size() == groupInfo.successfulMembers && !groupInfo.outputs.isEmpty()) {
                    // group has outputs and all vertex members are done
                    LOG.info("All members of group: " + groupInfo.groupName + " are succeeded. Commiting outputs");
                    commitList.add(groupInfo);
                }
            }
            for (VertexGroupInfo groupInfo : commitList) {
                if (recoveryData != null && recoveryData.isVertexGroupCommitted(groupInfo.groupName)) {
                    LOG.info("VertexGroup was already committed as per recovery" + " data, groupName=" + groupInfo.groupName);
                    for (String vertexName : groupInfo.groupMembers) {
                        VertexRecoveryData vertexRecoveryData = recoveryData.getVertexRecoveryData(getVertex(vertexName).getVertexId());
                        Preconditions.checkArgument(vertexRecoveryData != null, "Vertex Group has been committed" + ", but no VertexRecoveryData found for its vertex " + vertexName);
                        VertexFinishedEvent vertexFinishedEvent = vertexRecoveryData.getVertexFinishedEvent();
                        Preconditions.checkArgument(vertexFinishedEvent != null, "Vertex Group has been committed" + ", but no VertexFinishedEvent found in its vertex " + vertexName);
                        Preconditions.checkArgument(vertexFinishedEvent.getState() == VertexState.SUCCEEDED, "Vertex Group has been committed, but unexpected vertex state of its vertex " + vertexName + ", vertexstate=" + vertexFinishedEvent.getState());
                    }
                    continue;
                }
                groupInfo.commitStarted = true;
                final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
                try {
                    Collection<TezVertexID> vertexIds = getVertexIds(groupInfo.groupMembers);
                    appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(), new VertexGroupCommitStartedEvent(dagId, groupInfo.groupName, vertexIds, clock.getTime())));
                } catch (IOException e) {
                    LOG.error("Failed to send commit recovery event to handler", e);
                    recoveryFailed = true;
                }
                if (!recoveryFailed) {
                    for (final String outputName : groupInfo.outputs) {
                        OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
                        CommitCallback groupCommitCallback = new CommitCallback(outputKey);
                        CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {

                            public Void call() throws Exception {
                                OutputCommitter committer = v.getOutputCommitters().get(outputName);
                                LOG.info("Committing output: " + outputName);
                                commitOutput(committer);
                                return null;
                            }
                        };
                        ListenableFuture<Void> groupCommitFuture = appContext.getExecService().submit(groupCommitCallableEvent);
                        Futures.addCallback(groupCommitFuture, groupCommitCallableEvent.getCallback());
                        commitFutures.put(outputKey, groupCommitFuture);
                    }
                }
            }
        }
    }
    if (recoveryFailed) {
        LOG.info("Recovery failure occurred during commit");
        enactKill(DAGTerminationCause.RECOVERY_FAILURE, VertexTerminationCause.COMMIT_FAILURE);
    }
    return !recoveryFailed;
}

Example 7 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class DAGImpl method logJobHistoryUnsuccesfulEvent.

void logJobHistoryUnsuccesfulEvent(DAGState state, TezCounters counters) throws IOException {
    if (recoveryData == null || recoveryData.getDAGFinishedEvent() == null) {
        Map<String, Integer> taskStats = constructTaskStats(getDAGProgress());
        if (finishTime < startTime) {
            LOG.warn("DAG finish time is smaller than start time. " + "startTime=" + startTime + ", finishTime=" + finishTime);
        }
        DAGFinishedEvent finishEvt = new DAGFinishedEvent(dagId, startTime, finishTime, state, StringUtils.join(getDiagnostics(), LINE_SEPARATOR), counters, this.userName, this.dagName, taskStats, this.appContext.getApplicationAttemptId(), this.jobPlan);
        this.appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(dagId, finishEvt));
    }
}

Example 8 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class DAGImpl method commitCompleted.

private boolean commitCompleted(DAGEventCommitCompleted commitCompletedEvent) {
    Preconditions.checkState(commitFutures.remove(commitCompletedEvent.getOutputKey()) != null, "Unknown commit:" + commitCompletedEvent.getOutputKey());
    boolean commitFailed = false;
    boolean recoveryFailed = false;
    if (commitCompletedEvent.isSucceeded()) {
        LOG.info("Commit succeeded for output:" + commitCompletedEvent.getOutputKey());
        OutputKey outputKey = commitCompletedEvent.getOutputKey();
        if (outputKey.isVertexGroupOutput) {
            VertexGroupInfo vertexGroup = vertexGroups.get(outputKey.getEntityName());
            vertexGroup.successfulCommits++;
            if (vertexGroup.isCommitted()) {
                if (!commitAllOutputsOnSuccess) {
                    try {
                        Collection<TezVertexID> vertexIds = getVertexIds(vertexGroup.groupMembers);
                        appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(), new VertexGroupCommitFinishedEvent(getID(), commitCompletedEvent.getOutputKey().getEntityName(), vertexIds, clock.getTime())));
                    } catch (IOException e) {
                        String diag = "Failed to send commit recovery event to handler, " + ExceptionUtils.getStackTrace(e);
                        addDiagnostic(diag);
                        LOG.error(diag);
                        recoveryFailed = true;
                    }
                }
            }
        }
    } else {
        String diag = "Commit failed for output: " + commitCompletedEvent.getOutputKey() + ", " + ExceptionUtils.getStackTrace(commitCompletedEvent.getException());
        addDiagnostic(diag);
        LOG.error(diag);
        commitFailed = true;
    }
    if (commitFailed) {
        enactKill(DAGTerminationCause.COMMIT_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE);
        cancelCommits();
    }
    if (recoveryFailed) {
        enactKill(DAGTerminationCause.RECOVERY_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE);
        cancelCommits();
    }
    return !commitFailed && !recoveryFailed;
}

Example 9 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class DAGImpl method logJobHistoryInitedEvent.

void logJobHistoryInitedEvent() {
    if (recoveryData == null || recoveryData.getDAGInitializedEvent() == null) {
        DAGInitializedEvent initEvt = new DAGInitializedEvent(this.dagId, clock.getTime(), this.userName, this.dagName, this.getVertexNameIDMapping());
        this.appContext.getHistoryHandler().handle(new DAGHistoryEvent(dagId, initEvt));
    }
}

Example 10 with DAGHistoryEvent

use of org.apache.tez.dag.history.DAGHistoryEvent in project tez by apache.

the class TestVertexImpl method verifyHistoryEvents.

private void verifyHistoryEvents(List<DAGHistoryEvent> events, HistoryEventType eventType, int expectedTimes) {
    int actualTimes = 0;
    LOG.info("");
    for (DAGHistoryEvent event : events) {
        LOG.info(event.getHistoryEvent().getEventType() + "");
        if (event.getHistoryEvent().getEventType() == eventType) {
            actualTimes++;
        }
    }
    Assert.assertEquals(actualTimes, expectedTimes);
}