Examples with OutputCommitter org.apache.tez.runtime.api.OutputCommitter used on opensource projects

Example 1 with OutputCommitter

use of org.apache.tez.runtime.api.OutputCommitter in project tez by apache.

the class VertexImpl method commitOrFinish.

private static VertexState commitOrFinish(final VertexImpl vertex) {
    // commit only once. Dont commit shared outputs
    if (vertex.outputCommitters != null && !vertex.outputCommitters.isEmpty()) {
        if (vertex.recoveryData != null && vertex.recoveryData.isVertexCommitted()) {
            LOG.info("Vertex was already committed as per recovery" + " data, vertex=" + vertex.logIdentifier);
            return vertex.finished(VertexState.SUCCEEDED);
        }
        boolean firstCommit = true;
        for (Entry<String, OutputCommitter> entry : vertex.outputCommitters.entrySet()) {
            final OutputCommitter committer = entry.getValue();
            final String outputName = entry.getKey();
            if (vertex.sharedOutputs.contains(outputName)) {
                // dont commit shared committers. Will be committed by the DAG
                continue;
            }
            if (firstCommit) {
                LOG.info("Invoking committer commit for vertex, vertexId=" + vertex.logIdentifier);
                // Log commit start event on first actual commit
                try {
                    vertex.appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(vertex.getDAGId(), new VertexCommitStartedEvent(vertex.vertexId, vertex.clock.getTime())));
                } catch (IOException e) {
                    LOG.error("Failed to persist commit start event to recovery, vertex=" + vertex.logIdentifier, e);
                    vertex.trySetTerminationCause(VertexTerminationCause.RECOVERY_ERROR);
                    return vertex.finished(VertexState.FAILED);
                }
                firstCommit = false;
            }
            VertexCommitCallback commitCallback = new VertexCommitCallback(vertex, outputName);
            CallableEvent commitCallableEvent = new CallableEvent(commitCallback) {

                @Override
                public Void call() throws Exception {
                    try {
                        TezUtilsInternal.setHadoopCallerContext(vertex.appContext.getHadoopShim(), vertex.vertexId);
                        vertex.dagUgi.doAs(new PrivilegedExceptionAction<Void>() {

                            @Override
                            public Void run() throws Exception {
                                LOG.info("Invoking committer commit for output=" + outputName + ", vertexId=" + vertex.logIdentifier);
                                committer.commitOutput();
                                return null;
                            }
                        });
                    } finally {
                        vertex.appContext.getHadoopShim().clearHadoopCallerContext();
                    }
                    return null;
                }
            };
            ListenableFuture<Void> commitFuture = vertex.getAppContext().getExecService().submit(commitCallableEvent);
            Futures.addCallback(commitFuture, commitCallableEvent.getCallback());
            vertex.commitFutures.put(outputName, commitFuture);
        }
    }
    if (vertex.commitFutures.isEmpty()) {
        return vertex.finished(VertexState.SUCCEEDED);
    } else {
        return VertexState.COMMITTING;
    }
}

Example 2 with OutputCommitter

use of org.apache.tez.runtime.api.OutputCommitter in project tez by apache.

the class DAGImpl method vertexSucceeded.

private boolean vertexSucceeded(Vertex vertex) {
    numSuccessfulVertices++;
    boolean recoveryFailed = false;
    if (!commitAllOutputsOnSuccess && isCommittable()) {
        // committing successful outputs immediately. check for shared outputs
        List<VertexGroupInfo> groupsList = vertexGroupInfo.get(vertex.getName());
        if (groupsList != null) {
            List<VertexGroupInfo> commitList = Lists.newArrayListWithCapacity(groupsList.size());
            for (VertexGroupInfo groupInfo : groupsList) {
                groupInfo.successfulMembers++;
                if (groupInfo.groupMembers.size() == groupInfo.successfulMembers && !groupInfo.outputs.isEmpty()) {
                    // group has outputs and all vertex members are done
                    LOG.info("All members of group: " + groupInfo.groupName + " are succeeded. Commiting outputs");
                    commitList.add(groupInfo);
                }
            }
            for (VertexGroupInfo groupInfo : commitList) {
                if (recoveryData != null && recoveryData.isVertexGroupCommitted(groupInfo.groupName)) {
                    LOG.info("VertexGroup was already committed as per recovery" + " data, groupName=" + groupInfo.groupName);
                    for (String vertexName : groupInfo.groupMembers) {
                        VertexRecoveryData vertexRecoveryData = recoveryData.getVertexRecoveryData(getVertex(vertexName).getVertexId());
                        Preconditions.checkArgument(vertexRecoveryData != null, "Vertex Group has been committed" + ", but no VertexRecoveryData found for its vertex " + vertexName);
                        VertexFinishedEvent vertexFinishedEvent = vertexRecoveryData.getVertexFinishedEvent();
                        Preconditions.checkArgument(vertexFinishedEvent != null, "Vertex Group has been committed" + ", but no VertexFinishedEvent found in its vertex " + vertexName);
                        Preconditions.checkArgument(vertexFinishedEvent.getState() == VertexState.SUCCEEDED, "Vertex Group has been committed, but unexpected vertex state of its vertex " + vertexName + ", vertexstate=" + vertexFinishedEvent.getState());
                    }
                    continue;
                }
                groupInfo.commitStarted = true;
                final Vertex v = getVertex(groupInfo.groupMembers.iterator().next());
                try {
                    Collection<TezVertexID> vertexIds = getVertexIds(groupInfo.groupMembers);
                    appContext.getHistoryHandler().handleCriticalEvent(new DAGHistoryEvent(getID(), new VertexGroupCommitStartedEvent(dagId, groupInfo.groupName, vertexIds, clock.getTime())));
                } catch (IOException e) {
                    LOG.error("Failed to send commit recovery event to handler", e);
                    recoveryFailed = true;
                }
                if (!recoveryFailed) {
                    for (final String outputName : groupInfo.outputs) {
                        OutputKey outputKey = new OutputKey(outputName, groupInfo.groupName, true);
                        CommitCallback groupCommitCallback = new CommitCallback(outputKey);
                        CallableEvent groupCommitCallableEvent = new CallableEvent(groupCommitCallback) {

                            public Void call() throws Exception {
                                OutputCommitter committer = v.getOutputCommitters().get(outputName);
                                LOG.info("Committing output: " + outputName);
                                commitOutput(committer);
                                return null;
                            }
                        };
                        ListenableFuture<Void> groupCommitFuture = appContext.getExecService().submit(groupCommitCallableEvent);
                        Futures.addCallback(groupCommitFuture, groupCommitCallableEvent.getCallback());
                        commitFutures.put(outputKey, groupCommitFuture);
                    }
                }
            }
        }
    }
    if (recoveryFailed) {
        LOG.info("Recovery failure occurred during commit");
        enactKill(DAGTerminationCause.RECOVERY_FAILURE, VertexTerminationCause.COMMIT_FAILURE);
    }
    return !recoveryFailed;
}

Example 3 with OutputCommitter

use of org.apache.tez.runtime.api.OutputCommitter in project tez by apache.

the class TestDAGImpl method testDAGCompletionWithCommitSuccess.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGCompletionWithCommitSuccess() {
    // all vertices completed -> DAG completion and commit
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    // no commit yet
    for (Vertex v : mrrDag.vertices.values()) {
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // dag completion and commit
    Vertex v = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
    Assert.assertEquals(DAGState.SUCCEEDED, mrrDag.getState());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(1, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
}

Example 4 with OutputCommitter

use of org.apache.tez.runtime.api.OutputCommitter in project tez by apache.

the class TestDAGImpl method testDAGErrorAbortNonSuccessfulOutputs.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGErrorAbortNonSuccessfulOutputs() {
    // vertex success -> vertex output commit. failed dag aborts only non-successful vertices
    mrrDag.getConf().setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(1, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // error on vertex -> dag error
    Vertex errorVertex = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(errorVertex.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
    dispatcher.await();
    Assert.assertEquals(VertexState.ERROR, errorVertex.getState());
    dispatcher.await();
    Assert.assertEquals(DAGState.ERROR, mrrDag.getState());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            if (vertex == errorVertex) {
                Assert.assertEquals(1, committer.abortCounter);
                Assert.assertEquals(0, committer.commitCounter);
                Assert.assertEquals(1, committer.initCounter);
                Assert.assertEquals(1, committer.setupCounter);
            } else {
                // abort operation should take no side effort on the successful commit
                Assert.assertEquals(1, committer.abortCounter);
                Assert.assertEquals(1, committer.commitCounter);
                Assert.assertEquals(1, committer.initCounter);
                Assert.assertEquals(1, committer.setupCounter);
            }
        }
    }
}

Example 5 with OutputCommitter

use of org.apache.tez.runtime.api.OutputCommitter in project tez by apache.

the class TestDAGImpl method testDAGErrorAbortAllOutputs.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGErrorAbortAllOutputs() {
    // error on a vertex -> dag error -> all outputs aborted.
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 2; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    // no commit yet
    for (Vertex v : mrrDag.vertices.values()) {
        for (OutputCommitter c : v.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(0, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
    // vertex error -> dag error -> abort all outputs
    Vertex v = mrrDag.getVertex("vertex3");
    dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
    dispatcher.await();
    Assert.assertEquals(VertexState.ERROR, v.getState());
    Assert.assertEquals(DAGState.ERROR, mrrDag.getState());
    for (Vertex vertex : mrrDag.vertices.values()) {
        for (OutputCommitter c : vertex.getOutputCommitters().values()) {
            CountingOutputCommitter committer = (CountingOutputCommitter) c;
            Assert.assertEquals(1, committer.abortCounter);
            Assert.assertEquals(0, committer.commitCounter);
            Assert.assertEquals(1, committer.initCounter);
            Assert.assertEquals(1, committer.setupCounter);
        }
    }
}