Examples with VertexEventTaskCompleted org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted used on opensource projects

Example 56 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testDAGCommitVertexRerunWhileCommitting_OnDAGSuccess.

@Test(timeout = 5000)
public void testDAGCommitVertexRerunWhileCommitting_OnDAGSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true);
    setupDAG(createDAGPlan(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    waitUntil(dag, DAGState.COMMITTING);
    TezTaskID newTaskId = TezTaskID.getInstance(v1.getVertexId(), 1);
    v1.handle(new VertexEventTaskReschedule(newTaskId));
    // dag is in TERMINATING, wait for the complete of its rescheduled tasks
    waitUntil(dag, DAGState.TERMINATING);
    waitUntil(v1, VertexState.TERMINATING);
    // reschedueled task is killed
    v1.handle(new VertexEventTaskCompleted(newTaskId, TaskState.KILLED));
    waitUntil(dag, DAGState.FAILED);
    Assert.assertEquals(VertexState.FAILED, v1.getState());
    Assert.assertEquals(DAGState.FAILED, dag.getState());
    Assert.assertEquals(VertexTerminationCause.VERTEX_RERUN_IN_COMMITTING, v1.getTerminationCause());
    Assert.assertEquals(DAGTerminationCause.VERTEX_RERUN_IN_COMMITTING, dag.getTerminationCause());
    Assert.assertTrue(dag.commitFutures.isEmpty());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    // VertexFinishedEvent is logged twice due to vertex-rerun
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 2);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 1);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    // commit may not have started, so can't verify commitCounter
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}

Example 57 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method _testCommitCanceled_OnDAGSuccess.

// test commit will be canceled no matter it is started or still in the threadpool
// ControlledThreadPoolExecutor is used for to not schedule the commits
private void _testCommitCanceled_OnDAGSuccess(DAGTerminationCause terminationCause) throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true);
    setupDAG(createDAGPlan(true, true));
    // create customized ThreadPoolExecutor to wait before schedule new task
    rawExecutor = new ControlledThreadPoolExecutor(1);
    execService = MoreExecutors.listeningDecorator(rawExecutor);
    doReturn(execService).when(appContext).getExecService();
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    waitUntil(dag, DAGState.COMMITTING);
    // mean the commits have been submitted to ThreadPool
    Assert.assertEquals(2, dag.commitFutures.size());
    dag.handle(new DAGEventTerminateDag(dag.getID(), terminationCause, null));
    waitUntil(dag, terminationCause.getFinishedState());
    Assert.assertEquals(terminationCause, dag.getTerminationCause());
    // mean the commits have been canceled
    Assert.assertTrue(dag.commitFutures.isEmpty());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 1);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    // commit is not started because ControlledThreadPoolExecutor wait before schedule tasks
    Assert.assertEquals(0, v12OutputCommitter.commitCounter);
    Assert.assertEquals(1, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    // commit is not started because ControlledThreadPoolExecutor  wait before schedule tasks
    Assert.assertEquals(0, v3OutputCommitter.commitCounter);
    Assert.assertEquals(1, v3OutputCommitter.abortCounter);
}

Example 58 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestCommit method testDAGCommitSucceeded_OnDAGSuccess.

@Test(timeout = 5000)
public void testDAGCommitSucceeded_OnDAGSuccess() throws Exception {
    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true);
    setupDAG(createDAGPlan(true, true));
    initDAG(dag);
    startDAG(dag);
    VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2");
    VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3");
    // need to make vertices to go to SUCCEEDED
    v1.handle(new VertexEventTaskCompleted(v1.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v2.handle(new VertexEventTaskCompleted(v2.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    v3.handle(new VertexEventTaskCompleted(v3.getTask(0).getTaskId(), TaskState.SUCCEEDED));
    waitUntil(dag, DAGState.COMMITTING);
    CountingOutputCommitter v12OutputCommitter = (CountingOutputCommitter) v1.getOutputCommitter("v12Out");
    v12OutputCommitter.unblockCommit();
    // still in COMMITTING due to another pending commit
    waitUntil(dag, DAGState.COMMITTING);
    CountingOutputCommitter v3OutputCommitter = (CountingOutputCommitter) v3.getOutputCommitter("v3Out");
    v3OutputCommitter.unblockCommit();
    waitUntil(dag, DAGState.SUCCEEDED);
    Assert.assertTrue(dag.commitFutures.isEmpty());
    Assert.assertNull(dag.getTerminationCause());
    historyEventHandler.verifyVertexGroupCommitStartedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("uv12", 0);
    historyEventHandler.verifyVertexGroupCommitStartedEvent("v1", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("v1", 0);
    historyEventHandler.verifyVertexGroupCommitStartedEvent("v3", 0);
    historyEventHandler.verifyVertexGroupCommitFinishedEvent("v3", 0);
    historyEventHandler.verifyVertexCommitStartedEvent(v1.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v1.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v2.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v2.getVertexId(), 1);
    historyEventHandler.verifyVertexCommitStartedEvent(v3.getVertexId(), 0);
    historyEventHandler.verifyVertexFinishedEvent(v3.getVertexId(), 1);
    historyEventHandler.verifyDAGCommitStartedEvent(dag.getID(), 1);
    historyEventHandler.verifyDAGFinishedEvent(dag.getID(), 1);
    Assert.assertEquals(1, v12OutputCommitter.initCounter);
    Assert.assertEquals(1, v12OutputCommitter.setupCounter);
    Assert.assertEquals(1, v12OutputCommitter.commitCounter);
    Assert.assertEquals(0, v12OutputCommitter.abortCounter);
    Assert.assertEquals(1, v3OutputCommitter.initCounter);
    Assert.assertEquals(1, v3OutputCommitter.setupCounter);
    Assert.assertEquals(1, v3OutputCommitter.commitCounter);
    Assert.assertEquals(0, v3OutputCommitter.abortCounter);
}

Example 59 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestDAGImpl method testCounterLimits.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testCounterLimits() {
    initDAG(mrrDag);
    dispatcher.await();
    startDAG(mrrDag);
    dispatcher.await();
    for (int i = 0; i < 3; ++i) {
        Vertex v = mrrDag.getVertex("vertex" + (i + 1));
        TezCounters ctrs = new TezCounters();
        for (int j = 0; j < 50; ++j) {
            ctrs.findCounter("g", "c" + i + "_" + j).increment(1);
        }
        ((VertexImpl) v).setCounters(ctrs);
        dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
        dispatcher.await();
        Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
        Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
    }
    Assert.assertEquals(3, mrrDag.getSuccessfulVertices());
    Assert.assertEquals(DAGState.FAILED, mrrDag.getState());
    Assert.assertTrue("Diagnostics should contain counter limits error message", StringUtils.join(mrrDag.getDiagnostics(), ",").contains("Counters limit exceeded"));
}

Example 60 with VertexEventTaskCompleted

use of org.apache.tez.dag.app.dag.event.VertexEventTaskCompleted in project tez by apache.

the class TestDAGImpl method testVertexCompletion.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testVertexCompletion() {
    initDAG(dag);
    Assert.assertTrue(0.0f == dag.getCompletedTaskProgress());
    startDAG(dag);
    Assert.assertTrue(0.0f == dag.getCompletedTaskProgress());
    dispatcher.await();
    TezVertexID vId = TezVertexID.getInstance(dagId, 1);
    Vertex v = dag.getVertex(vId);
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 0), TaskState.SUCCEEDED));
    dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(vId, 1), TaskState.SUCCEEDED));
    dispatcher.await();
    Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
    Assert.assertEquals(1, dag.getSuccessfulVertices());
    // 2 tasks completed, total plan has 11 vertices
    Assert.assertEquals((float) 2 / 11, dag.getCompletedTaskProgress(), 0.05);
}