use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TestDAGImpl method _testDAGTerminate.
// Couple of vertices succeed. DAG_KILLED processed, which causes the rest of the vertices to be
// marked as KILLED.
@SuppressWarnings("unchecked")
private void _testDAGTerminate(DAGTerminationCause terminationCause) {
initDAG(dag);
startDAG(dag);
dispatcher.await();
dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, 0), VertexState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(DAGState.RUNNING, dag.getState());
dispatcher.getEventHandler().handle(new DAGEventVertexCompleted(TezVertexID.getInstance(dagId, 1), VertexState.SUCCEEDED));
dispatcher.getEventHandler().handle(new DAGEventTerminateDag(dagId, terminationCause, null));
dispatcher.await();
Assert.assertEquals(terminationCause.getFinishedState(), dag.getState());
Assert.assertEquals(terminationCause, dag.getTerminationCause());
Assert.assertEquals(2, dag.getSuccessfulVertices());
int killedCount = 0;
for (Map.Entry<TezVertexID, Vertex> vEntry : dag.getVertices().entrySet()) {
if (vEntry.getValue().getState() == VertexState.KILLED) {
killedCount++;
}
}
Assert.assertEquals(4, killedCount);
for (Vertex v : dag.getVertices().values()) {
Assert.assertEquals(VertexTerminationCause.DAG_TERMINATED, v.getTerminationCause());
}
Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents);
}
use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TestDAGImpl method testDAGErrorAbortNonSuccessfulOutputs.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDAGErrorAbortNonSuccessfulOutputs() {
// vertex success -> vertex output commit. failed dag aborts only non-successful vertices
mrrDag.getConf().setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
initDAG(mrrDag);
dispatcher.await();
startDAG(mrrDag);
dispatcher.await();
for (int i = 0; i < 2; ++i) {
Vertex v = mrrDag.getVertex("vertex" + (i + 1));
dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices());
for (OutputCommitter c : v.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
Assert.assertEquals(0, committer.abortCounter);
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
// error on vertex -> dag error
Vertex errorVertex = mrrDag.getVertex("vertex3");
dispatcher.getEventHandler().handle(new VertexEvent(errorVertex.getVertexId(), VertexEventType.V_INTERNAL_ERROR));
dispatcher.await();
Assert.assertEquals(VertexState.ERROR, errorVertex.getState());
dispatcher.await();
Assert.assertEquals(DAGState.ERROR, mrrDag.getState());
for (Vertex vertex : mrrDag.vertices.values()) {
for (OutputCommitter c : vertex.getOutputCommitters().values()) {
CountingOutputCommitter committer = (CountingOutputCommitter) c;
if (vertex == errorVertex) {
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(0, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
} else {
// abort operation should take no side effort on the successful commit
Assert.assertEquals(1, committer.abortCounter);
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(1, committer.initCounter);
Assert.assertEquals(1, committer.setupCounter);
}
}
}
}
use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TestAMWebController method getAttemptsTestHelper.
Map<String, Object> getAttemptsTestHelper(List<TaskAttempt> attempts, List<List<Integer>> attemptMinIds, List<Integer> vertexMinIds, List<Integer> taskMinIds, Integer limit) {
// Creating mock DAG
DAG mockDAG = mock(DAG.class);
doReturn(TezDAGID.fromString("dag_1441301219877_0109_1")).when(mockDAG).getID();
// Creating mock vertex and attaching to mock DAG
TezVertexID vertexID = TezVertexID.fromString("vertex_1441301219877_0109_1_00");
Vertex mockVertex = mock(Vertex.class);
doReturn(vertexID).when(mockVertex).getVertexId();
doReturn(mockVertex).when(mockDAG).getVertex(vertexID);
doReturn(ImmutableMap.of(vertexID, mockVertex)).when(mockDAG).getVertices();
// Creating mock task and attaching to mock Vertex
TezTaskID taskID = TezTaskID.fromString("task_1441301219877_0109_1_00_000000");
Task mockTask = mock(Task.class);
doReturn(taskID).when(mockTask).getTaskId();
int taskIndex = taskID.getId();
doReturn(mockTask).when(mockVertex).getTask(taskIndex);
doReturn(ImmutableMap.of(taskID, mockTask)).when(mockVertex).getTasks();
// Creating mock tasks and attaching to mock vertex
Map<TezTaskAttemptID, TaskAttempt> attemptsMap = Maps.newHashMap();
for (TaskAttempt attempt : attempts) {
TezTaskAttemptID attemptId = attempt.getID();
doReturn(attempt).when(mockTask).getAttempt(attemptId);
attemptsMap.put(attemptId, attempt);
}
doReturn(attemptsMap).when(mockTask).getAttempts();
// Creates & setup controller spy
AMWebController amWebController = new AMWebController(mockRequestContext, mockAppContext, "TEST_HISTORY_URL");
AMWebController spy = spy(amWebController);
doReturn(true).when(spy).setupResponse();
doNothing().when(spy).renderJSON(any());
// Set mock query params
doReturn(limit).when(spy).getQueryParamInt(WebUIService.LIMIT);
doReturn(vertexMinIds).when(spy).getIntegersFromRequest(WebUIService.VERTEX_ID, limit);
doReturn(taskMinIds).when(spy).getIDsFromRequest(WebUIService.TASK_ID, limit, 2);
doReturn(attemptMinIds).when(spy).getIDsFromRequest(WebUIService.ATTEMPT_ID, limit, 3);
// Set function mocks
doReturn(mockDAG).when(spy).checkAndGetDAGFromRequest();
Map<String, Set<String>> counterList = new TreeMap<String, Set<String>>();
doReturn(counterList).when(spy).getCounterListFromRequest();
spy.getAttemptsInfo();
verify(spy).renderJSON(returnResultCaptor.capture());
return returnResultCaptor.getValue();
}
use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TestAMWebController method getTasksTestHelper.
Map<String, Object> getTasksTestHelper(List<Task> tasks, List<List<Integer>> taskMinIds, List<Integer> vertexMinIds, Integer limit) {
// Creating mock DAG
DAG mockDAG = mock(DAG.class);
doReturn(TezDAGID.fromString("dag_1441301219877_0109_1")).when(mockDAG).getID();
// Creating mock vertex and attaching to mock DAG
TezVertexID vertexID = TezVertexID.fromString("vertex_1441301219877_0109_1_00");
Vertex mockVertex = mock(Vertex.class);
doReturn(vertexID).when(mockVertex).getVertexId();
doReturn(mockVertex).when(mockDAG).getVertex(vertexID);
doReturn(ImmutableMap.of(vertexID, mockVertex)).when(mockDAG).getVertices();
// Creating mock tasks and attaching to mock vertex
Map<TezTaskID, Task> taskMap = Maps.newHashMap();
for (Task task : tasks) {
TezTaskID taskId = task.getTaskId();
int taskIndex = taskId.getId();
doReturn(task).when(mockVertex).getTask(taskIndex);
taskMap.put(taskId, task);
}
doReturn(taskMap).when(mockVertex).getTasks();
// Creates & setup controller spy
AMWebController amWebController = new AMWebController(mockRequestContext, mockAppContext, "TEST_HISTORY_URL");
AMWebController spy = spy(amWebController);
doReturn(true).when(spy).setupResponse();
doNothing().when(spy).renderJSON(any());
// Set mock query params
doReturn(limit).when(spy).getQueryParamInt(WebUIService.LIMIT);
doReturn(vertexMinIds).when(spy).getIntegersFromRequest(WebUIService.VERTEX_ID, limit);
doReturn(taskMinIds).when(spy).getIDsFromRequest(WebUIService.TASK_ID, limit, 2);
// Set function mocks
doReturn(mockDAG).when(spy).checkAndGetDAGFromRequest();
Map<String, Set<String>> counterList = new TreeMap<String, Set<String>>();
doReturn(counterList).when(spy).getCounterListFromRequest();
spy.getTasksInfo();
verify(spy).renderJSON(returnResultCaptor.capture());
return returnResultCaptor.getValue();
}
use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.
the class TestAMWebController method testGetVerticesInfoGetPartial.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testGetVerticesInfoGetPartial() {
Vertex mockVertex1 = createMockVertex("vertex_1422960590892_0007_42_00", VertexState.RUNNING, 0.33f, 3);
Vertex mockVertex2 = createMockVertex("vertex_1422960590892_0007_42_01", VertexState.SUCCEEDED, 1.0f, 5);
final Map<String, Object> result = getVerticesTestHelper(1, mockVertex1, mockVertex2);
Assert.assertEquals(1, result.size());
Assert.assertTrue(result.containsKey("vertices"));
List<Map<String, String>> verticesInfo = (List<Map<String, String>>) result.get("vertices");
Assert.assertEquals(1, verticesInfo.size());
Map<String, String> vertex1Result = verticesInfo.get(0);
verifySingleVertexResult(mockVertex1, vertex1Result);
}
Aggregations