use of org.apache.tez.dag.app.dag.DAG in project tez by apache.
the class TaskCommunicatorManager method canCommit.
/**
* Child checking whether it can commit.
* <p/>
* <br/>
* Repeatedly polls the ApplicationMaster whether it
* {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
* centralized commit protocol handling by the JobTracker.
*/
// @Override
public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
// An attempt is asking if it can commit its output. This can be decided
// only by the task which is managing the multiple attempts. So redirect the
// request there.
taskHeartbeatHandler.progressing(taskAttemptId);
pingContainerHeartbeatHandler(taskAttemptId);
DAG job = context.getCurrentDAG();
Task task = job.getVertex(taskAttemptId.getTaskID().getVertexID()).getTask(taskAttemptId.getTaskID());
return task.canCommit(taskAttemptId);
}
use of org.apache.tez.dag.app.dag.DAG in project tez by apache.
the class TestVertexImpl method setupPostDagCreation.
@SuppressWarnings({ "unchecked", "rawtypes" })
public void setupPostDagCreation() throws TezException {
String dagName = "dag0";
// dispatcher may be created multiple times (setupPostDagCreation may be called multiples)
if (dispatcher != null) {
dispatcher.stop();
}
dispatcher = new DrainDispatcher();
appContext = mock(AppContext.class);
when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim());
when(appContext.getContainerLauncherName(anyInt())).thenReturn(TezConstants.getTezYarnServicePluginName());
thh = mock(TaskHeartbeatHandler.class);
historyEventHandler = mock(HistoryEventHandler.class);
TaskSchedulerManager taskScheduler = mock(TaskSchedulerManager.class);
UserGroupInformation ugi;
try {
ugi = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new RuntimeException(e);
}
DAG dag = mock(DAG.class);
doReturn(ugi).when(dag).getDagUGI();
doReturn(dagName).when(dag).getName();
Map<String, LocalResource> localResources = new HashMap<>();
for (PlanLocalResource planLR : dagPlan.getLocalResourceList()) {
localResources.put(planLR.getName(), DagTypeConverters.convertPlanLocalResourceToLocalResource(planLR));
}
when(dag.getLocalResources()).thenReturn(localResources);
doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
doReturn(dag).when(appContext).getCurrentDAG();
execService = mock(ListeningExecutorService.class);
final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class);
Mockito.doAnswer(new Answer() {
public ListenableFuture<Void> answer(InvocationOnMock invocation) {
Object[] args = invocation.getArguments();
CallableEvent e = (CallableEvent) args[0];
dispatcher.getEventHandler().handle(e);
return mockFuture;
}
}).when(execService).submit((Callable<Void>) any());
MockClock clock = new MockClock();
doReturn(execService).when(appContext).getExecService();
doReturn(conf).when(appContext).getAMConf();
doReturn(new Credentials()).when(dag).getCredentials();
doReturn(DAGPlan.getDefaultInstance()).when(dag).getJobPlan();
doReturn(dagId).when(appContext).getCurrentDAGID();
doReturn(dagId).when(dag).getID();
doReturn(taskScheduler).when(appContext).getTaskScheduler();
doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources(0);
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler();
doReturn(clock).when(appContext).getClock();
vertexGroups = Maps.newHashMap();
for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) {
vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo));
}
// updateTracker may be created multiple times (setupPostDagCreation may be called multiples)
if (updateTracker != null) {
updateTracker.stop();
}
updateTracker = new StateChangeNotifierForTest(appContext.getCurrentDAG());
setupVertices();
when(dag.getVertex(any(TezVertexID.class))).thenAnswer(new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
if (args.length != 1) {
return null;
}
TezVertexID vId = (TezVertexID) args[0];
return vertexIdMap.get(vId);
}
});
when(dag.getVertex(any(String.class))).thenAnswer(new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
if (args.length != 1) {
return null;
}
String vId = (String) args[0];
return vertices.get(vId);
}
});
// TODO - this test logic is tightly linked to impl DAGImpl code.
edges = new HashMap<String, Edge>();
for (EdgePlan edgePlan : dagPlan.getEdgeList()) {
EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(edgePlan);
edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler(), conf));
}
parseVertexEdges();
for (Edge edge : edges.values()) {
edge.initialize();
}
dispatcher.register(CallableEventType.class, new CallableEventDispatcher());
taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
taskEventDispatcher = new TaskEventDispatcher();
dispatcher.register(TaskEventType.class, taskEventDispatcher);
vertexEventDispatcher = new VertexEventDispatcher();
dispatcher.register(VertexEventType.class, vertexEventDispatcher);
dagEventDispatcher = new DagEventDispatcher();
dispatcher.register(DAGEventType.class, dagEventDispatcher);
amSchedulerEventDispatcher = new AMSchedulerEventDispatcher();
dispatcher.register(AMSchedulerEventType.class, amSchedulerEventDispatcher);
dispatcher.init(conf);
dispatcher.start();
}
use of org.apache.tez.dag.app.dag.DAG in project tez by apache.
the class TestDAGSchedulerNaturalOrderControlled method createMockDag.
// Test parallelism updated form -1
// Reduce parallelism
// Different attempts scheduled for a single task.
private DAG createMockDag() {
DAG dag = mock(DAG.class);
/*
v0 v1
\ /
\ /
v2 v3
\ /
\ /
\ /
v4
v0 - Root
v1 - Root with 0 tasks.
v2 - can simulate AutoReduce. Parallelism goes down. Slow schedule.
v3 - can simulate ImmediateStart
v4 - Simulate one shuffle input, one broadcast input.
*/
int numVertices = 5;
Vertex[] vertices = new Vertex[numVertices];
vertices[0] = createMockVertex("vertex0", 0, 10, 0);
vertices[1] = createMockVertex("vertex1", 1, 0, 0);
vertices[2] = createMockVertex("vertex2", 2, 10, 1);
vertices[3] = createMockVertex("vertex3", 3, 10, 1);
vertices[4] = createMockVertex("vertex4", 4, 10, 2);
for (int i = 0; i < numVertices; i++) {
String name = vertices[i].getName();
TezVertexID vertexId = vertices[i].getVertexId();
doReturn(vertices[i]).when(dag).getVertex(name);
doReturn(vertices[i]).when(dag).getVertex(vertexId);
}
updateMockVertexWithConnections(vertices[0], createConnectionMap((Vertex[]) null), createConnectionMap(vertices[2]));
updateMockVertexWithConnections(vertices[1], createConnectionMap((Vertex[]) null), createConnectionMap(vertices[3]));
updateMockVertexWithConnections(vertices[2], createConnectionMap(vertices[0]), createConnectionMap(vertices[4]));
updateMockVertexWithConnections(vertices[3], createConnectionMap(vertices[1]), createConnectionMap(vertices[4]));
updateMockVertexWithConnections(vertices[4], createConnectionMap(vertices[2], vertices[3]), createConnectionMap((Vertex[]) null));
return dag;
}
use of org.apache.tez.dag.app.dag.DAG in project tez by apache.
the class TestDAGSchedulerNaturalOrderControlled method testParallelismUpdated.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testParallelismUpdated() {
EventHandler eventHandler = mock(EventHandler.class);
DAG dag = createMockDag();
DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
int numVertices = 5;
Vertex[] vertices = new Vertex[numVertices];
for (int i = 0; i < numVertices; i++) {
vertices[i] = dag.getVertex("vertex" + i);
}
// Schedule all tasks belonging to v0
for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
}
verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
reset(eventHandler);
assertEquals(10, vertices[2].getTotalTasks());
// Schedule all tasks belonging to v3
for (int i = 0; i < vertices[3].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
}
verify(eventHandler, times(vertices[3].getTotalTasks())).handle(any(Event.class));
reset(eventHandler);
// Schedule all tasks belonging to v4
for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[4].getVertexId(), i, 0));
}
verify(eventHandler, never()).handle(any(Event.class));
reset(eventHandler);
// Reset the parallelism for v2.
updateParallelismOnMockVertex(vertices[2], 3);
assertEquals(3, vertices[2].getTotalTasks());
// Schedule all tasks belonging to v2
for (int i = 0; i < vertices[2].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
}
verify(eventHandler, times(vertices[2].getTotalTasks() + vertices[4].getTotalTasks())).handle(any(Event.class));
reset(eventHandler);
}
use of org.apache.tez.dag.app.dag.DAG in project tez by apache.
the class TestDAGSchedulerNaturalOrderControlled method testSourceRequestDelayed.
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testSourceRequestDelayed() {
// ShuffleVertexHandler - slowstart simulation
EventHandler eventHandler = mock(EventHandler.class);
DAG dag = createMockDag();
DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
int numVertices = 5;
Vertex[] vertices = new Vertex[numVertices];
for (int i = 0; i < numVertices; i++) {
vertices[i] = dag.getVertex("vertex" + i);
}
// Schedule all tasks belonging to v0
for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
}
verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
reset(eventHandler);
// Schedule all tasks belonging to v3.
for (int i = 0; i < vertices[3].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
}
verify(eventHandler, times(vertices[3].getTotalTasks())).handle(any(Event.class));
reset(eventHandler);
// Scheduling all tasks belonging to v4. None should get scheduled.
for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[4].getVertexId(), i, 0));
}
verify(eventHandler, never()).handle(any(Event.class));
reset(eventHandler);
// Schedule 3 tasks for v2 initially.
for (int i = 0; i < 3; i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
}
verify(eventHandler, times(3)).handle(any(Event.class));
reset(eventHandler);
// Schedule remaining tasks belonging to v2
for (int i = 3; i < vertices[2].getTotalTasks(); i++) {
dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
}
ArgumentCaptor<Event> args = ArgumentCaptor.forClass(Event.class);
// All of v2 and v3 should be sent out.
verify(eventHandler, times(vertices[2].getTotalTasks() - 3 + vertices[4].getTotalTasks())).handle(args.capture());
int count = 0;
// Verify the order in which the events were sent out.
for (Event raw : args.getAllValues()) {
TaskAttemptEventSchedule event = (TaskAttemptEventSchedule) raw;
if (count < vertices[2].getTotalTasks() - 3) {
assertEquals(2, event.getTaskAttemptID().getTaskID().getVertexID().getId());
} else {
assertEquals(4, event.getTaskAttemptID().getTaskID().getVertexID().getId());
}
count++;
}
reset(eventHandler);
}
Aggregations