Examples with DAG org.apache.tez.dag.app.dag.DAG used on opensource projects

Example 26 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestTaskSchedulerManager method testReportFailureFromTaskScheduler.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskScheduler() {
    String dagName = DAG_NAME;
    Configuration conf = new TezConfiguration();
    String taskSchedulerName = "testTaskScheduler";
    String expIdentifier = "[0:" + taskSchedulerName + "]";
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn(taskSchedulerName).when(appContext).getTaskSchedulerName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(conf).when(appContext).getAMConf();
    InetSocketAddress address = new InetSocketAddress("host", 55000);
    DAGClientServer dagClientServer = mock(DAGClientServer.class);
    doReturn(address).when(dagClientServer).getBindAddress();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskSchedulerDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>(taskSchedulerName, TaskSchedulerForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, dagClientServer, eventHandler, mock(ContainerSignatureMatcher.class), mock(WebUIService.class), list, false, new HadoopShimsLoader(appContext.getAMConf()).getHadoopShim()) {

        @Override
        TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) {
            // Avoid wrapping in threads
            return rawContext;
        }
    };
    try {
        taskSchedulerManager.init(new TezConfiguration());
        taskSchedulerManager.start();
        taskSchedulerManager.getTotalResources(0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier));
        reset(eventHandler);
        taskSchedulerManager.getAvailableResources(0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains(expIdentifier));
    } finally {
        taskSchedulerManager.stop();
    }
}

Example 27 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestDAGSchedulerNaturalOrderControlled method testSimpleFlow.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testSimpleFlow() {
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = createMockDag();
    DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
    int numVertices = 5;
    Vertex[] vertices = new Vertex[numVertices];
    for (int i = 0; i < numVertices; i++) {
        vertices[i] = dag.getVertex("vertex" + i);
    }
    // Schedule all tasks belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[0].getTotalTasks())).handle(any(Event.class));
    reset(eventHandler);
    // Schedule 3 tasks belonging to v2
    for (int i = 0; i < 3; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    verify(eventHandler, times(3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule 3 tasks belonging to v3
    for (int i = 0; i < 3; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
    }
    verify(eventHandler, times(3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule remaining tasks belonging to v2
    for (int i = 3; i < vertices[2].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[2].getTotalTasks() - 3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule remaining tasks belonging to v3
    for (int i = 3; i < vertices[3].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[3].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[3].getTotalTasks() - 3)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule all tasks belonging to v4
    for (int i = 0; i < vertices[4].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[4].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[4].getTotalTasks())).handle(any(Event.class));
    reset(eventHandler);
}

Example 28 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestDAGSchedulerNaturalOrderControlled method testMultipleRequestsForSameTask.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testMultipleRequestsForSameTask() {
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = createMockDag();
    DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
    int numVertices = 5;
    Vertex[] vertices = new Vertex[numVertices];
    for (int i = 0; i < numVertices; i++) {
        vertices[i] = dag.getVertex("vertex" + i);
    }
    // Schedule all but 1 task belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule all tasks belonging to v2
    for (int i = 0; i < vertices[2].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    // Nothing should be scheduled
    verify(eventHandler, never()).handle(any(Event.class));
    reset(eventHandler);
    // Schedule an extra attempt for all but 1 task belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 1));
    }
    // Only v0 requests should have gone out
    verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule last task of v0, with attempt 1
    dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), vertices[0].getTotalTasks() - 1, 1));
    // One v0 request and all of v2 should have gone out
    verify(eventHandler, times(1 + vertices[2].getTotalTasks())).handle(any(Event.class));
}

Example 29 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestDAGScheduler method testConcurrencyLimitWithKilledNonRunningTask.

@Test(timeout = 5000)
public void testConcurrencyLimitWithKilledNonRunningTask() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    when(mockDag.getTotalVertices()).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezTaskID tId0 = TezTaskID.getInstance(vId0, 0);
    TaskAttempt mockAttempt;
    Vertex mockVertex = mock(Vertex.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0);
    when(mockVertex.getVertexId()).thenReturn(vId0);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    List<TaskAttempt> mockAttempts = Lists.newArrayList();
    int completed = 0;
    int requested = 0;
    int scheduled = 0;
    // effective
    scheduler.addVertexConcurrencyLimit(vId0, 1);
    // schedule beyond limit and it gets buffered
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(1)));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(0).getID(), // matches order
    mockEventHandler.events.get(0).getTaskAttemptID());
}

Example 30 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestDAGScheduler method testDAGSchedulerNaturalOrder.

@Test(timeout = 5000)
public void testDAGSchedulerNaturalOrder() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    Vertex mockVertex = mock(Vertex.class);
    TaskAttempt mockAttempt = mock(TaskAttempt.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockDag.getTotalVertices()).thenReturn(4);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0).thenReturn(1).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01");
    TezVertexID vId2 = TezVertexID.fromString("vertex_1436907267600_195589_1_02");
    TezVertexID vId3 = TezVertexID.fromString("vertex_1436907267600_195589_1_03");
    when(mockVertex.getVertexId()).thenReturn(vId0).thenReturn(vId1).thenReturn(vId2).thenReturn(vId3);
    DAGEventSchedulerUpdate event = new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(10, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(12, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(25, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(27, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(40, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(42, mockEventHandler.event.getPriorityLowLimit());
    scheduler.scheduleTaskEx(event);
    Assert.assertEquals(43, mockEventHandler.event.getPriorityHighLimit());
    Assert.assertEquals(45, mockEventHandler.event.getPriorityLowLimit());
}