Examples with DAG org.apache.tez.dag.app.dag.DAG used on opensource projects

Example 6 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestDAGScheduler method testConcurrencyLimit.

@Test(timeout = 5000)
public void testConcurrencyLimit() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    when(mockDag.getTotalVertices()).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01");
    TezTaskID tId0 = TezTaskID.getInstance(vId0, 0);
    TezTaskID tId1 = TezTaskID.getInstance(vId1, 0);
    TaskAttempt mockAttempt;
    Vertex mockVertex = mock(Vertex.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0);
    when(mockVertex.getVertexId()).thenReturn(vId0);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    // not effective
    scheduler.addVertexConcurrencyLimit(vId0, 0);
    // schedule beyond limit and it gets scheduled
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 0));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(1, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 1));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(2, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 2));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(3, mockEventHandler.events.size());
    mockEventHandler.events.clear();
    List<TaskAttempt> mockAttempts = Lists.newArrayList();
    int completed = 0;
    int requested = 0;
    int scheduled = 0;
    // effective
    scheduler.addVertexConcurrencyLimit(vId1, 2);
    // schedule beyond limit and it gets buffered
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // no extra scheduling
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
}

Example 7 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestVertexImpl2 method createDefaultMockAppContext.

private static AppContext createDefaultMockAppContext() {
    AppContext appContext = mock(AppContext.class);
    DAG mockDag = mock(DAG.class);
    doReturn(new Credentials()).when(mockDag).getCredentials();
    doReturn(mockDag).when(appContext).getCurrentDAG();
    return appContext;
}

Example 8 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestContainerLauncherManager method testReportFailureFromContainerLauncher.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromContainerLauncher() throws ServicePluginException, TezException {
    final String dagName = DAG_NAME;
    final int dagIndex = DAG_INDEX;
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(0, 0), dagIndex);
    DAG dag = mock(DAG.class);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn("testlauncher").when(appContext).getContainerLauncherName(0);
    NamedEntityDescriptor<TaskCommunicatorDescriptor> taskCommDescriptor = new NamedEntityDescriptor<>("testlauncher", ContainerLauncherForTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(taskCommDescriptor);
    ContainerLauncherManager containerLauncherManager = new ContainerLauncherManager(appContext, mock(TaskCommunicatorManagerInterface.class), "", list, false);
    try {
        ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class);
        Container container1 = mock(Container.class);
        ContainerLauncherLaunchRequestEvent launchRequestEvent = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0);
        containerLauncherManager.handle(launchRequestEvent);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains("[0:testlauncher]"));
        reset(eventHandler);
        // stop container
        ContainerId containerId2 = mock(ContainerId.class);
        NodeId nodeId2 = mock(NodeId.class);
        ContainerLauncherStopRequestEvent stopRequestEvent = new ContainerLauncherStopRequestEvent(containerId2, nodeId2, null, 0, 0, 0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        containerLauncherManager.handle(stopRequestEvent);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains("[0:testlauncher]"));
    } finally {
        containerLauncherManager.stop();
    }
}

Example 9 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestAMNodeTracker method testNodeCompletedAndCleanup.

@Test(timeout = 10000L)
public void testNodeCompletedAndCleanup() {
    AppContext appContext = mock(AppContext.class);
    Configuration conf = new Configuration(false);
    conf.setInt(TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, 2);
    TestEventHandler handler = new TestEventHandler();
    AMNodeTracker amNodeTracker = new AMNodeTracker(handler, appContext);
    doReturn(amNodeTracker).when(appContext).getNodeTracker();
    AMContainerMap amContainerMap = mock(AMContainerMap.class);
    TaskSchedulerManager taskSchedulerManager = mock(TaskSchedulerManager.class);
    dispatcher.register(AMNodeEventType.class, amNodeTracker);
    dispatcher.register(AMContainerEventType.class, amContainerMap);
    dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager);
    amNodeTracker.init(conf);
    amNodeTracker.start();
    try {
        NodeId nodeId = NodeId.newInstance("fakenode", 3333);
        amNodeTracker.nodeSeen(nodeId, 0);
        AMNode amNode = amNodeTracker.get(nodeId, 0);
        ContainerId[] containerIds = new ContainerId[7];
        // Start 5 containers.
        for (int i = 0; i < 5; i++) {
            containerIds[i] = mock(ContainerId.class);
            amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, 0, containerIds[i]));
        }
        assertEquals(5, amNode.getContainers().size());
        // Finnish 1st dag
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(5, amNode.getContainers().size());
        // Mark 2 as complete. Finish 2nd dag.
        for (int i = 0; i < 2; i++) {
            amNodeTracker.handle(new AMNodeEventContainerCompleted(nodeId, 0, containerIds[i]));
        }
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(3, amNode.getContainers().size());
        // Add 2 more containers. Mark all as complete. Finish 3rd dag.
        for (int i = 5; i < 7; i++) {
            containerIds[i] = mock(ContainerId.class);
            amNodeTracker.handle(new AMNodeEventContainerAllocated(nodeId, 0, containerIds[i]));
        }
        assertEquals(5, amNode.getContainers().size());
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(5, amNode.getContainers().size());
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(5, amNode.getContainers().size());
        for (int i = 2; i < 7; i++) {
            amNodeTracker.handle(new AMNodeEventContainerCompleted(nodeId, 0, containerIds[i]));
        }
        assertEquals(5, amNode.getContainers().size());
        amNodeTracker.dagComplete(mock(DAG.class));
        assertEquals(0, amNode.getContainers().size());
    } finally {
        amNodeTracker.stop();
    }
}

Example 10 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestAMWebController method testDagProgressResponse.

@Test(timeout = 5000)
public void testDagProgressResponse() {
    AMWebController amWebController = new AMWebController(mockRequestContext, mockAppContext, "TEST_HISTORY_URL");
    AMWebController spy = spy(amWebController);
    DAG mockDAG = mock(DAG.class);
    doReturn(true).when(spy).hasAccess();
    doNothing().when(spy).setCorsHeaders();
    doReturn("42").when(spy).$(WebUIService.DAG_ID);
    doReturn(mockResponse).when(spy).response();
    doReturn(TezDAGID.fromString("dag_1422960590892_0007_42")).when(mockDAG).getID();
    doReturn(66.0f).when(mockDAG).getCompletedTaskProgress();
    doReturn(mockDAG).when(mockAppContext).getCurrentDAG();
    doNothing().when(spy).renderJSON(any());
    spy.getDagProgress();
    verify(spy).renderJSON(singleResultCaptor.capture());
    final Map<String, AMWebController.ProgressInfo> result = singleResultCaptor.getValue();
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.containsKey("dagProgress"));
    AMWebController.ProgressInfo progressInfo = result.get("dagProgress");
    Assert.assertTrue("dag_1422960590892_0007_42".equals(progressInfo.getId()));
    Assert.assertEquals(66.0, progressInfo.getProgress(), 0.1);
}