Examples with DAG org.apache.tez.dag.app.dag.DAG used on opensource projects

Example 21 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class AMWebController method getDagProgress.

public void getDagProgress() {
    setCorsHeaders();
    if (!hasAccess()) {
        sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED, "Access denied for user: " + request().getRemoteUser(), null);
        return;
    }
    int dagID;
    try {
        dagID = getQueryParamInt(WebUIService.DAG_ID);
    } catch (NumberFormatException e) {
        sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag id:", e);
        return;
    }
    DAG currentDAG = appContext.getCurrentDAG();
    if (currentDAG == null || dagID != currentDAG.getID().getId()) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
        return;
    }
    Map<String, ProgressInfo> result = new HashMap<String, ProgressInfo>();
    result.put(DAG_PROGRESS, new ProgressInfo(currentDAG.getID().toString(), currentDAG.getCompletedTaskProgress()));
    renderJSON(result);
}

Example 22 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class AMWebController method getDagInfo.

public void getDagInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    Map<String, Object> dagInfo = new HashMap<String, Object>();
    dagInfo.put("id", dag.getID().toString());
    dagInfo.put("progress", Float.toString(dag.getCompletedTaskProgress()));
    dagInfo.put("status", dag.getState().toString());
    try {
        if (counterNames != null && !counterNames.isEmpty()) {
            TezCounters counters = dag.getCachedCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                dagInfo.put("counters", counterMap);
            }
        }
    } catch (LimitExceededException e) {
    // Ignore
    // TODO: add an error message instead for counter key
    }
    renderJSON(ImmutableMap.of("dag", dagInfo));
}

Example 23 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class AMWebController method getVertexProgress.

public void getVertexProgress() {
    int dagID;
    int vertexID;
    setCorsHeaders();
    if (!hasAccess()) {
        sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED, "Access denied for user: " + request().getRemoteUser(), null);
        return;
    }
    try {
        dagID = getQueryParamInt(WebUIService.DAG_ID);
        vertexID = getQueryParamInt(WebUIService.VERTEX_ID);
    } catch (NumberFormatException e) {
        sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag or vertex id", e);
        return;
    }
    DAG currentDAG = appContext.getCurrentDAG();
    if (currentDAG == null || currentDAG.getID().getId() != dagID) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
        return;
    }
    final TezVertexID tezVertexID = TezVertexID.getInstance(currentDAG.getID(), vertexID);
    Vertex vertex = currentDAG.getVertex(tezVertexID);
    if (vertex == null) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "vertex not found: " + vertexID, null);
        return;
    }
    Map<String, ProgressInfo> result = new HashMap<String, ProgressInfo>();
    result.put(VERTEX_PROGRESS, new ProgressInfo(tezVertexID.toString(), vertex.getCompletedTaskProgress()));
    renderJSON(result);
}

Example 24 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestTaskCommunicatorManager method testReportFailureFromTaskCommunicator.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testReportFailureFromTaskCommunicator() throws TezException {
    String dagName = DAG_NAME;
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
    doReturn(eventHandler).when(appContext).getEventHandler();
    DAG dag = mock(DAG.class);
    TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(1, 0), DAG_INDEX);
    doReturn(dagName).when(dag).getName();
    doReturn(dagId).when(dag).getID();
    doReturn(dag).when(appContext).getCurrentDAG();
    NamedEntityDescriptor<TaskCommunicatorDescriptor> namedEntityDescriptor = new NamedEntityDescriptor<>("testTaskCommunicator", TaskCommForFailureTest.class.getName());
    List<NamedEntityDescriptor> list = new LinkedList<>();
    list.add(namedEntityDescriptor);
    TaskCommunicatorManager taskCommManager = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), list);
    try {
        taskCommManager.init(new Configuration());
        taskCommManager.start();
        taskCommManager.registerRunningContainer(mock(ContainerId.class), 0);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGEventTerminateDag);
        DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent;
        assertTrue(killEvent.getDiagnosticInfo().contains("ReportError"));
        assertTrue(killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name()));
        assertTrue(killEvent.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
        reset(eventHandler);
        taskCommManager.dagComplete(dag);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError"));
        assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
        assertTrue(event.getDiagnosticInfo().contains("[0:testTaskCommunicator]"));
    } finally {
        taskCommManager.stop();
    }
}

Example 25 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestTaskCommunicatorManager method testTaskCommunicatorUserError.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testTaskCommunicatorUserError() {
    TaskCommunicatorContextImpl taskCommContext = mock(TaskCommunicatorContextImpl.class);
    TaskCommunicator taskCommunicator = mock(TaskCommunicator.class, new ExceptionAnswer());
    doReturn(taskCommContext).when(taskCommunicator).getContext();
    EventHandler eventHandler = mock(EventHandler.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getEventHandler()).thenReturn(eventHandler);
    doReturn("testTaskCommunicator").when(appContext).getTaskCommunicatorName(0);
    String expectedId = "[0:testTaskCommunicator]";
    Configuration conf = new Configuration(false);
    TaskCommunicatorManager taskCommunicatorManager = new TaskCommunicatorManager(taskCommunicator, appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class));
    try {
        taskCommunicatorManager.init(conf);
        taskCommunicatorManager.start();
        // Invoking a couple of random methods.
        DAG mockDag = mock(DAG.class, RETURNS_DEEP_STUBS);
        when(mockDag.getID().getId()).thenReturn(1);
        taskCommunicatorManager.dagComplete(mockDag);
        ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(1)).handle(argumentCaptor.capture());
        Event rawEvent = argumentCaptor.getValue();
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "dagComplete"));
        assertTrue(event.getDiagnosticInfo().contains("DAG completion"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
        when(appContext.getAllContainers().get(any(ContainerId.class)).getContainer().getNodeId()).thenReturn(mock(NodeId.class));
        taskCommunicatorManager.registerRunningContainer(mock(ContainerId.class), 0);
        argumentCaptor = ArgumentCaptor.forClass(Event.class);
        verify(eventHandler, times(2)).handle(argumentCaptor.capture());
        rawEvent = argumentCaptor.getAllValues().get(1);
        assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError);
        event = (DAGAppMasterEventUserServiceFatalError) rawEvent;
        assertEquals(DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR, event.getType());
        assertTrue(event.getError().getMessage().contains("TestException_" + "registerRunningContainer"));
        assertTrue(event.getDiagnosticInfo().contains("registering running Container"));
        assertTrue(event.getDiagnosticInfo().contains(expectedId));
    } finally {
        taskCommunicatorManager.stop();
    }
}