Examples with DAG org.apache.tez.dag.app.dag.DAG used on opensource projects

Example 31 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class TestTaskCommunicatorManager1 method setUp.

@Before
public void setUp() throws TezException {
    appId = ApplicationId.newInstance(1000, 1);
    appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    dag = mock(DAG.class);
    TezDAGID dagID = TezDAGID.getInstance(appId, 1);
    vertexID = TezVertexID.getInstance(dagID, 1);
    taskID = TezTaskID.getInstance(vertexID, 1);
    taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
    credentials = new Credentials();
    amContainerMap = mock(AMContainerMap.class);
    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
    eventHandler = mock(EventHandler.class);
    MockClock clock = new MockClock();
    appContext = mock(AppContext.class);
    doReturn(eventHandler).when(appContext).getEventHandler();
    doReturn(dag).when(appContext).getCurrentDAG();
    doReturn(appAcls).when(appContext).getApplicationACLs();
    doReturn(amContainerMap).when(appContext).getAllContainers();
    doReturn(clock).when(appContext).getClock();
    doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
    doReturn(credentials).when(appContext).getAppCredentials();
    NodeId nodeId = NodeId.newInstance("localhost", 0);
    AMContainer amContainer = mock(AMContainer.class);
    Container container = mock(Container.class);
    doReturn(nodeId).when(container).getNodeId();
    doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class));
    doReturn(container).when(amContainer).getContainer();
    Configuration conf = new TezConfiguration();
    UserPayload defaultPayload;
    try {
        defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    } catch (IOException e) {
        throw new TezUncheckedException(e);
    }
    taskAttemptListener = new TaskCommunicatorManagerInterfaceImplForTest(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload)));
    taskSpec = mock(TaskSpec.class);
    doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
    amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0);
    containerTask = null;
}

Example 32 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class AMWebController method getVerticesInfo.

public void getVerticesInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    Collection<Integer> requestedIDs = getVertexIDsFromRequest();
    if (requestedIDs == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    Collection<Vertex> vertexList;
    if (requestedIDs.isEmpty()) {
        // no ids specified return all.
        vertexList = dag.getVertices().values();
    } else {
        vertexList = getVerticesByIdx(dag, requestedIDs);
    }
    ArrayList<Map<String, Object>> verticesInfo = new ArrayList<Map<String, Object>>();
    for (Vertex v : vertexList) {
        verticesInfo.add(getVertexInfoMap(v, counterNames));
    }
    renderJSON(ImmutableMap.of("vertices", verticesInfo));
}

Example 33 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class AMWebController method getAttemptsInfo.

/**
 * Renders the response JSON for attemptsInfo API
 * The JSON will have an array of attempt objects under the key attempts.
 */
public void getAttemptsInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    int limit = MAX_QUERIED;
    try {
        limit = getQueryParamInt(WebUIService.LIMIT);
    } catch (NumberFormatException e) {
    // Ignore
    }
    List<TaskAttempt> attempts = getRequestedAttempts(dag, limit);
    if (attempts == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    ArrayList<Map<String, Object>> attemptsInfo = new ArrayList<Map<String, Object>>();
    for (TaskAttempt a : attempts) {
        Map<String, Object> attemptInfo = new HashMap<String, Object>();
        attemptInfo.put("id", a.getID().toString());
        attemptInfo.put("progress", Float.toString(a.getProgress()));
        attemptInfo.put("status", a.getState().toString());
        try {
            TezCounters counters = a.getCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                attemptInfo.put("counters", counterMap);
            }
        } catch (LimitExceededException e) {
        // Ignore
        // TODO: add an error message instead for counter key
        }
        attemptsInfo.add(attemptInfo);
    }
    renderJSON(ImmutableMap.of("attempts", attemptsInfo));
}

Example 34 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class AMWebController method getVertexProgresses.

public void getVertexProgresses() {
    int dagID;
    setCorsHeaders();
    if (!hasAccess()) {
        sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED, "Access denied for user: " + request().getRemoteUser(), null);
        return;
    }
    List<Integer> vertexIDs = new ArrayList<Integer>();
    try {
        dagID = getQueryParamInt(WebUIService.DAG_ID);
        for (String vertexIDStr : $(WebUIService.VERTEX_ID).trim().split(",", MAX_QUERIED)) {
            vertexIDs.add(Integer.parseInt(vertexIDStr));
        }
    } catch (NumberFormatException e) {
        sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag or vertices id", e);
        return;
    }
    DAG currentDAG = appContext.getCurrentDAG();
    if (currentDAG == null || currentDAG.getID().getId() != dagID) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
        return;
    }
    Collection<Vertex> vertices;
    if (vertexIDs.isEmpty()) {
        vertices = currentDAG.getVertices().values();
    } else {
        vertices = getVerticesByIdx(currentDAG, vertexIDs);
    }
    Collection<ProgressInfo> progresses = new ArrayList<ProgressInfo>(vertices.size());
    for (Vertex vertex : vertices) {
        progresses.add(new ProgressInfo(vertex.getVertexId().toString(), vertex.getCompletedTaskProgress()));
    }
    Map<String, Collection<ProgressInfo>> result = new HashMap<String, Collection<ProgressInfo>>();
    result.put(VERTEX_PROGRESSES, progresses);
    renderJSON(result);
}

Example 35 with DAG

use of org.apache.tez.dag.app.dag.DAG in project tez by apache.

the class AMWebController method checkAndGetDAGFromRequest.

DAG checkAndGetDAGFromRequest() {
    DAG dag = null;
    int errorCode = HttpServletResponse.SC_OK;
    String message = null;
    Exception ex = null;
    try {
        int dagID = getQueryParamInt(WebUIService.DAG_ID);
        dag = appContext.getCurrentDAG();
        if (dag == null || dag.getID().getId() != dagID) {
            errorCode = HttpServletResponse.SC_NOT_FOUND;
            message = "Not current Dag: " + dagID;
        }
    } catch (NumberFormatException e) {
        errorCode = HttpServletResponse.SC_BAD_REQUEST;
        message = "Invalid dag id";
        ex = e;
    }
    if (errorCode != HttpServletResponse.SC_OK) {
        dag = null;
        sendErrorResponse(errorCode, message, ex);
    }
    return dag;
}