Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 61 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class AMWebController method getVerticesInfo.

public void getVerticesInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    Collection<Integer> requestedIDs = getVertexIDsFromRequest();
    if (requestedIDs == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    Collection<Vertex> vertexList;
    if (requestedIDs.isEmpty()) {
        // no ids specified return all.
        vertexList = dag.getVertices().values();
    } else {
        vertexList = getVerticesByIdx(dag, requestedIDs);
    }
    ArrayList<Map<String, Object>> verticesInfo = new ArrayList<Map<String, Object>>();
    for (Vertex v : vertexList) {
        verticesInfo.add(getVertexInfoMap(v, counterNames));
    }
    renderJSON(ImmutableMap.of("vertices", verticesInfo));
}

Example 62 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class AMWebController method getVertexProgresses.

public void getVertexProgresses() {
    int dagID;
    setCorsHeaders();
    if (!hasAccess()) {
        sendErrorResponse(HttpServletResponse.SC_UNAUTHORIZED, "Access denied for user: " + request().getRemoteUser(), null);
        return;
    }
    List<Integer> vertexIDs = new ArrayList<Integer>();
    try {
        dagID = getQueryParamInt(WebUIService.DAG_ID);
        for (String vertexIDStr : $(WebUIService.VERTEX_ID).trim().split(",", MAX_QUERIED)) {
            vertexIDs.add(Integer.parseInt(vertexIDStr));
        }
    } catch (NumberFormatException e) {
        sendErrorResponse(HttpServletResponse.SC_BAD_REQUEST, "Invalid dag or vertices id", e);
        return;
    }
    DAG currentDAG = appContext.getCurrentDAG();
    if (currentDAG == null || currentDAG.getID().getId() != dagID) {
        sendErrorResponse(HttpServletResponse.SC_NOT_FOUND, "Not current Dag: " + dagID, null);
        return;
    }
    Collection<Vertex> vertices;
    if (vertexIDs.isEmpty()) {
        vertices = currentDAG.getVertices().values();
    } else {
        vertices = getVerticesByIdx(currentDAG, vertexIDs);
    }
    Collection<ProgressInfo> progresses = new ArrayList<ProgressInfo>(vertices.size());
    for (Vertex vertex : vertices) {
        progresses.add(new ProgressInfo(vertex.getVertexId().toString(), vertex.getCompletedTaskProgress()));
    }
    Map<String, Collection<ProgressInfo>> result = new HashMap<String, Collection<ProgressInfo>>();
    result.put(VERTEX_PROGRESSES, progresses);
    renderJSON(result);
}

Example 63 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class VertexImpl method startVertex.

private VertexState startVertex() {
    Preconditions.checkState(getState() == VertexState.INITED, "Vertex must be inited " + logIdentifier);
    if (recoveryData != null && recoveryData.isVertexStarted()) {
        VertexStartedEvent vertexStartedEvent = recoveryData.getVertexStartedEvent();
        this.startedTime = vertexStartedEvent.getStartTime();
    } else {
        this.startedTime = clock.getTime();
    }
    try {
        vertexManager.onVertexStarted(getTaskAttemptIdentifiers(dag, pendingReportedSrcCompletions));
    } catch (AMUserCodeException e) {
        String msg = "Exception in " + e.getSource() + ", vertex=" + logIdentifier;
        LOG.error(msg, e);
        addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause()));
        tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE, TaskTerminationCause.AM_USERCODE_FAILURE);
        return VertexState.TERMINATING;
    }
    pendingReportedSrcCompletions.clear();
    logJobHistoryVertexStartedEvent();
    // the vertex is fully configured by the time it starts. Always notify completely configured
    // unless the vertex manager has told us that it is going to reconfigure it further.
    // If the vertex was pre-configured then the event would have been sent out earlier. Calling again
    // would be a no-op. If the vertex was not fully configured and waiting for that to complete then
    // we would start immediately after that. Either parallelism updated (now) or IPO changed (future)
    // or vertex added (future). Simplify these cases by sending the event now automatically for the
    // user as if they had invoked the planned()/done() API's.
    maybeSendConfiguredEvent();
    // when we are ready
    if (targetVertices != null) {
        for (Vertex targetVertex : targetVertices.keySet()) {
            eventHandler.handle(new VertexEventSourceVertexStarted(targetVertex.getVertexId(), getVertexId(), distanceFromRoot));
        }
    }
    // If we have no tasks, just transition to vertex completed
    if (this.numTasks == 0) {
        eventHandler.handle(new VertexEvent(this.vertexId, VertexEventType.V_COMPLETED));
    }
    return VertexState.RUNNING;
}

Example 64 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TaskSchedulerManager method handleTaLaunchRequest.

private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
    TaskAttempt taskAttempt = event.getTaskAttempt();
    TaskLocationHint locationHint = event.getLocationHint();
    String[] hosts = null;
    String[] racks = null;
    if (locationHint != null) {
        TaskBasedLocationAffinity taskAffinity = locationHint.getAffinitizedTask();
        if (taskAffinity != null) {
            Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName());
            Preconditions.checkNotNull(vertex, "Invalid vertex in task based affinity " + taskAffinity + " for attempt: " + taskAttempt.getID());
            int taskIndex = taskAffinity.getTaskIndex();
            Preconditions.checkState(taskIndex >= 0 && taskIndex < vertex.getTotalTasks(), "Invalid taskIndex in task based affinity " + taskAffinity + " for attempt: " + taskAttempt.getID());
            TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt();
            if (affinityAttempt != null) {
                Preconditions.checkNotNull(affinityAttempt.getAssignedContainerID(), affinityAttempt.getID());
                try {
                    taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt, event.getCapability(), affinityAttempt.getAssignedContainerID(), Priority.newInstance(event.getPriority()), event.getContainerContext(), event);
                } catch (Exception e) {
                    String msg = "Error in TaskScheduler for handling Task Allocation" + ", eventType=" + event.getType() + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(event.getSchedulerId(), appContext) + ", taskAttemptId=" + taskAttempt.getID();
                    LOG.error(msg, e);
                    sendEvent(new DAGAppMasterEventUserServiceFatalError(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, msg, e));
                }
                return;
            }
            LOG.info("No attempt for task affinity to " + taskAffinity + " for attempt " + taskAttempt.getID() + " Ignoring.");
        // fall through with null hosts/racks
        } else {
            hosts = (locationHint.getHosts() != null) ? locationHint.getHosts().toArray(new String[locationHint.getHosts().size()]) : null;
            racks = (locationHint.getRacks() != null) ? locationHint.getRacks().toArray(new String[locationHint.getRacks().size()]) : null;
        }
    }
    try {
        taskSchedulers[event.getSchedulerId()].allocateTask(taskAttempt, event.getCapability(), hosts, racks, Priority.newInstance(event.getPriority()), event.getContainerContext(), event);
    } catch (Exception e) {
        String msg = "Error in TaskScheduler for handling Task Allocation" + ", eventType=" + event.getType() + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(event.getSchedulerId(), appContext) + ", taskAttemptId=" + taskAttempt.getID();
        LOG.error(msg, e);
        sendEvent(new DAGAppMasterEventUserServiceFatalError(DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR, msg, e));
    }
}

Example 65 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method initializeDAG.

DAGState initializeDAG() {
    commitAllOutputsOnSuccess = dagConf.getBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT);
    // If we have no vertices, fail the dag
    numVertices = getJobPlan().getVertexCount();
    if (numVertices == 0) {
        addDiagnostic("No vertices for dag");
        trySetTerminationCause(DAGTerminationCause.ZERO_VERTICES);
        return finished(DAGState.FAILED);
    }
    if (jobPlan.getVertexGroupsCount() > 0) {
        for (PlanVertexGroupInfo groupInfo : jobPlan.getVertexGroupsList()) {
            vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo));
        }
        for (VertexGroupInfo groupInfo : vertexGroups.values()) {
            for (String vertexName : groupInfo.groupMembers) {
                List<VertexGroupInfo> groupList = vertexGroupInfo.get(vertexName);
                if (groupList == null) {
                    groupList = Lists.newLinkedList();
                    vertexGroupInfo.put(vertexName, groupList);
                }
                groupList.add(groupInfo);
            }
        }
    }
    // create the vertices`
    for (int i = 0; i < numVertices; ++i) {
        String vertexName = getJobPlan().getVertex(i).getName();
        VertexImpl v = createVertex(this, vertexName, i);
        addVertex(v);
    }
    // check task resources, only check it in non-local mode
    if (!appContext.isLocal()) {
        for (Vertex v : vertexMap.values()) {
            // TODO TEZ-2003 (post) TEZ-2624 Ideally, this should be per source.
            if (v.getTaskResource().compareTo(appContext.getClusterInfo().getMaxContainerCapability()) > 0) {
                String msg = "Vertex's TaskResource is beyond the cluster container capability," + "Vertex=" + v.getLogIdentifier() + ", Requested TaskResource=" + v.getTaskResource() + ", Cluster MaxContainerCapability=" + appContext.getClusterInfo().getMaxContainerCapability();
                LOG.error(msg);
                addDiagnostic(msg);
                finished(DAGState.FAILED);
                return DAGState.FAILED;
            }
        }
    }
    try {
        createDAGEdges(this);
    } catch (TezException e2) {
        String msg = "Fail to create edges, " + ExceptionUtils.getStackTrace(e2);
        addDiagnostic(msg);
        LOG.error(msg);
        trySetTerminationCause(DAGTerminationCause.INIT_FAILURE);
        finished(DAGState.FAILED);
        return DAGState.FAILED;
    }
    Map<String, EdgePlan> edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(getJobPlan().getEdgeList());
    // setup the dag
    for (Vertex v : vertices.values()) {
        parseVertexEdges(this, edgePlans, v);
    }
    computeVertexDescendants();
    // Initialize the edges, now that the payload and vertices have been set.
    for (Edge e : edges.values()) {
        try {
            e.initialize();
        } catch (AMUserCodeException ex) {
            String msg = "Exception in " + ex.getSource();
            LOG.error(msg, ex);
            addDiagnostic(msg + ", " + ex.getMessage() + ", " + ExceptionUtils.getStackTrace(ex.getCause()));
            finished(DAGState.FAILED);
            return DAGState.FAILED;
        }
    }
    try {
        assignDAGScheduler(this);
    } catch (TezException e1) {
        String msg = "Fail to assign DAGScheduler for dag:" + dagName + " due to " + ExceptionUtils.getStackTrace(e1);
        LOG.error(msg);
        addDiagnostic(msg);
        trySetTerminationCause(DAGTerminationCause.INIT_FAILURE);
        finished(DAGState.FAILED);
        return DAGState.FAILED;
    }
    for (Map.Entry<String, VertexGroupInfo> entry : vertexGroups.entrySet()) {
        String groupName = entry.getKey();
        VertexGroupInfo groupInfo = entry.getValue();
        if (!groupInfo.outputs.isEmpty()) {
            // shared outputs
            for (String vertexName : groupInfo.groupMembers) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Setting shared outputs for group: " + groupName + " on vertex: " + vertexName);
                }
                Vertex v = getVertex(vertexName);
                v.addSharedOutputs(groupInfo.outputs);
            }
        }
    }
    return DAGState.INITED;
}