Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 71 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGImpl method getCompletedTaskProgress.

@Override
public float getCompletedTaskProgress() {
    this.readLock.lock();
    try {
        int totalTasks = 0;
        int completedTasks = 0;
        for (Vertex v : getVertices().values()) {
            int vTotalTasks = v.getTotalTasks();
            int vCompletedTasks = v.getSucceededTasks();
            if (vTotalTasks > 0) {
                totalTasks += vTotalTasks;
                completedTasks += vCompletedTasks;
            }
        }
        if (totalTasks == 0) {
            DAGState state = getStateMachine().getCurrentState();
            if (state == DAGState.ERROR || state == DAGState.FAILED || state == DAGState.KILLED || state == DAGState.SUCCEEDED) {
                return 1.0f;
            } else {
                return 0.0f;
            }
        }
        return ((float) completedTasks / totalTasks);
    } finally {
        this.readLock.unlock();
    }
}

Example 72 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGSchedulerNaturalOrderControlled method scheduleTaskEx.

// TODO Does ordering matter - it currently depends on the order returned by vertex.getOutput*
@Override
public void scheduleTaskEx(DAGEventSchedulerUpdate event) {
    TaskAttempt attempt = event.getAttempt();
    Vertex vertex = dag.getVertex(attempt.getVertexID());
    int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
    // natural priority. Handles failures and retries.
    int priorityLowLimit = ((vertexDistanceFromRoot + 1) * dag.getTotalVertices() * 3) + (vertex.getVertexId().getId() * 3);
    int priorityHighLimit = priorityLowLimit - 2;
    TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(attempt.getID(), priorityLowLimit, priorityHighLimit);
    taskAttemptSeen(vertex.getName(), attempt.getID());
    if (vertexAlreadyScheduled(vertex)) {
        // Vertex previously marked ready for scheduling.
        if (LOG.isDebugEnabled()) {
            LOG.debug("Scheduling " + attempt.getID() + " between priorityLow: " + priorityLowLimit + " and priorityHigh: " + priorityHighLimit);
        }
        sendEvent(attemptEvent);
        // A new taks coming in here could send us over the enough tasks scheduled limit.
        processDownstreamVertices(vertex);
    } else {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Attempting to schedule vertex: " + vertex.getLogIdentifier() + " due to schedule event");
        }
        boolean scheduled = trySchedulingVertex(vertex);
        if (scheduled) {
            LOG.info("Scheduled vertex: " + vertex.getLogIdentifier());
            // If ready to be scheduled, send out pending events and the current event.
            // Send events out for this vertex first. Then try scheduling downstream vertices.
            sendEventsForVertex(vertex.getName());
            sendEvent(attemptEvent);
            if (LOG.isDebugEnabled()) {
                LOG.debug("Processing downstream vertices for vertex: " + vertex.getLogIdentifier());
            }
            processDownstreamVertices(vertex);
        } else {
            pendingEvents.put(vertex.getName(), attemptEvent);
        }
    }
}

Example 73 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGSchedulerNaturalOrderControlled method processDownstreamVertices.

private void processDownstreamVertices(Vertex vertex) {
    List<Vertex> newlyScheduledVertices = Lists.newLinkedList();
    Map<Vertex, Edge> outputVertexEdgeMap = vertex.getOutputVertices();
    for (Vertex destVertex : outputVertexEdgeMap.keySet()) {
        if (vertexAlreadyScheduled(destVertex)) {
        // Nothing to do if already scheduled.
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Attempting to schedule vertex: " + destVertex.getLogIdentifier() + " due to upstream event from " + vertex.getLogIdentifier());
            }
            boolean scheduled = trySchedulingVertex(destVertex);
            if (scheduled) {
                LOG.info("Scheduled vertex: " + destVertex.getLogIdentifier() + " due to upstream event from " + vertex.getLogIdentifier());
                sendEventsForVertex(destVertex.getName());
                newlyScheduledVertices.add(destVertex);
            }
        }
    }
    // Try scheduling all downstream vertices which were scheduled in this run.
    for (Vertex downStreamVertex : newlyScheduledVertices) {
        processDownstreamVertices(downStreamVertex);
    }
}

Example 74 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class DAGSchedulerNaturalOrder method scheduleTaskEx.

@Override
public void scheduleTaskEx(DAGEventSchedulerUpdate event) {
    TaskAttempt attempt = event.getAttempt();
    Vertex vertex = dag.getVertex(attempt.getVertexID());
    int vertexDistanceFromRoot = vertex.getDistanceFromRoot();
    // natural priority. Handles failures and retries.
    int priorityLowLimit = ((vertexDistanceFromRoot + 1) * dag.getTotalVertices() * 3) + (vertex.getVertexId().getId() * 3);
    int priorityHighLimit = priorityLowLimit - 2;
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scheduling " + attempt.getID() + " between priorityLow: " + priorityLowLimit + " and priorityHigh: " + priorityHighLimit);
    }
    TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule(attempt.getID(), priorityLowLimit, priorityHighLimit);
    sendEvent(attemptEvent);
}

Example 75 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TaskAttemptImpl method sendInputFailedToConsumers.

@VisibleForTesting
protected void sendInputFailedToConsumers() {
    Vertex vertex = getVertex();
    Map<Vertex, Edge> edges = vertex.getOutputVertices();
    if (edges != null && !edges.isEmpty()) {
        List<TezEvent> tezIfEvents = Lists.newArrayListWithCapacity(edges.size());
        for (Vertex edgeVertex : edges.keySet()) {
            tezIfEvents.add(new TezEvent(new InputFailedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, vertex.getName(), edgeVertex.getName(), getID()), appContext.getClock().getTime()));
        }
        sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents));
    }
}